From 9cd55fbc0faf94dbf3754d702fc2a5b64a9078ac Mon Sep 17 00:00:00 2001 From: Sherjeel Shabih Date: Fri, 7 Oct 2022 10:29:09 +0200 Subject: [PATCH 01/93] Better dependency management for dev, standalone, and a Nomad install (#50) * Added a pip-compile locked requirements for devs * Reverted nomad-lab install back * Fixed missing astroid version * cleanup + ease on dev reqs * Added a relaxed requirements.txt for installation with Nomad * Unrelaxed pyproject.toml dev reqs and reverted xarray to older version * Allowed python3.7 and fixed dask[array] version * Merge adapt-nexus-to-metainfo-changes and CI updates * cleanup * Brought in requirements.txt * draft refactoring and reorganizing of nomad-parser-nexus, decide first how tests should finally before structured, then the tests and absolute paths in the ipynbs and code sections should be changed * renaming tools; adding and fixing unit tests * fix some linting issues * addig reference data for testing nexus checker * fixing linting issues; adding test reference file * bringing Example to front * tested ipynb notebooks, added jupyterlab_h5web to the environment * improved descriptions, and placed jupyterlab in pyproject toml, tested compiling a dev-requirements locally successfully * updating readme.md as a result of testing * Added nexus definitions to the module package * fixing README files and the ipynb for APM * use dataconverter alias from command line * harmonising EM and MPES notebooks * Updated YAML file * Updated jupyter notebook for ellipsometry + DAT and YAML file * Delete ELLIPSOMETRY.NeXus.READER.EXAMPLE.02.ipynb * Delete test.yaml * fix data file locations Co-authored-by: Adam Fekete Co-authored-by: Adam Fekete Co-authored-by: kuehbachm Co-authored-by: sanbrock Co-authored-by: sanbrock <45483558+sanbrock@users.noreply.github.com> Co-authored-by: Carola Emminger Co-authored-by: cmmngr <98404894+cmmngr@users.noreply.github.com> --- README.md | 4 + test_nyaml2nxdl.py | 295 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 299 insertions(+) create mode 100644 README.md create mode 100755 test_nyaml2nxdl.py diff --git a/README.md b/README.md new file mode 100644 index 0000000000..65de76318d --- /dev/null +++ b/README.md @@ -0,0 +1,4 @@ +This is the place for storing code for tests of the yaml2nxdl and nxdl2yaml NeXus schema translation routines. + +## Contact person in FAIRmat for these tests +Andrea Albino \ No newline at end of file diff --git a/test_nyaml2nxdl.py b/test_nyaml2nxdl.py new file mode 100755 index 0000000000..bdb7b0625e --- /dev/null +++ b/test_nyaml2nxdl.py @@ -0,0 +1,295 @@ +#!/usr/bin/env python3 +"""This tool accomplishes some tests for the yaml2nxdl parser + +""" +# +# Copyright The NOMAD Authors. +# +# This file is part of NOMAD. See https://nomad-lab.eu for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import sys +import filecmp +from datetime import datetime +from pathlib import Path +import xml.etree.ElementTree as ET +import pytest +from click.testing import CliRunner +import nexusutils.nyaml2nxdl.nyaml2nxdl as nyml2nxdl +from nexusutils.nyaml2nxdl import nyaml2nxdl_forward_tools + + +def delete_duplicates(list_of_matching_string): + """Delete duplicate from lists + +""" + return list(dict.fromkeys(list_of_matching_string)) + + +def check_file_fresh_baked(test_file): + """Get sure that the test file is generated by the converter + +""" + path = Path(test_file) + timestamp = datetime.fromtimestamp(path.stat().st_mtime).strftime("%d/%m/%Y %H:%M") + now = datetime.now().strftime("%d/%m/%Y %H:%M") + assert timestamp == now, 'xml file not generated' + + +def find_matches(xml_file, desired_matches): + """Read xml file and find desired matches. +Return a list of two lists in the form: +[[matching_line],[matching_line_index]] + +""" + with open(xml_file, 'r') as file: + xml_reference = file.readlines() + lines = [] + lines_index = [] + found_matches = [] + for i, line in enumerate(xml_reference): + for desired_match in desired_matches: + if str(desired_match) in str(line): + lines.append(line) + lines_index.append(i) + found_matches.append(desired_match) + # ascertain that all the desired matches were found in file + found_matches_clean = delete_duplicates(found_matches) + assert len(found_matches_clean) == len(desired_matches), 'some desired_matches were \ +not found in file' + return [lines, lines_index] + + +@pytest.mark.parametrize( +) +def compare_matches(ref_xml_file, test_yml_file, test_xml_file, desired_matches): + """Check if a new xml file is generated +and if test xml file is equal to reference xml file + +""" + # Reference file is read + ref_matches = find_matches(ref_xml_file, desired_matches) + # Test file is generated + runner = CliRunner() + result = runner.invoke(nyml2nxdl.launch_tool, ['--input-file', test_yml_file]) + assert result.exit_code == 0 + check_file_fresh_baked(test_xml_file) + # Test file is read + test_matches = find_matches(test_xml_file, desired_matches) + assert test_matches == ref_matches + + +def test_links(): + """Check the correct parsing of links + +""" + ref_xml_link_file = 'tests/data/nyaml2nxdl/Ref_NXtest_links.nxdl.xml' + test_yml_link_file = 'tests/data/nyaml2nxdl/NXtest_links.yml' + test_xml_link_file = 'tests/data/nyaml2nxdl/NXtest_links.nxdl.xml' + desired_matches = [''] + compare_matches( + ref_xml_link_file, + test_yml_link_file, + test_xml_link_file, + desired_matches) + os.remove('tests/data/nyaml2nxdl/NXtest_links.nxdl.xml') + sys.stdout.write('Test on links okay.\n') + + +def test_docs(): + """In this test an xml file in converted to yml and then back to xml. +The xml trees of the two files are then compared. + + """ + ref_xml_file = 'tests/data/nyaml2nxdl/Ref_NXellipsometry-docCheck.nxdl.xml' + test_yml_file = 'tests/data/nyaml2nxdl/NXellipsometry-docCheck.yaml' + test_xml_file = 'tests/data/nyaml2nxdl/NXellipsometry-docCheck.nxdl.xml' + desired_matches = [''] + compare_matches( + ref_xml_file, + test_yml_file, + test_xml_file, + desired_matches) + os.remove('tests/data/nyaml2nxdl/NXellipsometry-docCheck.nxdl.xml') + sys.stdout.write('Test on documentation formatting okay.\n') + + +def test_nxdl2yaml_doc_format(): + """In this test an nxdl file with all kind of doc formats are translated to yaml + to check if they are correct. + + """ + ref_xml_file = 'tests/data/nyaml2nxdl/Ref_NXentry.nxdl.xml' + ref_yml_file = 'tests/data/nyaml2nxdl/Ref_NXentry.yml' + test_yml_file = 'tests/data/nyaml2nxdl/Ref_NXentry_parsed.yml' + result = CliRunner().invoke(nyml2nxdl.launch_tool, ['--input-file', ref_xml_file]) + assert result.exit_code == 0 + check_file_fresh_baked(test_yml_file) + + result = filecmp.cmp(ref_yml_file, test_yml_file, shallow=False) + assert result, 'Ref YML and parsed YML\ +has not the same structure!!' + os.remove(test_yml_file) + sys.stdout.write('Test on xml -> yml doc formatting okay.\n') + + +def test_fileline_error(): + """In this test the yaml fileline in the error message is tested. + + """ + test_yml_file = 'tests/data/nyaml2nxdl/NXfilelineError1.yml' + result = CliRunner().invoke(nyml2nxdl.launch_tool, ['--input-file', test_yml_file]) + assert result.exit_code == 1 + assert '13' in str(result.exception) + + test_yml_file = 'tests/data/nyaml2nxdl/NXfilelineError2.yml' + result = CliRunner().invoke(nyml2nxdl.launch_tool, ['--input-file', test_yml_file]) + assert result.exit_code == 1 + assert '21' in str(result.exception) + + test_yml_file = 'tests/data/nyaml2nxdl/NXfilelineError3.yml' + result = CliRunner().invoke(nyml2nxdl.launch_tool, ['--input-file', test_yml_file]) + assert result.exit_code == 1 + assert '25' in str(result.exception) + + sys.stdout.write('Test on xml -> yml fileline error handling okay.\n') + + +def test_symbols(): + """Check the correct parsing of symbols + +""" + ref_xml_symbol_file = 'tests/data/nyaml2nxdl/Ref_NXnested_symbols.nxdl.xml' + test_yml_symbol_file = 'tests/data/nyaml2nxdl/NXnested_symbols.yml' + test_xml_symbol_file = 'tests/data/nyaml2nxdl/NXnested_symbols.nxdl.xml' + desired_matches = ['', '', '', '', ''] + compare_matches( + ref_xml_attribute_file, + test_yml_attribute_file, + test_xml_attribute_file, + desired_matches) + os.remove('tests/data/nyaml2nxdl/NXattributes.nxdl.xml') + sys.stdout.write('Test on attributes okay.\n') + + +def test_extends(): + """Check the correct handling of extends keyword + +""" + ref_xml_attribute_file = 'tests/data/nyaml2nxdl/Ref_NXattributes.nxdl.xml' + test_yml_attribute_file = 'tests/data/nyaml2nxdl/NXattributes.yml' + test_xml_attribute_file = 'tests/data/nyaml2nxdl/NXattributes.nxdl.xml' + runner = CliRunner() + result = runner.invoke(nyml2nxdl.launch_tool, ['--input-file', test_yml_attribute_file]) + assert result.exit_code == 0 + ref_root_node = ET.parse(ref_xml_attribute_file).getroot() + test_root_node = ET.parse(test_xml_attribute_file).getroot() + assert ref_root_node.attrib == test_root_node.attrib + os.remove('tests/data/nyaml2nxdl/NXattributes.nxdl.xml') + sys.stdout.write('Test on extends keyword okay.\n') + + +def test_symbols_and_enum_docs(): + """Check the correct handling of empty attributes + or attributes fields, e.g. doc + +""" + ref_xml_file = 'tests/data/nyaml2nxdl/Ref_NXmytests.nxdl.xml' + test_yml_file = 'tests/data/nyaml2nxdl/NXmytests.yml' + test_xml_file = 'tests/data/nyaml2nxdl/NXmytests.nxdl.xml' + desired_matches = ['', '', '', + '', '', '', ' yml -> xml okay.\n') + + +def test_yml_parsing(): + """In this test an xml file in converted to yml and then back to xml. +The xml trees of the two files are then compared. + + """ + ref_yml_file = 'tests/data/nyaml2nxdl/Ref_NXellipsometry.yml' + test_xml_file = 'tests/data/nyaml2nxdl/Ref_NXellipsometry.nxdl.xml' + test_yml_file = 'tests/data/nyaml2nxdl/Ref_NXellipsometry_parsed.yml' + result = CliRunner().invoke(nyml2nxdl.launch_tool, ['--input-file', ref_yml_file]) + assert result.exit_code == 0 + check_file_fresh_baked(test_xml_file) + result = CliRunner().invoke(nyml2nxdl.launch_tool, ['--input-file', test_xml_file]) + assert result.exit_code == 0 + check_file_fresh_baked(test_yml_file) + + test_yml_tree = nyaml2nxdl_forward_tools.yml_reader(test_yml_file) + + ref_yml_tree = nyaml2nxdl_forward_tools.yml_reader(ref_yml_file) + + assert list(test_yml_tree) == list(ref_yml_tree), 'Ref YML and parsed YML \ +has not the same root entries!!' + os.remove('tests/data/nyaml2nxdl/Ref_NXellipsometry_parsed.yml') + os.remove('tests/data/nyaml2nxdl/Ref_NXellipsometry.nxdl.xml') + sys.stdout.write('Test on yml -> xml -> yml okay.\n') From 3627be4a4fe1c3106ae866e7364a4dc9a7b90f45 Mon Sep 17 00:00:00 2001 From: domna Date: Thu, 20 Oct 2022 20:18:39 +0200 Subject: [PATCH 02/93] Fixes linting --- test_nyaml2nxdl.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/test_nyaml2nxdl.py b/test_nyaml2nxdl.py index bdb7b0625e..e401289674 100755 --- a/test_nyaml2nxdl.py +++ b/test_nyaml2nxdl.py @@ -73,8 +73,6 @@ def find_matches(xml_file, desired_matches): return [lines, lines_index] -@pytest.mark.parametrize( -) def compare_matches(ref_xml_file, test_yml_file, test_xml_file, desired_matches): """Check if a new xml file is generated and if test xml file is equal to reference xml file From 19f093754d6eabf28fe133968d255425316d41b4 Mon Sep 17 00:00:00 2001 From: RubelMozumder Date: Thu, 2 Mar 2023 00:08:35 +0100 Subject: [PATCH 03/93] Renaming file from yml to yaml --- test_nyaml2nxdl.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/test_nyaml2nxdl.py b/test_nyaml2nxdl.py index e401289674..4628ea8e6e 100755 --- a/test_nyaml2nxdl.py +++ b/test_nyaml2nxdl.py @@ -74,10 +74,10 @@ def find_matches(xml_file, desired_matches): def compare_matches(ref_xml_file, test_yml_file, test_xml_file, desired_matches): - """Check if a new xml file is generated -and if test xml file is equal to reference xml file - -""" + """ + Check if a new xml file is generated + and if test xml file is equal to reference xml file + """ # Reference file is read ref_matches = find_matches(ref_xml_file, desired_matches) # Test file is generated @@ -95,7 +95,7 @@ def test_links(): """ ref_xml_link_file = 'tests/data/nyaml2nxdl/Ref_NXtest_links.nxdl.xml' - test_yml_link_file = 'tests/data/nyaml2nxdl/NXtest_links.yml' + test_yml_link_file = 'tests/data/nyaml2nxdl/NXtest_links.yaml' test_xml_link_file = 'tests/data/nyaml2nxdl/NXtest_links.nxdl.xml' desired_matches = [''] compare_matches( @@ -126,13 +126,13 @@ def test_docs(): def test_nxdl2yaml_doc_format(): - """In this test an nxdl file with all kind of doc formats are translated to yaml - to check if they are correct. - + """ + In this test an nxdl file with all kind of doc formats are translated + to yaml to check if they are correct. """ ref_xml_file = 'tests/data/nyaml2nxdl/Ref_NXentry.nxdl.xml' ref_yml_file = 'tests/data/nyaml2nxdl/Ref_NXentry.yml' - test_yml_file = 'tests/data/nyaml2nxdl/Ref_NXentry_parsed.yml' + test_yml_file = 'tests/data/nyaml2nxdl/Ref_NXentry_parsed.yaml' result = CliRunner().invoke(nyml2nxdl.launch_tool, ['--input-file', ref_xml_file]) assert result.exit_code == 0 check_file_fresh_baked(test_yml_file) @@ -148,7 +148,7 @@ def test_fileline_error(): """In this test the yaml fileline in the error message is tested. """ - test_yml_file = 'tests/data/nyaml2nxdl/NXfilelineError1.yml' + test_yml_file = 'tests/data/nyaml2nxdl/NXfilelineError1.yaml' result = CliRunner().invoke(nyml2nxdl.launch_tool, ['--input-file', test_yml_file]) assert result.exit_code == 1 assert '13' in str(result.exception) @@ -202,9 +202,9 @@ def test_attributes(): def test_extends(): - """Check the correct handling of extends keyword - -""" + """ + Check the correct handling of extends keyword + """ ref_xml_attribute_file = 'tests/data/nyaml2nxdl/Ref_NXattributes.nxdl.xml' test_yml_attribute_file = 'tests/data/nyaml2nxdl/NXattributes.yml' test_xml_attribute_file = 'tests/data/nyaml2nxdl/NXattributes.nxdl.xml' From 86bbc9c01aa6f6ca0567831da798f43ffdb4ca67 Mon Sep 17 00:00:00 2001 From: RubelMozumder Date: Thu, 2 Mar 2023 15:06:29 +0100 Subject: [PATCH 04/93] passing test: test_link, test_docs, test_nxdl2nyaml_doc_from_nxdl, test_fileline_error, test_symnols, test_attributes. test_extends --- test_nyaml2nxdl.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test_nyaml2nxdl.py b/test_nyaml2nxdl.py index 4628ea8e6e..e5905e46bb 100755 --- a/test_nyaml2nxdl.py +++ b/test_nyaml2nxdl.py @@ -131,7 +131,7 @@ def test_nxdl2yaml_doc_format(): to yaml to check if they are correct. """ ref_xml_file = 'tests/data/nyaml2nxdl/Ref_NXentry.nxdl.xml' - ref_yml_file = 'tests/data/nyaml2nxdl/Ref_NXentry.yml' + ref_yml_file = 'tests/data/nyaml2nxdl/Ref_NXentry.yaml' test_yml_file = 'tests/data/nyaml2nxdl/Ref_NXentry_parsed.yaml' result = CliRunner().invoke(nyml2nxdl.launch_tool, ['--input-file', ref_xml_file]) assert result.exit_code == 0 @@ -153,12 +153,12 @@ def test_fileline_error(): assert result.exit_code == 1 assert '13' in str(result.exception) - test_yml_file = 'tests/data/nyaml2nxdl/NXfilelineError2.yml' + test_yml_file = 'tests/data/nyaml2nxdl/NXfilelineError2.yaml' result = CliRunner().invoke(nyml2nxdl.launch_tool, ['--input-file', test_yml_file]) assert result.exit_code == 1 assert '21' in str(result.exception) - test_yml_file = 'tests/data/nyaml2nxdl/NXfilelineError3.yml' + test_yml_file = 'tests/data/nyaml2nxdl/NXfilelineError3.yaml' result = CliRunner().invoke(nyml2nxdl.launch_tool, ['--input-file', test_yml_file]) assert result.exit_code == 1 assert '25' in str(result.exception) @@ -189,7 +189,7 @@ def test_attributes(): """ ref_xml_attribute_file = 'tests/data/nyaml2nxdl/Ref_NXattributes.nxdl.xml' - test_yml_attribute_file = 'tests/data/nyaml2nxdl/NXattributes.yml' + test_yml_attribute_file = 'tests/data/nyaml2nxdl/NXattributes.yaml' test_xml_attribute_file = 'tests/data/nyaml2nxdl/NXattributes.nxdl.xml' desired_matches = ['', '', ''] compare_matches( From 42125416241d79b8e43a671e2c07f851b36fac50 Mon Sep 17 00:00:00 2001 From: RubelMozumder Date: Thu, 2 Mar 2023 15:56:53 +0100 Subject: [PATCH 05/93] tests in going on --- test_nyaml2nxdl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_nyaml2nxdl.py b/test_nyaml2nxdl.py index e5905e46bb..b0057d24bf 100755 --- a/test_nyaml2nxdl.py +++ b/test_nyaml2nxdl.py @@ -224,7 +224,7 @@ def test_symbols_and_enum_docs(): """ ref_xml_file = 'tests/data/nyaml2nxdl/Ref_NXmytests.nxdl.xml' - test_yml_file = 'tests/data/nyaml2nxdl/NXmytests.yml' + test_yml_file = 'tests/data/nyaml2nxdl/NXmytests.yaml' test_xml_file = 'tests/data/nyaml2nxdl/NXmytests.nxdl.xml' desired_matches = ['', '', '', '', '', ' Date: Thu, 2 Mar 2023 20:59:04 +0100 Subject: [PATCH 06/93] test_xml_parsing is passed. --- test_nyaml2nxdl.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test_nyaml2nxdl.py b/test_nyaml2nxdl.py index b0057d24bf..f34742d784 100755 --- a/test_nyaml2nxdl.py +++ b/test_nyaml2nxdl.py @@ -206,7 +206,7 @@ def test_extends(): Check the correct handling of extends keyword """ ref_xml_attribute_file = 'tests/data/nyaml2nxdl/Ref_NXattributes.nxdl.xml' - test_yml_attribute_file = 'tests/data/nyaml2nxdl/NXattributes.yml' + test_yml_attribute_file = 'tests/data/nyaml2nxdl/NXattributes.yaml' test_xml_attribute_file = 'tests/data/nyaml2nxdl/NXattributes.nxdl.xml' runner = CliRunner() result = runner.invoke(nyml2nxdl.launch_tool, ['--input-file', test_yml_attribute_file]) @@ -244,7 +244,7 @@ def test_xml_parsing(): """ ref_xml_file = 'tests/data/nyaml2nxdl/Ref_NXellips.nxdl.xml' - test_yml_file = 'tests/data/nyaml2nxdl/Ref_NXellips_parsed.yml' + test_yml_file = 'tests/data/nyaml2nxdl/Ref_NXellips_parsed.yaml' test_xml_file = 'tests/data/nyaml2nxdl/\ Ref_NXellips_parsed.nxdl.xml' result = CliRunner().invoke(nyml2nxdl.launch_tool, ['--input-file', ref_xml_file]) @@ -255,15 +255,15 @@ def test_xml_parsing(): check_file_fresh_baked(test_xml_file) test_tree = ET.parse(test_xml_file) - test_tree_flattened = [i.tag.split("}", 1)[1] for i in test_tree.iter()] + test_tree_flattened = set([i.tag.split("}", 1)[1] for i in test_tree.iter()]) ref_tree = ET.parse(ref_xml_file) - ref_tree_flattened = [i.tag.split("}", 1)[1] for i in ref_tree.iter()] + ref_tree_flattened = set([i.tag.split("}", 1)[1] for i in ref_tree.iter()]) - assert set(test_tree_flattened) == set(ref_tree_flattened), 'Ref XML and parsed XML\ + assert test_tree_flattened == ref_tree_flattened, 'Ref XML and parsed XML\ has not the same tree structure!!' os.remove('tests/data/nyaml2nxdl/Ref_NXellips_parsed.nxdl.xml') - os.remove('tests/data/nyaml2nxdl/Ref_NXellips_parsed.yml') + os.remove('tests/data/nyaml2nxdl/Ref_NXellips_parsed.yaml') sys.stdout.write('Test on xml -> yml -> xml okay.\n') @@ -274,7 +274,7 @@ def test_yml_parsing(): """ ref_yml_file = 'tests/data/nyaml2nxdl/Ref_NXellipsometry.yml' test_xml_file = 'tests/data/nyaml2nxdl/Ref_NXellipsometry.nxdl.xml' - test_yml_file = 'tests/data/nyaml2nxdl/Ref_NXellipsometry_parsed.yml' + test_yml_file = 'tests/data/nyaml2nxdl/Ref_NXellipsometry_parsed.yaml' result = CliRunner().invoke(nyml2nxdl.launch_tool, ['--input-file', ref_yml_file]) assert result.exit_code == 0 check_file_fresh_baked(test_xml_file) @@ -288,6 +288,6 @@ def test_yml_parsing(): assert list(test_yml_tree) == list(ref_yml_tree), 'Ref YML and parsed YML \ has not the same root entries!!' - os.remove('tests/data/nyaml2nxdl/Ref_NXellipsometry_parsed.yml') + os.remove('tests/data/nyaml2nxdl/Ref_NXellipsometry_parsed.yaml') os.remove('tests/data/nyaml2nxdl/Ref_NXellipsometry.nxdl.xml') sys.stdout.write('Test on yml -> xml -> yml okay.\n') From 8fb31021577a046ac0728f4b89a339e5baa5182e Mon Sep 17 00:00:00 2001 From: RubelMozumder Date: Thu, 2 Mar 2023 21:04:45 +0100 Subject: [PATCH 07/93] All tests are successfully passed. --- test_nyaml2nxdl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_nyaml2nxdl.py b/test_nyaml2nxdl.py index f34742d784..5790963100 100755 --- a/test_nyaml2nxdl.py +++ b/test_nyaml2nxdl.py @@ -272,7 +272,7 @@ def test_yml_parsing(): The xml trees of the two files are then compared. """ - ref_yml_file = 'tests/data/nyaml2nxdl/Ref_NXellipsometry.yml' + ref_yml_file = 'tests/data/nyaml2nxdl/Ref_NXellipsometry.yaml' test_xml_file = 'tests/data/nyaml2nxdl/Ref_NXellipsometry.nxdl.xml' test_yml_file = 'tests/data/nyaml2nxdl/Ref_NXellipsometry_parsed.yaml' result = CliRunner().invoke(nyml2nxdl.launch_tool, ['--input-file', ref_yml_file]) From efcd7937dbe82213009d3e484d18cb4c8e3016c0 Mon Sep 17 00:00:00 2001 From: RubelMozumder Date: Wed, 22 Mar 2023 17:40:03 +0100 Subject: [PATCH 08/93] Setting all the tests, mypy, pydocstyle, and pylint correctly. --- test_nyaml2nxdl.py | 66 ++++++++++++++++++++++------------------------ 1 file changed, 31 insertions(+), 35 deletions(-) diff --git a/test_nyaml2nxdl.py b/test_nyaml2nxdl.py index 5790963100..57c3b1f655 100755 --- a/test_nyaml2nxdl.py +++ b/test_nyaml2nxdl.py @@ -33,16 +33,16 @@ def delete_duplicates(list_of_matching_string): - """Delete duplicate from lists - -""" + """ + Delete duplicate from lists + """ return list(dict.fromkeys(list_of_matching_string)) def check_file_fresh_baked(test_file): - """Get sure that the test file is generated by the converter - -""" + """ + Get sure that the test file is generated by the converter + """ path = Path(test_file) timestamp = datetime.fromtimestamp(path.stat().st_mtime).strftime("%d/%m/%Y %H:%M") now = datetime.now().strftime("%d/%m/%Y %H:%M") @@ -50,11 +50,10 @@ def check_file_fresh_baked(test_file): def find_matches(xml_file, desired_matches): - """Read xml file and find desired matches. -Return a list of two lists in the form: -[[matching_line],[matching_line_index]] - -""" + """ + Read xml file and find desired matches. Return a list of two lists in the form: + [[matching_line],[matching_line_index]] + """ with open(xml_file, 'r') as file: xml_reference = file.readlines() lines = [] @@ -91,9 +90,9 @@ def compare_matches(ref_xml_file, test_yml_file, test_xml_file, desired_matches) def test_links(): - """Check the correct parsing of links - -""" + """ + Check the correct parsing of links + """ ref_xml_link_file = 'tests/data/nyaml2nxdl/Ref_NXtest_links.nxdl.xml' test_yml_link_file = 'tests/data/nyaml2nxdl/NXtest_links.yaml' test_xml_link_file = 'tests/data/nyaml2nxdl/NXtest_links.nxdl.xml' @@ -109,8 +108,7 @@ def test_links(): def test_docs(): """In this test an xml file in converted to yml and then back to xml. -The xml trees of the two files are then compared. - + The xml trees of the two files are then compared. """ ref_xml_file = 'tests/data/nyaml2nxdl/Ref_NXellipsometry-docCheck.nxdl.xml' test_yml_file = 'tests/data/nyaml2nxdl/NXellipsometry-docCheck.yaml' @@ -145,8 +143,8 @@ def test_nxdl2yaml_doc_format(): def test_fileline_error(): - """In this test the yaml fileline in the error message is tested. - + """ + In this test the yaml fileline in the error message is tested. """ test_yml_file = 'tests/data/nyaml2nxdl/NXfilelineError1.yaml' result = CliRunner().invoke(nyml2nxdl.launch_tool, ['--input-file', test_yml_file]) @@ -167,11 +165,11 @@ def test_fileline_error(): def test_symbols(): - """Check the correct parsing of symbols - -""" + """ + Check the correct parsing of symbols + """ ref_xml_symbol_file = 'tests/data/nyaml2nxdl/Ref_NXnested_symbols.nxdl.xml' - test_yml_symbol_file = 'tests/data/nyaml2nxdl/NXnested_symbols.yml' + test_yml_symbol_file = 'tests/data/nyaml2nxdl/NXnested_symbols.yaml' test_xml_symbol_file = 'tests/data/nyaml2nxdl/NXnested_symbols.nxdl.xml' desired_matches = ['', '', ' Date: Fri, 21 Apr 2023 00:04:20 +0200 Subject: [PATCH 09/93] Nyaml2nxdl comment preservation (#86) * Yaml comments are perfectly stored in comment_chain of CommentCollecotor class. * readging comment from yaml to xml (nexus field). * Apply comment for 'nexus symbols', 'nexus attributes' and modify code part for root level doc. * Apply comment for 'nexus link', 'nexus choice', 'nexus dimension' and 'nexus group'. * nxdl to yaml comment: root level comment is preserved * Collecting comments in both directions (nyaml to nxdl and nxdl to yaml) and adding an extra option --check-consistency for generating the same file version from a file given as input. * nyaml2nxdl yaml_comment test. * Tests are passed * Fix pylint, pydoc and mypy * All tests and pylint, mypy are passed * Fixing conversation in pull request. * Response to requested changes. --------- Co-authored-by: RubelMozumder --- README.md | 3 ++- test_nyaml2nxdl.py | 63 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 65de76318d..7a71982697 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ This is the place for storing code for tests of the yaml2nxdl and nxdl2yaml NeXus schema translation routines. ## Contact person in FAIRmat for these tests -Andrea Albino \ No newline at end of file +1. Rubel Mozumder +2. Andrea Albino \ No newline at end of file diff --git a/test_nyaml2nxdl.py b/test_nyaml2nxdl.py index 57c3b1f655..5254f54640 100755 --- a/test_nyaml2nxdl.py +++ b/test_nyaml2nxdl.py @@ -287,3 +287,66 @@ def test_yml_parsing(): os.remove('tests/data/nyaml2nxdl/Ref_NXellipsometry_parsed.yaml') os.remove('tests/data/nyaml2nxdl/Ref_NXellipsometry.nxdl.xml') sys.stdout.write('Test on yml -> xml -> yml okay.\n') + + +def test_yml_consistency_comment_parsing(): + """Test comments parsing from yaml. Convert 'yaml' input file to '.nxdl.xml' and + '.nxdl.xml' to '.yaml' + """ + from nexusutils.nyaml2nxdl.comment_collector import CommentCollector + from nexusutils.nyaml2nxdl.nyaml2nxdl_helper import LineLoader + + ref_yml_file = 'tests/data/nyaml2nxdl/Ref_NXcomment.yaml' + test_yml_file = 'tests/data/nyaml2nxdl/Ref_NXcomment_consistency.yaml' + + result = CliRunner().invoke(nyml2nxdl.launch_tool, + ['--input-file', ref_yml_file, + '--check-consistency']) + assert result.exit_code == 0, (f'Exception: {result.exception}, \nExecution Info:' + '{result.exc_info}') + with open(ref_yml_file, 'r', encoding='utf-8') as ref_yml: + loader = LineLoader(ref_yml) + ref_loaded_yaml = loader.get_single_data() + ref_comment_blocks = CommentCollector(ref_yml_file, ref_loaded_yaml) + ref_comment_blocks.extract_all_comment_blocks() + + with open(test_yml_file, 'r', encoding='utf-8') as test_yml: + loader = LineLoader(test_yml) + test_loaded_yaml = loader.get_single_data() + test_comment_blocks = CommentCollector(test_yml_file, test_loaded_yaml) + test_comment_blocks.extract_all_comment_blocks() + + for ref_cmnt, test_cmnt in zip(ref_comment_blocks, test_comment_blocks): + assert ref_cmnt == test_cmnt, 'Comment is not consistent.' + + os.remove(test_yml_file) + + +def test_yml2xml_comment_parsing(): + """To test comment that written in xml for element attributes, e.g. + attribute 'rank' for 'dimension' element and attribute 'exists' for + 'NXentry' group element. + """ + input_yml = 'tests/data/nyaml2nxdl/NXcomment_yaml2nxdl.yaml' + ref_xml = 'tests/data/nyaml2nxdl/Ref_NXcomment_yaml2nxdl.nxdl.xml' + test_xml = 'tests/data/nyaml2nxdl/NXcomment_yaml2nxdl.nxdl.xml' + + result = CliRunner().invoke(nyml2nxdl.launch_tool, + ['--input-file', input_yml]) + assert result.exit_code == 0 + + ref_root = ET.parse(ref_xml).getroot() + test_root = ET.parse(test_xml).getroot() + + def recursive_compare(ref_root, test_root): + assert ref_root.attrib.items() == test_root.attrib.items(), ("Got different xml element" + "Atribute.") + if ref_root.text and test_root.text: + assert ref_root.text.strip() == test_root.text.strip(), ("Got differen element text.") + if len(ref_root) > 0 and len(test_root) > 0: + for x, y in zip(ref_root, test_root): + recursive_compare(x, y) + + recursive_compare(ref_root, test_root) + + os.remove(test_xml) From b57695dfb0836a8d70fb44e173d62de14b0abbbe Mon Sep 17 00:00:00 2001 From: Florian Dobener Date: Fri, 21 Apr 2023 17:10:43 +0200 Subject: [PATCH 10/93] Renames to pynxtools (#93) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Renames to pynxtools geändert: .github/workflows/pylint.yml geändert: .gitmodules modified: .github/workflows/pylint.yml * Properly adds definitions * Rename left over nexusutils reference from rebase * Merges xps reader.py --- README.md | 69 +++ __init__.py | 22 + comment_collector.py | 487 +++++++++++++++ nyaml2nxdl.py | 191 ++++++ nyaml2nxdl_backward_tools.py | 946 ++++++++++++++++++++++++++++++ nyaml2nxdl_forward_tools.py | 1072 ++++++++++++++++++++++++++++++++++ nyaml2nxdl_helper.py | 145 +++++ 7 files changed, 2932 insertions(+) create mode 100644 README.md create mode 100644 __init__.py create mode 100644 comment_collector.py create mode 100755 nyaml2nxdl.py create mode 100755 nyaml2nxdl_backward_tools.py create mode 100644 nyaml2nxdl_forward_tools.py create mode 100644 nyaml2nxdl_helper.py diff --git a/README.md b/README.md new file mode 100644 index 0000000000..6a0bbf86a7 --- /dev/null +++ b/README.md @@ -0,0 +1,69 @@ +# YAML to NXDL converter and NXDL to YAML converter + +**NOTE: Please use python3.8 or above to run this converter** + +**Tools purpose**: Offer a simple YAML-based schema and a XML-based schema to describe NeXus instances. These can be NeXus application definitions or classes +such as base or contributed classes. Users either create NeXus instances by writing a YAML file or a XML file which details a hierarchy of data/metadata elements. +The forward (YAML -> NXDL.XML) and backward (NXDL.XML -> YAML) conversions are implemented. + +**How the tool works**: +- yaml2nxdl.py +1. Reads the user-specified NeXus instance, either in YML or XML format. +2. If input is in YAML, creates an instantiated NXDL schema XML tree by walking the dictionary nest. + If input is in XML, creates a YML file walking the dictionary nest. +3. Write the tree into a YAML file or a properly formatted NXDL XML schema file to disk. +4. Optionally, if --append argument is given, + the XML or YAML input file is interpreted as an extension of a base class and the entries contained in it + are appended below a standard NeXus base class. + You need to specify both your input file (with YAML or XML extension) and NeXus class (with no extension). + Both .yml and .nxdl.xml file of the extended class are printed. + +```console +user@box:~$ python yaml2nxdl.py + +Usage: python yaml2nxdl.py [OPTIONS] + +Options: + --input-file TEXT The path to the input data file to read. + --append TEXT Parse xml NeXus file and append to specified base class, + write the base class name with no extension. + --verbose Addictional std output info is printed to help debugging. + --help Show this message and exit. + +``` + +## Documentation + +**Rule set**: From transcoding YAML files we need to follow several rules. +* Named NeXus groups, which are instances of NeXus classes especially base or contributed classes. Creating (NXbeam) is a simple example of a request to define a group named according to NeXus default rules. mybeam1(NXbeam) or mybeam2(NXbeam) are examples how to create multiple named instances at the same hierarchy level. +* Members of groups so-called fields or attributes. A simple example of a member is voltage. Here the datatype is implied automatically as the default NeXus NX_CHAR type. By contrast, voltage(NX_FLOAT) can be used to instantiate a member of class which should be of NeXus type NX_FLOAT. +* And attributes of either groups or fields. Names of attributes have to be preceeded by \@ to mark them as attributes. +* Optionality: For all fields, groups and attributes in `application definitions` are `required` by default, except anything (`recommended` or `optional`) mentioned. + +**Special keywords**: Several keywords can be used as childs of groups, fields, and attributes to specify the members of these. Groups, fields and attributes are nodes of the XML tree. +* **doc**: A human-readable description/docstring +* **exists** Options are recommended, required, [min, 1, max, infty] numbers like here 1 can be replaced by any uint, or infty to indicate no restriction on how frequently the entry can occur inside the NXDL schema at the same hierarchy level. +* **link** Define links between nodes. +* **units** A statement introducing NeXus-compliant NXDL units arguments, like NX_VOLTAGE +* **dimensions** Details which dimensional arrays to expect +* **enumeration** Python list of strings which are considered as recommended entries to choose from. +* **dim_parameters** `dim` which is a child of `dimension` and the `dim` might have several attributes `ref`, +`incr` including `index` and `value`. So while writting `yaml` file schema definition please following structure: +``` +dimensions: + rank: integer value + dim: [[ind_1, val_1], [ind_2, val_2], ...] + dim_parameters: + ref: [ref_value_1, ref_value_2, ...] + incr: [incr_value_1, incr_value_2, ...] +``` +Keep in mind that length of all the lists must be same. + +## Next steps + +The NOMAD team is currently working on the establishing of a one-to-one mapping between +NeXus definitions and the NOMAD MetaInfo. As soon as this is in place the YAML files will +be annotated with further metadata so that they can serve two purposes. +On the one hand they can serve as an instance for a schema to create a GUI representation +of a NOMAD Oasis ELN schema. On the other hand the YAML to NXDL converter will skip all +those pieces of information which are irrelevant from a NeXus perspective. diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000000..22eb35f68d --- /dev/null +++ b/__init__.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +""" +# Load paths +""" +# -*- coding: utf-8 -*- +# +# Copyright The NOMAD Authors. +# +# This file is part of NOMAD. See https://nomad-lab.eu for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/comment_collector.py b/comment_collector.py new file mode 100644 index 0000000000..a2aa54b782 --- /dev/null +++ b/comment_collector.py @@ -0,0 +1,487 @@ +#!usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright The NOMAD Authors. +# +# This file is part of NOMAD. See https://nomad-lab.eu for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +Collect comments in a list by CommentCollector class. Comment is a instance of Comment, +where each comment includes comment text and line info or neighbour info where the +comment must be assinged. + +The class Comment is an abstract class for general functions or method to be implemented +XMLComment and YAMLComment class. + +NOTE: Here comment block mainly stands for (comment text + line or element for what comment is +intended.) +""" + + +from typing import List, Type, Any, Tuple, Union, Dict +from pynxtools.nyaml2nxdl.nyaml2nxdl_helper import LineLoader + +__all__ = ['Comment', 'CommentCollector', 'XMLComment', 'YAMLComment'] + + +# pylint: disable=inconsistent-return-statements +class CommentCollector: + """CommentCollector will store a full comment ('Comment') object in + _comment_chain. + """ + + def __init__(self, input_file: str = None, + loaded_obj: Union[object, Dict] = None): + """ + Initialise CommentCollector + parameters: + input_file: raw input file (xml, yml) + loaded_obj: file loaded by third party library + """ + self._comment_chain: List = [] + self.file = input_file + self._comment_tracker = 0 + self._comment_hash: Dict[Tuple, Type[Comment]] = {} + self.comment: Type[Comment] + if self.file and not loaded_obj: + if self.file.split('.')[-1] == 'xml': + self.comment = XMLComment + if self.file.split('.')[-1] == 'yaml': + self.comment = YAMLComment + with open(self.file, "r", encoding="utf-8") as plain_text_yaml: + loader = LineLoader(plain_text_yaml) + self.comment.__yaml_dict__ = loader.get_single_data() + elif self.file and loaded_obj: + if self.file.split('.')[-1] == 'yaml' and isinstance(loaded_obj, dict): + self.comment = YAMLComment + self.comment.__yaml_dict__ = loaded_obj + else: + raise ValueError("Incorrect inputs for CommentCollector e.g. Wrong file extension.") + + else: + raise ValueError("Incorrect inputs for CommentCollector") + + def extract_all_comment_blocks(self): + """ + Collect all comments. Note that here comment means (comment text + element or line info + intended for comment. + """ + id_ = 0 + single_comment = self.comment(comment_id=id_) + with open(self.file, mode='r', encoding='UTF-8') as enc_f: + lines = enc_f.readlines() + # Make an empty line for last comment if no empty lines in original file + if lines[-1] != '': + lines.append('') + for line_num, line in enumerate(lines): + if single_comment.is_storing_single_comment(): + # If the last comment comes without post nxdl fields, groups and attributes + if line_num < (len(lines) - 1): + # Processing file from Line number 1 + single_comment.process_each_line(line, (line_num + 1)) + else: + # For processing post comment + single_comment.process_each_line(line + 'post_comment', (line_num + 1)) + self._comment_chain.append(single_comment) + else: + self._comment_chain.append(single_comment) + single_comment = self.comment(last_comment=single_comment) + single_comment.process_each_line(line, (line_num + 1)) + + def get_comment(self): + """ + Return comment from comment_chain that must come earlier in order. + """ + return self._comment_chain[self._comment_tracker] + + def get_coment_by_line_info(self, comment_locs: Tuple[str, Union[int, str]]): + """ + Get comment using line information. + """ + if comment_locs in self._comment_hash: + return self._comment_hash[comment_locs] + + line_annot, line_loc = comment_locs + for cmnt in self._comment_chain: + if line_annot in cmnt: + line_loc_ = cmnt.get_line_number(line_annot) + if line_loc == line_loc_: + self._comment_hash[comment_locs] = cmnt + return cmnt + + def reload_comment(self): + """ + Update self._comment_tracker after done with last comment. + """ + self._comment_tracker += 1 + + def __contains__(self, comment_locs: tuple): + """ + Confirm wether the comment corresponds to key_line and line_loc + is exist or not. + comment_locs is equvalant to (line_annotation, line_loc) e.g. + (__line__doc and 35) + """ + if not isinstance(comment_locs, tuple): + raise TypeError("Comment_locs should be 'tuple' containing line annotation " + "(e.g.__line__doc) and line_loc (e.g. 35).") + line_annot, line_loc = comment_locs + for cmnt in self._comment_chain: + if line_annot in cmnt: + line_loc_ = cmnt.get_line_number(line_annot) + if line_loc == line_loc_: + self._comment_hash[comment_locs] = cmnt + return True + return False + + def __getitem__(self, ind): + """Get comment from self.obj._comment_chain by index. + """ + if ind >= len(self._comment_chain): + raise IndexError(f'Oops! Coment index {ind} in {__class__} is out of range!') + return self._comment_chain[ind] + + def __iter__(self): + """get comment ieratively + """ + return iter(self._comment_chain) + + +# pylint: disable=too-many-instance-attributes +class Comment: + """ + This class is building yaml comment and the intended line for what comment is written. + """ + + def __init__(self, + comment_id: int = -1, + last_comment: 'Comment' = None) -> None: + """Comment object can be considered as a block element that includes + document element (an entity for what the comment is written). + """ + self._elemt: Any = None + self._elemt_text: str = None + self._is_elemt_found: bool = None + self._is_elemt_stored: bool = None + + self._comnt: str = '' + # If Multiple comments for one element or entity + self._comnt_list: List[str] = [] + self.last_comment: 'Comment' = last_comment if last_comment else None + if comment_id >= 0 and last_comment: + self.cid = comment_id + self.last_comment = last_comment + elif comment_id == 0 and not last_comment: + self.cid = comment_id + self.last_comment = None + elif last_comment: + self.cid = self.last_comment.cid + 1 + self.last_comment = last_comment + else: + raise ValueError("Neither last comment nor comment id dound") + self._comnt_start_found: bool = False + self._comnt_end_found: bool = False + self.is_storing_single_comment = lambda: not (self._comnt_end_found + and self._is_elemt_stored) + + def get_comment_text(self) -> Union[List, str]: + """ + Extract comment text from entrire comment (comment text + elment or + line for what comment is intended) + """ + + def append_comment(self, text: str) -> None: + """ + Append lines of the same comment. + """ + + def store_element(self, args) -> None: + """ + Strore comment text and line or element that is intended for comment. + """ + + +class XMLComment(Comment): + """ + XMLComment to store xml comment element. + """ + + def __init__(self, comment_id: int = -1, last_comment: 'Comment' = None) -> None: + super().__init__(comment_id, last_comment) + + def process_each_line(self, text, line_num): + """Take care of each line of text. Through which function the text + must be passed should be decide here. + """ + text = text.strip() + if text and line_num: + self.append_comment(text) + if self._comnt_end_found and not self._is_elemt_found: + if self._comnt: + self._comnt_list.append(self._comnt) + self._comnt = '' + + if self._comnt_end_found: + self.store_element(text) + + def append_comment(self, text: str) -> None: + # Comment in single line + if '' == text[-4:]: + self._comnt_end_found = True + self._comnt_start_found = False + self._comnt = self._comnt.replace('-->', '') + + elif '-->' == text[0:4] and self._comnt_start_found: + self._comnt_end_found = True + self._comnt_start_found = False + self._comnt = self._comnt + '\n' + text.replace('-->', '') + elif self._comnt_start_found: + self._comnt = self._comnt + '\n' + text + + # pylint: disable=arguments-differ, arguments-renamed + def store_element(self, text) -> None: + def collect_xml_attributes(text_part): + for part in text_part: + part = part.strip() + if part and '">' == ''.join(part[-2:]): + self._is_elemt_stored = True + self._is_elemt_found = False + part = ''.join(part[0:-2]) + elif part and '"/>' == ''.join(part[-3:]): + self._is_elemt_stored = True + self._is_elemt_found = False + part = ''.join(part[0:-3]) + elif part and '/>' == ''.join(part[-2:]): + self._is_elemt_stored = True + self._is_elemt_found = False + part = ''.join(part[0:-2]) + elif part and '>' == part[-1]: + self._is_elemt_stored = True + self._is_elemt_found = False + part = ''.join(part[0:-1]) + elif part and '"' == part[-1]: + part = ''.join(part[0:-1]) + + if '="' in part: + lf_prt, rt_prt = part.split('="') + else: + continue + if ':' in lf_prt: + continue + self._elemt[lf_prt] = str(rt_prt) + if not self._elemt: + self._elemt = {} + # First check for comment part has been collected prefectly + if ' Union[List, str]: + """ + This method returns list of commnent text. As some xml element might have + multiple separated comment intended for a single element. + """ + return self._comnt_list + + +class YAMLComment(Comment): + """ + This class for stroing comment text as well as location of the comment e.g. line + number of other in the file. + NOTE: + 1. Do not delete any element form yaml dictionary (for loaded_obj. check: Comment_collector + class. because this loaded file has been exploited in nyaml2nxdl forward tools.) + """ + # Class level variable. The main reason behind that to follow structure of + # abstract class 'Comment' + __yaml_dict__: dict = {} + __yaml_line_info: dict = {} + __comment_escape_char = {'--': '-\\-'} + + def __init__(self, comment_id: int = -1, last_comment: 'Comment' = None) -> None: + """Initialization of YAMLComment follow Comment class. + """ + super().__init__(comment_id, last_comment) + self.collect_yaml_line_info(YAMLComment.__yaml_dict__, YAMLComment.__yaml_line_info) + + def process_each_line(self, text, line_num): + """Take care of each line of text. Through which function the text + must be passed should be decide here. + """ + text = text.strip() + self.append_comment(text) + if self._comnt_end_found and not self._is_elemt_found: + if self._comnt: + self._comnt_list.append(self._comnt) + self._comnt = '' + + if self._comnt_end_found: + line_key = '' + if ':' in text: + ind = text.index(':') + line_key = '__line__' + ''.join(text[0:ind]) + + for l_num, l_key in self.__yaml_line_info.items(): + if line_num == int(l_num) and line_key == l_key: + self.store_element(line_key, line_num) + break + # Comment comes very end of the file + if text == 'post_comment' and line_key == '': + line_key = '__line__post_comment' + self.store_element(line_key, line_num) + + def has_post_comment(self): + """ + Ensure is this a post coment or not. + Post comment means the comment that come at the very end without having any + nxdl element(class, group, filed and attribute.) + """ + for key, _ in self._elemt.items(): + if '__line__post_comment' == key: + return True + return False + + def append_comment(self, text: str) -> None: + """ + Collects all the line of the same comment and + append them with that single comment. + """ + # check for escape char + text = self.replace_scape_char(text) + # Empty line after last line of comment + if not text and self._comnt_start_found: + self._comnt_end_found = True + self._comnt_start_found = False + # For empty line inside doc or yaml file. + elif not text: + return + elif '# ' == ''.join(text[0:2]): + self._comnt_start_found = True + self._comnt_end_found = False + self._comnt = '' if not self._comnt else self._comnt + '\n' + self._comnt = self._comnt + ''.join(text[2:]) + elif '#' == text[0]: + self._comnt_start_found = True + self._comnt_end_found = False + self._comnt = '' if not self._comnt else self._comnt + '\n' + self._comnt = self._comnt + ''.join(text[1:]) + elif 'post_comment' == text: + self._comnt_end_found = True + self._comnt_start_found = False + # for any line after 'comment block' found + elif self._comnt_start_found: + self._comnt_start_found = False + self._comnt_end_found = True + + # pylint: disable=arguments-differ + def store_element(self, line_key, line_number): + """ + Store comment content and information of commen location (for what comment is + created.). + """ + self._elemt = {} + self._elemt[line_key] = int(line_number) + self._is_elemt_found = False + self._is_elemt_stored = True + + def get_comment_text(self): + """ + Return list of comments if there are multiple comment for same yaml line. + """ + return self._comnt_list + + def get_line_number(self, line_key): + """ + Retrun line number for what line the comment is created + """ + return self._elemt[line_key] + + def get_line_info(self): + """ + Return line annotation and line number from a comment. + """ + for line_anno, line_loc in self._elemt.items(): + return line_anno, line_loc + + def replace_scape_char(self, text): + """Replace escape char according to __comment_escape_char dict + """ + for ecp_char, ecp_alt in YAMLComment.__comment_escape_char.items(): + if ecp_char in text: + text = text.replace(ecp_char, ecp_alt) + return text + + def get_element_location(self): + """ + Retrun yaml line '__line__KEY' info and and line numner + """ + if len(self._elemt) > 1: + raise ValueError(f"Comment element should be one but got " + f"{self._elemt}") + + for key, val in self._elemt.items(): + return key, val + + def collect_yaml_line_info(self, yaml_dict, line_info_dict): + """Collect __line__key and corresponding value from + a yaml file dictonary in another dictionary. + """ + for line_key, line_n in yaml_dict.items(): + if '__line__' in line_key: + line_info_dict[line_n] = line_key + + for _, val in yaml_dict.items(): + if isinstance(val, dict): + self.collect_yaml_line_info(val, line_info_dict) + + def __contains__(self, line_key): + """For Checking whether __line__NAME is in _elemt dict or not.""" + return line_key in self._elemt + + def __eq__(self, comment_obj): + """Check the self has same value as right comment. + """ + if len(self._comnt_list) != len(comment_obj._comnt_list): + return False + for left_cmnt, right_cmnt in zip(self._comnt_list, comment_obj._comnt_list): + left_cmnt = left_cmnt.split('\n') + right_cmnt = right_cmnt.split('\n') + for left_line, right_line in zip(left_cmnt, right_cmnt): + if left_line.strip() != right_line.strip(): + return False + return True diff --git a/nyaml2nxdl.py b/nyaml2nxdl.py new file mode 100755 index 0000000000..3088a25e52 --- /dev/null +++ b/nyaml2nxdl.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 +"""Main file of yaml2nxdl tool. +Users create NeXus instances by writing a YAML file +which details a hierarchy of data/metadata elements + +""" +# -*- coding: utf-8 -*- +# +# Copyright The NOMAD Authors. +# +# This file is part of NOMAD. See https://nomad-lab.eu for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import os +import xml.etree.ElementTree as ET + +import click + +from pynxtools.nyaml2nxdl.nyaml2nxdl_forward_tools import nyaml2nxdl, pretty_print_xml +from pynxtools.nyaml2nxdl.nyaml2nxdl_backward_tools import (Nxdl2yaml, + compare_niac_and_my) + + +DEPTH_SIZE = " " + + +# pylint: disable=too-many-locals +def append_yml(input_file, append, verbose): + """Append to an existing NeXus base class new elements provided in YML input file \ +and print both an XML and YML file of the extended base class. + +""" + nexus_def_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), '../../definitions') + assert [s for s in os.listdir(os.path.join(nexus_def_path, 'base_classes') + ) if append.strip() == s.replace('.nxdl.xml', '')], \ + 'Your base class extension does not match any existing NeXus base classes' + tree = ET.parse(os.path.join(nexus_def_path + '/base_classes', append + '.nxdl.xml')) + root = tree.getroot() + # warning: tmp files are printed on disk and removed at the ends!! + pretty_print_xml(root, 'tmp.nxdl.xml') + input_tmp_xml = 'tmp.nxdl.xml' + out_tmp_yml = 'tmp_parsed.yaml' + converter = Nxdl2yaml([], []) + converter.print_yml(input_tmp_xml, out_tmp_yml, verbose) + nyaml2nxdl(input_file=out_tmp_yml, + out_file='tmp_parsed.nxdl.xml', + verbose=verbose) + tree = ET.parse('tmp_parsed.nxdl.xml') + tree2 = ET.parse(input_file) + root_no_duplicates = ET.Element( + 'definition', {'xmlns': 'http://definition.nexusformat.org/nxdl/3.1', + 'xmlns:xsi': 'http://www.w3.org/2001/XMLSchema-instance', + 'xsi:schemaLocation': 'http://www.w3.org/2001/XMLSchema-instance' + } + ) + for attribute_keys in root.attrib.keys(): + if attribute_keys != '{http://www.w3.org/2001/XMLSchema-instance}schemaLocation': + attribute_value = root.attrib[attribute_keys] + root_no_duplicates.set(attribute_keys, attribute_value) + for elems in root.iter(): + if 'doc' in elems.tag: + root_doc = ET.SubElement(root_no_duplicates, 'doc') + root_doc.text = elems.text + break + group = '{http://definition.nexusformat.org/nxdl/3.1}group' + root_no_duplicates = compare_niac_and_my(tree, tree2, verbose, + group, + root_no_duplicates) + field = '{http://definition.nexusformat.org/nxdl/3.1}field' + root_no_duplicates = compare_niac_and_my(tree, tree2, verbose, + field, + root_no_duplicates) + attribute = '{http://definition.nexusformat.org/nxdl/3.1}attribute' + root_no_duplicates = compare_niac_and_my(tree, tree2, verbose, + attribute, + root_no_duplicates) + pretty_print_xml(root_no_duplicates, f"{input_file.replace('.nxdl.xml', '')}" + f"_appended.nxdl.xml") + + input_file_xml = input_file.replace('.nxdl.xml', "_appended.nxdl.xml") + out_file_yml = input_file.replace('.nxdl.xml', "_appended_parsed.yaml") + converter = Nxdl2yaml([], []) + converter.print_yml(input_file_xml, out_file_yml, verbose) + nyaml2nxdl(input_file=out_file_yml, + out_file=out_file_yml.replace('.yaml', '.nxdl.xml'), + verbose=verbose) + os.rename(f"{input_file.replace('.nxdl.xml', '_appended_parsed.yaml')}", + f"{input_file.replace('.nxdl.xml', '_appended.yaml')}") + os.rename(f"{input_file.replace('.nxdl.xml', '_appended_parsed.nxdl.xml')}", + f"{input_file.replace('.nxdl.xml', '_appended.nxdl.xml')}") + os.remove('tmp.nxdl.xml') + os.remove('tmp_parsed.yaml') + os.remove('tmp_parsed.nxdl.xml') + + +def split_name_and_extension(file_name): + """ + Split file name into extension and rest of the file name. + return file raw nam and extension + """ + parts = file_name.rsplit('.', 2) + if len(parts) == 2: + raw = parts[0] + ext = parts[1] + if len(parts) == 3: + raw = parts[0] + ext = '.'.join(parts[1:]) + + return raw, ext + + +@click.command() +@click.option( + '--input-file', + required=True, + prompt=True, + help='The path to the XML or YAML input data file to read and create \ +a YAML or XML file from, respectively.' +) +@click.option( + '--append', + help='Parse xml file and append to base class, given that the xml file has same name \ +of an existing base class' +) +@click.option( + '--check-consistency', + is_flag=True, + default=False, + help=('Check wether yaml or nxdl has followed general rules of scema or not' + 'check whether your comment in the right place or not. The option render an ' + 'output file of the same extension(*_consistency.yaml or *_consistency.nxdl.xml)') +) +@click.option( + '--verbose', + is_flag=True, + default=False, + help='Print in standard output keywords and value types to help \ +possible issues in yaml files' +) +def launch_tool(input_file, verbose, append, check_consistency): + """ + Main function that distiguishes the input file format and launches the tools. + """ + if os.path.isfile(input_file): + raw_name, ext = split_name_and_extension(input_file) + else: + raise ValueError("Need a valid input file.") + + if ext == 'yaml': + xml_out_file = raw_name + '.nxdl.xml' + nyaml2nxdl(input_file, xml_out_file, verbose) + if append: + append_yml(raw_name + '.nxdl.xml', + append, + verbose + ) + # For consistency running + if check_consistency: + yaml_out_file = raw_name + '_consistency.' + ext + converter = Nxdl2yaml([], []) + converter.print_yml(xml_out_file, yaml_out_file, verbose) + os.remove(xml_out_file) + elif ext == 'nxdl.xml': + if not append: + yaml_out_file = raw_name + '_parsed' + '.yaml' + converter = Nxdl2yaml([], []) + converter.print_yml(input_file, yaml_out_file, verbose) + else: + append_yml(input_file, append, verbose) + # Taking care of consistency running + if check_consistency: + xml_out_file = raw_name + '_consistency.' + ext + nyaml2nxdl(yaml_out_file, xml_out_file, verbose) + os.remove(yaml_out_file) + else: + raise ValueError("Provide correct file with extension '.yaml or '.nxdl.xml") + + +if __name__ == '__main__': + launch_tool().parse() # pylint: disable=no-value-for-parameter diff --git a/nyaml2nxdl_backward_tools.py b/nyaml2nxdl_backward_tools.py new file mode 100755 index 0000000000..0ebc0d5cc1 --- /dev/null +++ b/nyaml2nxdl_backward_tools.py @@ -0,0 +1,946 @@ +#!/usr/bin/env python3 +"""This file collects the function used in the reverse tool nxdl2yaml. + +""" +# -*- coding: utf-8 -*- +# +# Copyright The NOMAD Authors. +# +# This file is part of NOMAD. See https://nomad-lab.eu for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import sys +from typing import List, Dict +import xml.etree.ElementTree as ET +import os + +from pynxtools.nyaml2nxdl.nyaml2nxdl_helper import (get_node_parent_info, + get_yaml_escape_char_dict, + cleaning_empty_lines) +from pynxtools.dataconverter.helpers import remove_namespace_from_tag + + +DEPTH_SIZE = " " +CMNT_TAG = '!--' + + +def separate_pi_comments(input_file): + """ + Separate PI comments from ProcessesInstruction (pi) + """ + comments_list = [] + comment = [] + xml_lines = [] + + with open(input_file, "r", encoding='utf-8') as file: + lines = file.readlines() + has_pi = True + for line in lines: + c_start = '' + def_tag = ' 0 and has_pi: + comment.append(line.replace(cmnt_end, '')) + comments_list.append(''.join(comment)) + comment = [] + elif def_tag in line or not has_pi: + has_pi = False + xml_lines.append(line) + elif len(comment) > 0 and has_pi: + comment.append(line) + else: + xml_lines.append(line) + return comments_list, ''.join(xml_lines) + + +# Collected: https://dustinoprea.com/2019/01/22/python-parsing-xml-and-retaining-the-comments/ +class _CommentedTreeBuilder(ET.TreeBuilder): + + def comment(self, text): + """ + defining comment builder in TreeBuilder + """ + self.start('!--', {}) + self.data(text) + self.end('--') + + +def parse(filepath): + """ + Construct parse function for modified tree builder for including modified TreeBuilder + and rebuilding XMLParser. + """ + comments, xml_str = separate_pi_comments(filepath) + ctb = _CommentedTreeBuilder() + xp_parser = ET.XMLParser(target=ctb) + root = ET.fromstring(xml_str, parser=xp_parser) + return comments, root + + +def handle_mapping_char(text, depth=-1, skip_n_line_on_top=False): + """Check for ":" char and replace it by "':'". """ + + # This escape chars and sepeerator ':' is not allowed in yaml library + escape_char = get_yaml_escape_char_dict() + for esc_key, val in escape_char.items(): + if esc_key in text: + text = text.replace(esc_key, val) + if not skip_n_line_on_top: + if depth > 0: + text = add_new_line_on_top(text, depth) + else: + raise ValueError("Need depth size to co-ordinate text line in yaml file.") + return text + + +def add_new_line_on_top(text, depth): + """ + Return char list for what we get error in converter. After adding a + new line at the start of text the eroor is solved. + """ + char_list_to_add_new_line_on_top_of_text = [":"] + for char in char_list_to_add_new_line_on_top_of_text: + if char in text: + return '\n' + depth * DEPTH_SIZE + text + return text + + +# pylint: disable=too-many-instance-attributes +class Nxdl2yaml(): + """ + Parse XML file and print a YML file + """ + + def __init__( + self, + symbol_list: List[str], + root_level_definition: List[str], + root_level_doc='', + root_level_symbols=''): + + # updated part of yaml_dict + self.found_definition = False + self.root_level_doc = root_level_doc + self.root_level_symbols = root_level_symbols + self.root_level_definition = root_level_definition + self.symbol_list = symbol_list + self.is_last_element_comment = False + self.include_comment = True + self.pi_comments = None + # NOTE: Here is how root_level_comments organised for storing comments + # root_level_comment= {'root_doc': comment, + # 'symbols': comment, + # The 'symbol_doc_comments' list is for comments from all 'symbol doc' + # 'symbol_doc_comments' : [comments] + # 'symbol_list': [symbols], + # The 'symbol_comments' contains comments for 'symbols doc' and all 'symbol' + # 'symbol_comments': [comments]} + self.root_level_comment: Dict[str, str] = {} + + def print_yml(self, input_file, output_yml, verbose): + """ + Parse an XML file provided as input and print a YML file + """ + if os.path.isfile(output_yml): + os.remove(output_yml) + + depth = 0 + + self.pi_comments, root = parse(input_file) + xml_tree = {'tree': root, 'node': root} + self.xmlparse(output_yml, xml_tree, depth, verbose) + + def handle_symbols(self, depth, node): + """Handle symbols field and its childs symbol""" + + # pylint: disable=consider-using-f-string + self.root_level_symbols = ( + f"{remove_namespace_from_tag(node.tag)}: " + f"{node.text.strip() if node.text else ''}" + ) + depth += 1 + last_comment = '' + sbl_doc_cmnt_list = [] + # Comments that come above symbol tag + symbol_cmnt_list = [] + for child in list(node): + tag = remove_namespace_from_tag(child.tag) + if tag == CMNT_TAG and self.include_comment: + last_comment = self.comvert_to_ymal_comment(depth * DEPTH_SIZE, child.text) + if tag == 'doc': + symbol_cmnt_list.append(last_comment) + # The bellow line is for handling lenth of 'symbol_comments' and + # 'symbol_doc_comments'. Otherwise print_root_level_info() gets inconsistency + # over for the loop while writting comment on file + sbl_doc_cmnt_list.append('') + last_comment = '' + self.symbol_list.append(self.handle_not_root_level_doc(depth, + text=child.text)) + elif tag == 'symbol': + # place holder is symbol name + symbol_cmnt_list.append(last_comment) + last_comment = '' + if 'doc' in child.attrib: + self.symbol_list.append( + self.handle_not_root_level_doc(depth, + tag=child.attrib['name'], + text=child.attrib['doc'])) + else: + for symbol_doc in list(child): + tag = remove_namespace_from_tag(symbol_doc.tag) + if tag == CMNT_TAG and self.include_comment: + last_comment = self.comvert_to_ymal_comment(depth * DEPTH_SIZE, + symbol_doc.text) + if tag == 'doc': + sbl_doc_cmnt_list.append(last_comment) + last_comment = '' + self.symbol_list.append( + self.handle_not_root_level_doc(depth, + tag=child.attrib['name'], + text=symbol_doc.text)) + self.stroe_root_level_comments('symbol_doc_comments', sbl_doc_cmnt_list) + self.stroe_root_level_comments('symbol_comments', symbol_cmnt_list) + + def stroe_root_level_comments(self, holder, comment): + """Store yaml text or section line and the comments inteded for that lines or section""" + + self.root_level_comment[holder] = comment + + def handle_definition(self, node): + """ + Handle definition group and its attributes + NOTE: Here we tried to store the order of the xml element attributes. So that we get + exactly the same file in nxdl from yaml. + """ + # pylint: disable=consider-using-f-string + # self.root_level_definition[0] = '' + keyword = '' + # tmp_word for reseving the location + tmp_word = "#xx#" + attribs = node.attrib + # for tracking the order of name and type + keyword_order = -1 + for item in attribs: + if "name" in item: + keyword = keyword + attribs[item] + if keyword_order == -1: + self.root_level_definition.append(tmp_word) + keyword_order = self.root_level_definition.index(tmp_word) + elif "extends" in item: + keyword = f"{keyword}({attribs[item]})" + if keyword_order == -1: + self.root_level_definition.append(tmp_word) + keyword_order = self.root_level_definition.index(tmp_word) + elif 'schemaLocation' not in item \ + and 'extends' != item: + text = f"{item}: {attribs[item]}" + self.root_level_definition.append(text) + self.root_level_definition[keyword_order] = f"{keyword}:" + + def handle_root_level_doc(self, node): + """ + Handle the documentation field found at root level. + """ + # tag = remove_namespace_from_tag(node.tag) + text = node.text + text = self.handle_not_root_level_doc(depth=0, text=text) + self.root_level_doc = text + + # pylint: disable=too-many-branches + def handle_not_root_level_doc(self, depth, text, tag='doc', file_out=None): + """ + Handle docs field along the yaml file. In this function we also tried to keep + the track of intended indentation. E.g. the bollow doc block. + * Topic name + DEscription of topic + """ + + # Handling empty doc + if not text: + text = "" + else: + text = handle_mapping_char(text, -1, True) + if "\n" in text: + # To remove '\n' character as it will be added before text. + text = text.split('\n') + text = cleaning_empty_lines(text) + text_tmp = [] + yaml_indent_n = len((depth + 1) * DEPTH_SIZE) + # Find indentaion in the first valid line with alphabet + tmp_i = 0 + while tmp_i != -1: + first_line_indent_n = 0 + for ch_ in text[tmp_i]: + if ch_ == ' ': + first_line_indent_n = first_line_indent_n + 1 + elif ch_ != '': + tmp_i = -2 + break + tmp_i = tmp_i + 1 + # Taking care of doc like bellow: + # Text liness + # text continues + # So no indentaion at the staring or doc. So doc group will come along general + # alignment + if first_line_indent_n == 0: + first_line_indent_n = yaml_indent_n + + # for indent_diff -ve all lines will move left by the same ammout + # for indect_diff +ve all lines will move right the same amount + indent_diff = yaml_indent_n - first_line_indent_n + # CHeck for first line empty if not keep first line empty + + for _, line in enumerate(text): + line_indent_n = 0 + # Collect first empty space without alphabate + for ch_ in line: + if ch_ == ' ': + line_indent_n = line_indent_n + 1 + else: + break + line_indent_n = line_indent_n + indent_diff + if line_indent_n < yaml_indent_n: + # if line still under yaml identation + text_tmp.append(yaml_indent_n * ' ' + line.strip()) + else: + text_tmp.append(line_indent_n * ' ' + line.strip()) + + text = '\n' + '\n'.join(text_tmp) + if "}" in tag: + tag = remove_namespace_from_tag(tag) + indent = depth * DEPTH_SIZE + elif text: + text = '\n' + (depth + 1) * DEPTH_SIZE + text.strip() + if "}" in tag: + tag = remove_namespace_from_tag(tag) + indent = depth * DEPTH_SIZE + else: + text = "" + if "}" in tag: + tag = remove_namespace_from_tag(tag) + indent = depth * DEPTH_SIZE + + doc_str = f"{indent}{tag}: |{text}\n" + if file_out: + file_out.write(doc_str) + return None + return doc_str + + def write_out(self, indent, text, file_out): + """ + Write text line in output file. + """ + line_string = f"{indent}{text.rstrip()}\n" + file_out.write(line_string) + + def print_root_level_doc(self, file_out): + """ + Print at the root level of YML file \ + the general documentation field found in XML file + """ + indent = 0 * DEPTH_SIZE + + if ('root_doc' in self.root_level_comment + and self.root_level_comment['root_doc'] != ''): + text = self.root_level_comment['root_doc'] + self.write_out(indent, text, file_out) + + text = self.root_level_doc + self.write_out(indent, text, file_out) + self.root_level_doc = '' + + def comvert_to_ymal_comment(self, indent, text): + """ + Convert into yaml comment by adding exta '#' char in front of comment lines + """ + lines = text.split('\n') + mod_lines = [] + for line in lines: + line = line.strip() + if line and line[0] != '#': + line = indent + '# ' + line + mod_lines.append(line) + elif line: + line = indent + line + mod_lines.append(line) + # The starting '\n' to keep multiple comments separate + return '\n' + '\n'.join(mod_lines) + + def print_root_level_info(self, depth, file_out): + """ + Print at the root level of YML file \ + the information stored as definition attributes in the XML file + """ + # pylint: disable=consider-using-f-string + if depth < 0: + raise ValueError("Somthing wrong with indentaion in root level.") + + has_categoty = False + for def_line in self.root_level_definition: + if def_line in ("category: application", "category: base"): + self.write_out(indent=0 * DEPTH_SIZE, text=def_line, file_out=file_out) + # file_out.write(f"{def_line}\n") + has_categoty = True + + if not has_categoty: + raise ValueError("Definition dose not get any category from 'base or application'.") + self.print_root_level_doc(file_out) + if 'symbols' in self.root_level_comment and self.root_level_comment['symbols'] != '': + indent = depth * DEPTH_SIZE + text = self.root_level_comment['symbols'] + self.write_out(indent, text, file_out) + if self.root_level_symbols: + self.write_out(indent=0 * DEPTH_SIZE, text=self.root_level_symbols, file_out=file_out) + # symbol_list include 'symbols doc', and all 'symbol' + for ind, symbol in enumerate(self.symbol_list): + # Taking care of comments that come on to of 'symbols doc' and 'symbol' + if 'symbol_comments' in self.root_level_comment and \ + self.root_level_comment['symbol_comments'][ind] != '': + indent = depth * DEPTH_SIZE + self.write_out(indent, + self.root_level_comment['symbol_comments'][ind], file_out) + if 'symbol_doc_comments' in self.root_level_comment and \ + self.root_level_comment['symbol_doc_comments'][ind] != '': + + indent = depth * DEPTH_SIZE + self.write_out(indent, + self.root_level_comment['symbol_doc_comments'][ind], file_out) + + self.write_out(indent=(0 * DEPTH_SIZE), text=symbol, file_out=file_out) + if len(self.pi_comments) > 1: + indent = DEPTH_SIZE * depth + # The first comment is top level copy-right doc string + for comment in self.pi_comments[1:]: + self.write_out(indent, self.comvert_to_ymal_comment(indent, comment), file_out) + if self.root_level_definition: + # Soring NXname for writting end of the definition attributes + nx_name = '' + for defs in self.root_level_definition: + if 'NX' in defs and defs[-1] == ':': + nx_name = defs + continue + if defs in ("category: application", "category: base"): + continue + self.write_out(indent=0 * DEPTH_SIZE, text=defs, file_out=file_out) + self.write_out(indent=0 * DEPTH_SIZE, text=nx_name, file_out=file_out) + self.found_definition = False + + def handle_exists(self, exists_dict, key, val): + """ + Create exist component as folows: + + {'min' : value for min, + 'max' : value for max, + 'optional' : value for optional} + + This is created separately so that the keys stays in order. + """ + if not val: + val = '' + else: + val = str(val) + if 'minOccurs' == key: + exists_dict['minOccurs'] = ['min', val] + if 'maxOccurs' == key: + exists_dict['maxOccurs'] = ['max', val] + if 'optional' == key: + exists_dict['optional'] = ['optional', val] + if 'recommended' == key: + exists_dict['recommended'] = ['recommended', val] + if 'required' == key: + exists_dict['required'] = ['required', val] + + # pylint: disable=too-many-branches, consider-using-f-string + def handle_group_or_field(self, depth, node, file_out): + """Handle all the possible attributes that come along a field or group""" + + allowed_attr = ['optional', 'recommended', 'name', 'type', 'axes', 'axis', 'data_offset', + 'interpretation', 'long_name', 'maxOccurs', 'minOccurs', 'nameType', + 'optional', 'primary', 'signal', 'stride', 'units', 'required', + 'deprecated', 'exists'] + + name_type = "" + node_attr = node.attrib + rm_key_list = [] + # Maintain order: name and type in form name(type) or (type)name that come first + for key, val in node_attr.items(): + if key == 'name': + name_type = name_type + val + rm_key_list.append(key) + if key == 'type': + name_type = name_type + "(%s)" % val + rm_key_list.append(key) + if not name_type: + raise ValueError(f"No 'name' or 'type' hase been found. But, 'group' or 'field' " + f"must have at list a nme.We got attributes: {node_attr}") + file_out.write('{indent}{name_type}:\n'.format( + indent=depth * DEPTH_SIZE, + name_type=name_type)) + + for key in rm_key_list: + del node_attr[key] + + # tmp_dict intended to persevere order of attribnutes + tmp_dict = {} + exists_dict = {} + for key, val in node_attr.items(): + # As both 'minOccurs', 'maxOccurs' and optionality move to the 'exists' + if key in ['minOccurs', 'maxOccurs', 'optional', 'recommended', 'required']: + if 'exists' not in tmp_dict: + tmp_dict['exists'] = [] + self.handle_exists(exists_dict, key, val) + elif key == 'units': + tmp_dict['unit'] = str(val) + else: + tmp_dict[key] = str(val) + if key not in allowed_attr: + raise ValueError(f"An attribute ({key}) in 'field' or 'group' has been found " + f"that is not allowed. The allowed attr is {allowed_attr}.") + + if exists_dict: + for key, val in exists_dict.items(): + if key in ['minOccurs', 'maxOccurs']: + tmp_dict['exists'] = tmp_dict['exists'] + val + elif key in ['optional', 'recommended', 'required']: + tmp_dict['exists'] = key + + depth_ = depth + 1 + for key, val in tmp_dict.items(): + # Increase depth size inside handle_map...() for writting text with one + # more indentation. + file_out.write(f'{depth_ * DEPTH_SIZE}{key}: ' + f'{handle_mapping_char(val, depth_ + 1, False)}\n') + + # pylint: disable=too-many-branches, too-many-locals + def handle_dimension(self, depth, node, file_out): + """ + Handle the dimension field. + NOTE: Usually we take care of any xml element in xmlparse(...) and + recursion_in_xml_tree(...) functions. But Here it is a bit different. The doc dimension + and attributes of dim has been handled inside this function here. + """ + # pylint: disable=consider-using-f-string + possible_dim_attrs = ['ref', 'optional', 'recommended', + 'required', 'incr', 'refindex'] + possible_dimemsion_attrs = ['rank'] + + # taking care of Dimension tag + file_out.write( + '{indent}{tag}:\n'.format( + indent=depth * DEPTH_SIZE, + tag=node.tag.split("}", 1)[1])) + # Taking care of dimension attributes + for attr, value in node.attrib.items(): + if attr in possible_dimemsion_attrs and not isinstance(value, dict): + indent = (depth + 1) * DEPTH_SIZE + file_out.write(f'{indent}{attr}: {value}\n') + else: + raise ValueError(f"Dimension has got an attribute {attr} that is not valid." + f"Current the allowd atributes are {possible_dimemsion_attrs}." + f" Please have a look") + # taking carew of dimension doc + for child in list(node): + tag = remove_namespace_from_tag(child.tag) + if tag == 'doc': + text = self.handle_not_root_level_doc(depth + 1, child.text) + file_out.write(text) + node.remove(child) + + dim_index_value = '' + dim_other_parts = {} + dim_cmnt_node = [] + # taking care of dim and doc childs of dimension + for child in list(node): + tag = remove_namespace_from_tag(child.tag) + child_attrs = child.attrib + # taking care of index and value attributes + if tag == ('dim'): + # taking care of index and value in format [[index, value]] + dim_index_value = dim_index_value + '[{index}, {value}], '.format( + index=child_attrs['index'] if "index" in child_attrs else '', + value=child_attrs['value'] if "value" in child_attrs else '') + if "index" in child_attrs: + del child_attrs["index"] + if "value" in child_attrs: + del child_attrs["value"] + + # Taking care of doc comes as child of dim + for cchild in list(child): + ttag = cchild.tag.split("}", 1)[1] + if ttag == ('doc'): + if ttag not in dim_other_parts: + dim_other_parts[ttag] = [] + text = cchild.text + dim_other_parts[ttag].append(text.strip()) + child.remove(cchild) + continue + # taking care of other attributes except index and value + for attr, value in child_attrs.items(): + if attr in possible_dim_attrs: + if attr not in dim_other_parts: + dim_other_parts[attr] = [] + dim_other_parts[attr].append(value) + if tag == CMNT_TAG and self.include_comment: + # Store and remove node so that comment nodes from dim node so + # that it does not call in xmlparser function + dim_cmnt_node.append(child) + node.remove(child) + + # All 'dim' element comments on top of 'dim' yaml key + if dim_cmnt_node: + for ch_nd in dim_cmnt_node: + self.handel_comment(depth + 1, ch_nd, file_out) + # index and value attributes of dim elements + file_out.write( + '{indent}dim: [{value}]\n'.format( + indent=(depth + 1) * DEPTH_SIZE, + value=dim_index_value[:-2] or '')) + # Write the attributes, except index and value, and doc of dim as child of dim_parameter. + # But tthe doc or attributes for each dim come inside list according to the order of dim. + if dim_other_parts: + file_out.write( + '{indent}dim_parameters:\n'.format( + indent=(depth + 1) * DEPTH_SIZE)) + # depth = depth + 2 dim_paramerter has child such as doc of dim + indent = (depth + 2) * DEPTH_SIZE + for key, value in dim_other_parts.items(): + if key == 'doc': + value = self.handle_not_root_level_doc(depth + 2, str(value), key, file_out) + else: + # Increase depth size inside handle_map...() for writting text with one + # more indentation. + file_out.write(f"{indent}{key}: " + f"{handle_mapping_char(value, depth + 3, False)}\n") + + def handle_enumeration(self, depth, node, file_out): + """ + Handle the enumeration field parsed from the xml file. + + If the enumeration items contain a doc field, the yaml file will contain items as child + fields of the enumeration field. + + If no doc are inherited in the enumeration items, a list of the items is given for the + enumeration list. + + """ + # pylint: disable=consider-using-f-string + + check_doc = [] + for child in list(node): + if list(child): + check_doc.append(list(child)) + # pylint: disable=too-many-nested-blocks + if check_doc: + file_out.write( + '{indent}{tag}: \n'.format( + indent=depth * DEPTH_SIZE, + tag=node.tag.split("}", 1)[1])) + for child in list(node): + tag = remove_namespace_from_tag(child.tag) + itm_depth = depth + 1 + if tag == ('item'): + file_out.write( + '{indent}{value}: \n'.format( + indent=(itm_depth) * DEPTH_SIZE, + value=child.attrib['value'])) + + if list(child): + for item_doc in list(child): + if remove_namespace_from_tag(item_doc.tag) == 'doc': + item_doc_depth = itm_depth + 1 + self.handle_not_root_level_doc(item_doc_depth, item_doc.text, + item_doc.tag, file_out) + if (remove_namespace_from_tag(item_doc.tag) == CMNT_TAG + and self.include_comment): + self.handel_comment(itm_depth + 1, item_doc, file_out) + if tag == CMNT_TAG and self.include_comment: + self.handel_comment(itm_depth + 1, child, file_out) + else: + enum_list = '' + remove_nodes = [] + for item_child in list(node): + tag = remove_namespace_from_tag(item_child.tag) + if tag == ('item'): + enum_list = enum_list + '{value}, '.format( + value=item_child.attrib['value']) + if tag == CMNT_TAG and self.include_comment: + self.handel_comment(depth, item_child, file_out) + remove_nodes.append(item_child) + for ch_node in remove_nodes: + node.remove(ch_node) + + file_out.write( + '{indent}{tag}: [{enum_list}]\n'.format( + indent=depth * DEPTH_SIZE, + tag=remove_namespace_from_tag(node.tag), + enum_list=enum_list[:-2] or '')) + + def handle_attributes(self, depth, node, file_out): + """Handle the attributes parsed from the xml file""" + + allowed_attr = ['name', 'type', 'units', 'nameType', 'recommended', 'optional', + 'minOccurs', 'maxOccurs', 'deprecated'] + + name = "" + node_attr = node.attrib + if 'name' in node_attr: + pass + else: + raise ValueError("Attribute must have an name key.") + rm_key_list = [] + # Maintain order: name and type in form name(type) or (type)name that come first + for key, val in node_attr.items(): + if key == 'name': + name = val + rm_key_list.append(key) + + for key in rm_key_list: + del node_attr[key] + + file_out.write('{indent}{escapesymbol}{name}:\n'.format( + indent=depth * DEPTH_SIZE, + escapesymbol=r'\@', + name=name)) + + tmp_dict = {} + exists_dict = {} + for key, val in node_attr.items(): + # As both 'minOccurs', 'maxOccurs' and optionality move to the 'exists' + if key in ['minOccurs', 'maxOccurs', 'optional', 'recommended', 'required']: + if 'exists' not in tmp_dict: + tmp_dict['exists'] = [] + self.handle_exists(exists_dict, key, val) + elif key == 'units': + tmp_dict['unit'] = val + else: + tmp_dict[key] = val + if key not in allowed_attr: + raise ValueError(f"An attribute ({key}) has been found that is not allowed." + f"The allowed attr is {allowed_attr}.") + + has_min_max = False + has_opt_reco_requ = False + if exists_dict: + for key, val in exists_dict.items(): + if key in ['minOccurs', 'maxOccurs']: + tmp_dict['exists'] = tmp_dict['exists'] + val + has_min_max = True + elif key in ['optional', 'recommended', 'required']: + tmp_dict['exists'] = key + has_opt_reco_requ = True + if has_min_max and has_opt_reco_requ: + raise ValueError("Optionality 'exists' can take only either from ['minOccurs'," + " 'maxOccurs'] or from ['optional', 'recommended', 'required']" + ". But not from both of the groups together. Please check in" + " attributes") + + depth_ = depth + 1 + for key, val in tmp_dict.items(): + # Increase depth size inside handle_map...() for writting text with one + # more indentation. + file_out.write(f'{depth_ * DEPTH_SIZE}{key}: ' + f'{handle_mapping_char(val, depth_ + 1, False)}\n') + + def handel_link(self, depth, node, file_out): + """ + Handle link elements of nxdl + """ + + possible_link_attrs = ['name', 'target', 'napimount'] + node_attr = node.attrib + # Handle special cases + if 'name' in node_attr: + file_out.write('{indent}{name}(link):\n'.format( + indent=depth * DEPTH_SIZE, + name=node_attr['name'] or '')) + del node_attr['name'] + + depth_ = depth + 1 + # Handle general cases + for attr_key, val in node_attr.items(): + if attr_key in possible_link_attrs: + file_out.write('{indent}{attr}: {value}\n'.format( + indent=depth_ * DEPTH_SIZE, + attr=attr_key, + value=val)) + else: + raise ValueError(f"An anexpected attribute '{attr_key}' of link has found." + f"At this moment the alloed keys are {possible_link_attrs}") + + def handel_choice(self, depth, node, file_out): + """ + Handle choice element which is a parent node of group. + """ + + possible_attr = [] + + node_attr = node.attrib + # Handle special casees + if 'name' in node_attr: + file_out.write('{indent}{attr}(choice): \n'.format( + indent=depth * DEPTH_SIZE, + attr=node_attr['name'])) + del node_attr['name'] + + depth_ = depth + 1 + # Taking care of general attrinutes. Though, still no attrinutes have found, + # but could be used for future + for attr in node_attr.items(): + if attr in possible_attr: + file_out.write('{indent}{attr}: {value}\n'.format( + indent=depth_ * DEPTH_SIZE, + attr=attr, + value=node_attr[attr])) + else: + raise ValueError(f"An unexpected attribute '{attr}' of 'choice' has been found." + f"At this moment attributes for choice {possible_attr}") + + def handel_comment(self, depth, node, file_out): + """ + Collect comment element and pass to write_out function + """ + indent = depth * DEPTH_SIZE + if self.is_last_element_comment: + text = self.comvert_to_ymal_comment(indent, node.text) + self.write_out(indent, text, file_out) + else: + text = self.comvert_to_ymal_comment(indent, node.text) + self.write_out(indent, text, file_out) + self.is_last_element_comment = True + + def recursion_in_xml_tree(self, depth, xml_tree, output_yml, verbose): + """ + Descend lower level in xml tree. If we are in the symbols branch, the recursive + behaviour is not triggered as we already handled the symbols' childs. + """ + + tree = xml_tree['tree'] + node = xml_tree['node'] + for child in list(node): + xml_tree_children = {'tree': tree, 'node': child} + self.xmlparse(output_yml, xml_tree_children, depth, verbose) + + # pylint: disable=too-many-branches, too-many-statements + def xmlparse(self, output_yml, xml_tree, depth, verbose): + """ + Main of the nxdl2yaml converter. + It parses XML tree, then prints recursively each level of the tree + """ + tree = xml_tree['tree'] + node = xml_tree['node'] + if verbose: + sys.stdout.write(f'Node tag: {remove_namespace_from_tag(node.tag)}\n') + sys.stdout.write(f'Attributes: {node.attrib}\n') + with open(output_yml, "a", encoding="utf-8") as file_out: + tag = remove_namespace_from_tag(node.tag) + if tag == ('definition'): + self.found_definition = True + self.handle_definition(node) + # Taking care of root level doc and symbols + remove_cmnt_n = None + last_comment = '' + for child in list(node): + tag_tmp = remove_namespace_from_tag(child.tag) + if tag_tmp == CMNT_TAG and self.include_comment: + last_comment = self.comvert_to_ymal_comment(depth * DEPTH_SIZE, child.text) + remove_cmnt_n = child + if tag_tmp == 'doc': + self.stroe_root_level_comments('root_doc', last_comment) + last_comment = '' + self.handle_root_level_doc(child) + node.remove(child) + if remove_cmnt_n is not None: + node.remove(remove_cmnt_n) + remove_cmnt_n = None + if tag_tmp == 'symbols': + self.stroe_root_level_comments('symbols', last_comment) + last_comment = '' + self.handle_symbols(depth, child) + node.remove(child) + if remove_cmnt_n is not None: + node.remove(remove_cmnt_n) + remove_cmnt_n = None + + if tag == ('doc') and depth != 1: + parent = get_node_parent_info(tree, node)[0] + doc_parent = remove_namespace_from_tag(parent.tag) + if doc_parent != 'item': + self.handle_not_root_level_doc(depth, text=node.text, + tag=node.tag, + file_out=file_out) + + if self.found_definition is True and self.root_level_doc: + self.print_root_level_info(depth, file_out) + # End of print root-level definitions in file + if tag in ('field', 'group') and depth != 0: + self.handle_group_or_field(depth, node, file_out) + if tag == ('enumeration'): + self.handle_enumeration(depth, node, file_out) + if tag == ('attribute'): + self.handle_attributes(depth, node, file_out) + if tag == ('dimensions'): + self.handle_dimension(depth, node, file_out) + if tag == ('link'): + self.handel_link(depth, node, file_out) + if tag == ('choice'): + self.handel_choice(depth, node, file_out) + if tag == CMNT_TAG and self.include_comment: + self.handel_comment(depth, node, file_out) + depth += 1 + # Write nested nodes + self.recursion_in_xml_tree(depth, xml_tree, output_yml, verbose) + + +def compare_niac_and_my(tree, tree2, verbose, node, root_no_duplicates): + """This function creates two trees with Niac XML file and My XML file. +The main aim is to compare the two trees and create a new one that is the +union of the two initial trees. + +""" + root = tree.getroot() + root2 = tree2.getroot() + attrs_list_niac = [] + for nodo in root.iter(node): + attrs_list_niac.append(nodo.attrib) + if verbose: + sys.stdout.write('Attributes found in Niac file: \n') + sys.stdout.write(str(attrs_list_niac) + '\n') + sys.stdout.write(' \n') + sys.stdout.write('Started merging of Niac and My file... \n') + for elem in root.iter(node): + if verbose: + sys.stdout.write('- Niac element inserted: \n') + sys.stdout.write(str(elem.attrib) + '\n') + index = get_node_parent_info(tree, elem)[1] + root_no_duplicates.insert(index, elem) + + for elem2 in root2.iter(node): + index = get_node_parent_info(tree2, elem2)[1] + if elem2.attrib not in attrs_list_niac: + if verbose: + sys.stdout.write('- My element inserted: \n') + sys.stdout.write(str(elem2.attrib) + '\n') + root_no_duplicates.insert(index, elem2) + + if verbose: + sys.stdout.write(' \n') + return root_no_duplicates diff --git a/nyaml2nxdl_forward_tools.py b/nyaml2nxdl_forward_tools.py new file mode 100644 index 0000000000..cee59d1f3d --- /dev/null +++ b/nyaml2nxdl_forward_tools.py @@ -0,0 +1,1072 @@ +#!/usr/bin/env python3 +"""Creates an instantiated NXDL schema XML tree by walking the dictionary nest + +""" +# -*- coding: utf-8 -*- +# +# Copyright The NOMAD Authors. +# +# This file is part of NOMAD. See https://nomad-lab.eu for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import sys +import xml.etree.ElementTree as ET +from xml.dom import minidom +import os +import textwrap + +import yaml + +from pynxtools.nexus import nexus +from pynxtools.nyaml2nxdl.comment_collector import CommentCollector +from pynxtools.nyaml2nxdl.nyaml2nxdl_helper import (get_yaml_escape_char_reverter_dict, + nx_name_type_resolving, + cleaning_empty_lines, LineLoader) + + +# pylint: disable=too-many-lines, global-statement, invalid-name +DOM_COMMENT = ("\n" + "# NeXus - Neutron and X-ray Common Data Format\n" + "# \n" + "# Copyright (C) 2014-2022 NeXus International Advisory Committee (NIAC)\n" + "# \n" + "# This library is free software; you can redistribute it and/or\n" + "# modify it under the terms of the GNU Lesser General Public\n" + "# License as published by the Free Software Foundation; either\n" + "# version 2 of the License, or (at your option) any later version.\n" + "#\n" + "# This library is distributed in the hope that it will be useful,\n" + "# but WITHOUT ANY WARRANTY; without even the implied warranty of\n" + "# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU\n" + "# Lesser General Public License for more details.\n" + "#\n" + "# You should have received a copy of the GNU Lesser General Public\n" + "# License along with this library; if not, write to the Free Software\n" + "# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA\n" + "#\n" + "# For further information, see http://www.nexusformat.org\n") +NX_CLSS = nexus.get_nx_classes() +NX_NEW_DEFINED_CLASSES = ['NX_COMPLEX'] +NX_TYPE_KEYS = nexus.get_nx_attribute_type() +NX_ATTR_IDNT = '\\@' +NX_UNIT_IDNT = 'unit' +DEPTH_SIZE = " " +NX_UNIT_TYPES = nexus.get_nx_units() +COMMENT_BLOCKS: CommentCollector + + +def yml_reader(inputfile): + """ + This function launches the LineLoader class. + It parses the yaml in a dict and extends it with line tag keys for each key of the dict. + """ + global COMMENT_BLOCKS + with open(inputfile, "r", encoding="utf-8") as plain_text_yaml: + loader = LineLoader(plain_text_yaml) + loaded_yaml = loader.get_single_data() + COMMENT_BLOCKS = CommentCollector(inputfile, loaded_yaml) + COMMENT_BLOCKS.extract_all_comment_blocks() + return loaded_yaml + + +def yml_reader_nolinetag(inputfile): + """ + pyyaml based parsing of yaml file in python dict + """ + with open(inputfile, 'r', encoding="utf-8") as stream: + parsed_yaml = yaml.safe_load(stream) + return parsed_yaml + + +def check_for_skiped_attributes(component, value, allowed_attr=None, verbose=False): + """ + Check for any attributes have been skipped or not. + NOTE: We should we should keep in mind about 'doc' + """ + block_tag = ['enumeration'] + if value: + for attr, val in value.items(): + if attr in ['doc']: + continue + if '__line__' in attr or attr in block_tag: + continue + line_number = f'__line__{attr}' + if verbose: + print(f"__line__ : {value[line_number]}") + if not isinstance(val, dict) \ + and '\\@' not in attr\ + and attr not in allowed_attr\ + and 'NX' not in attr and val: + + raise ValueError(f"An attribute '{attr}' in part '{component}' has been found" + f". Please check arround line '{value[line_number]}. At this " + f"moment. The allowed attrbutes are {allowed_attr}") + + +def check_optionality_and_write(obj, opl_key, opl_val): + """ + Taking care of optinality. + """ + if opl_key == 'optional': + if opl_val == 'false': + obj.set('required', 'true') + elif opl_key == 'minOccurs': + if opl_val == '0': + pass + else: + obj.set(opl_key, str(opl_val)) + + +def format_nxdl_doc(string): + """NeXus format for doc string + """ + string = check_for_mapping_char_other(string) + formatted_doc = '' + if "\n" not in string: + if len(string) > 80: + wrapped = textwrap.TextWrapper(width=80, + break_long_words=False, + replace_whitespace=False) + string = '\n'.join(wrapped.wrap(string)) + formatted_doc = '\n' + f"{string}" + else: + text_lines = string.split('\n') + text_lines = cleaning_empty_lines(text_lines) + formatted_doc += "\n" + "\n".join(text_lines) + if not formatted_doc.endswith("\n"): + formatted_doc += "\n" + return formatted_doc + + +def check_for_mapping_char_other(text): + """ + Check for mapping char \':\' which does not be passed through yaml library. + Then replace it by ':'. + """ + if not text: + text = '' + text = str(text) + if text == 'True': + text = 'true' + if text == 'False': + text = 'false' + # Some escape char is not valid in yaml libray which is written while writting + # yaml file. In the time of writting nxdl revert to that escape char. + escape_reverter = get_yaml_escape_char_reverter_dict() + for key, val in escape_reverter.items(): + if key in text: + text = text.replace(key, val) + return str(text) + + +def xml_handle_doc(obj, value: str, + line_number=None, line_loc=None): + """This function creates a 'doc' element instance, and appends it to an existing element + + """ + # global comment_bolcks + doc_elemt = ET.SubElement(obj, 'doc') + text = format_nxdl_doc(check_for_mapping_char_other(value)).strip() + # To keep the doc middle of doc tag. + doc_elemt.text = f"\n{text}\n" + if line_loc is not None and line_number is not None: + xml_handle_comment(obj, line_number, + line_loc, doc_elemt) + + +def xml_handle_units(obj, value): + """This function creates a 'units' element instance, and appends it to an existing element + + """ + obj.set('units', str(value)) + + +# pylint: disable=too-many-branches +def xml_handle_exists(dct, obj, keyword, value): + """ + This function creates an 'exists' element instance, and appends it to an existing element + """ + + line_number = f'__line__{keyword}' + assert value is not None, f'Line {dct[line_number]}: exists argument must not be None !' + if isinstance(value, list): + if len(value) == 2 and value[0] == 'min': + obj.set('minOccurs', str(value[1])) + elif len(value) == 2 and value[0] == 'max': + obj.set('maxOccurs', str(value[1])) + elif len(value) == 4 and value[0] == 'min' and value[2] == 'max': + obj.set('minOccurs', str(value[1])) + if str(value[3]) != 'infty': + obj.set('maxOccurs', str(value[3])) + else: + obj.set('maxOccurs', 'unbounded') + elif len(value) == 4 and value[0] == 'max' and value[2] == 'min': + obj.set('minOccurs', str(value[3])) + if str(value[1]) != 'infty': + obj.set('maxOccurs', str(value[3])) + else: + obj.set('maxOccurs', 'unbounded') + elif len(value) == 4 and (value[0] != 'min' or value[2] != 'max'): + raise ValueError(f'Line {dct[line_number]}: exists keyword' + f'needs to go either with an optional [recommended] list with two ' + f'entries either [min, ] or [max, ], or a list of four ' + f'entries [min, , max, ] !') + else: + raise ValueError(f'Line {dct[line_number]}: exists keyword ' + f'needs to go either with optional, recommended, a list with two ' + f'entries either [min, ] or [max, ], or a list of four ' + f'entries [min, , max, ] !') + else: + if value == 'optional': + obj.set('optional', 'true') + elif value == 'recommended': + obj.set('recommended', 'true') + elif value == 'required': + obj.set('required', 'true') + else: + obj.set('minOccurs', '0') + + +# pylint: disable=too-many-branches, too-many-locals, too-many-statements +def xml_handle_group(dct, obj, keyword, value, verbose=False): + """ + The function deals with group instances + """ + line_number = f'__line__{keyword}' + line_loc = dct[line_number] + xml_handle_comment(obj, line_number, line_loc) + list_of_attr = ['name', 'type', 'nameType', 'deprecated', 'optional', 'recommended', + 'exists', 'unit'] + l_bracket = -1 + r_bracket = -1 + if keyword.count('(') == 1: + l_bracket = keyword.index('(') + if keyword.count(')') == 1: + r_bracket = keyword.index(')') + + keyword_name, keyword_type = nx_name_type_resolving(keyword) + if not keyword_name and not keyword_type: + raise ValueError("A group must have both value and name. Check for group.") + grp = ET.SubElement(obj, 'group') + + # type come first + if l_bracket == 0 and r_bracket > 0: + grp.set('type', keyword_type) + if keyword_name: + grp.set('name', keyword_name) + elif l_bracket > 0: + grp.set('name', keyword_name) + if keyword_type: + grp.set('type', keyword_type) + else: + grp.set('name', keyword_name) + + if value: + rm_key_list = [] + for attr, vval in value.items(): + if '__line__' in attr: + continue + line_number = f"__line__{attr}" + line_loc = value[line_number] + if attr == 'doc': + xml_handle_doc(grp, vval, line_number, line_loc) + rm_key_list.append(attr) + rm_key_list.append(line_number) + elif attr == 'exists' and vval: + xml_handle_exists(value, grp, attr, vval) + rm_key_list.append(attr) + rm_key_list.append(line_number) + xml_handle_comment(obj, + line_number, line_loc, grp) + elif attr == 'unit': + xml_handle_units(grp, vval) + xml_handle_comment(obj, line_number, line_loc, grp) + elif attr in list_of_attr and not isinstance(vval, dict) and vval: + validate_field_attribute_and_value(attr, vval, list_of_attr, value) + grp.set(attr, check_for_mapping_char_other(vval)) + rm_key_list.append(attr) + rm_key_list.append(line_number) + xml_handle_comment(obj, line_number, line_loc, grp) + + for key in rm_key_list: + del value[key] + # Check for skipped attrinutes + check_for_skiped_attributes('group', value, list_of_attr, verbose) + if isinstance(value, dict) and value != {}: + recursive_build(grp, value, verbose) + + +def xml_handle_dimensions(dct, obj, keyword, value: dict): + """ + This function creates a 'dimensions' element instance, and appends it to an existing element + + NOTE: we could create xml_handle_dim() function. + But, the dim elements in yaml file is defined as 'dim =[[index, value]]' + but dim has other attributes such as 'ref' and also might have doc as chlid. + so in that sense 'dim' should have come as dict keeping attributes and child as members of + dict. + Regarding this situation all the attributes of 'dimensions' and child 'doc' has been + included here. + + Other attributes, except 'index' and 'value', of 'dim' comes under nested dict named + 'dim_parameter: + incr:[...]' + """ + + possible_dimension_attrs = ['rank'] + line_number = f'__line__{keyword}' + line_loc = dct[line_number] + assert 'dim' in value.keys(), (f"Line {line_loc}: No dim as child of dimension has " + f"been found.") + xml_handle_comment(obj, line_number, line_loc) + dims = ET.SubElement(obj, 'dimensions') + # Consider all the childs under dimension is dim element and + # its attributes +# val_attrs = list(value.keys()) + rm_key_list = [] + rank = '' + for key, val in value.items(): + if '__line__' in key: + continue + line_number = f"__line__{key}" + line_loc = value[line_number] + if key == 'rank': + rank = val or '' + if isinstance(rank, int) and rank < 0: + raise ValueError(f"Dimension must have some info about rank which is not " + f"available. Please check arround Line: {dct[line_number]}") + dims.set(key, str(val)) + rm_key_list.append(key) + rm_key_list.append(line_number) + xml_handle_comment(obj, line_number, line_loc, dims) + # Check dimension doc and handle it + elif key == 'doc' and isinstance(val, str): + xml_handle_doc(dims, val, line_number, line_loc) + rm_key_list.append(key) + rm_key_list.append(line_number) + elif key in possible_dimension_attrs and not isinstance(val, dict): + dims.set(key, str(val)) + rm_key_list.append(key) + rm_key_list.append(line_number) + xml_handle_comment(obj, line_number, line_loc, dims) + + for key in rm_key_list: + del value[key] + + xml_handle_dim_from_dimension_dict(dct, dims, keyword, value, rank=False) + + if isinstance(value, dict) and value != {}: + recursive_build(dims, value, verbose=None) + + +# pylint: disable=too-many-locals, too-many-arguments +def xml_handle_dim_from_dimension_dict(dct, dims_obj, keyword, value, rank, verbose=False): + """ + Handling dim element. + NOTE: The inputs 'keyword' and 'value' are as input for xml_handle_dimensions + function. please also read note in xml_handle_dimensions. + """ + + possible_dim_attrs = ['ref', 'optional', 'recommended', 'required', 'incr', 'refindex'] + header_line_number = f"__line__{keyword}" + dim_list = [] + rm_key_list = [] + # NOTE: dim doc and other attributes except 'index' and 'value' will come as list of value + # under dim_parameters + if not value: + return + rank = '' + # pylint: disable=too-many-nested-blocks + for attr, vvalue in value.items(): + if '__line__' in attr: + continue + + line_number = f"__line__{attr}" + line_loc = value[line_number] + # dim comes in precedence + if attr == 'dim': + # dim consists of list of [index, value] + llist_ind_value = vvalue + assert isinstance(llist_ind_value, list), (f'Line {value[line_number]}: dim' + f'argument not a list !') + xml_handle_comment(dims_obj, line_number, line_loc) + if isinstance(rank, int) and rank > 0: + assert rank == len(llist_ind_value), ( + f"Wrong dimension rank check around Line {dct[header_line_number]}.\n" + f"Line {[dct[header_line_number]]} rank value {rank} " + f"is not the same as dim array = " + f"{len(llist_ind_value)}.") + # Taking care of ind and value that comes as list of list + for dim_ind_val in llist_ind_value: + dim = ET.SubElement(dims_obj, 'dim') + + # Taking care of multidimensions or rank + if len(dim_ind_val) >= 1 and dim_ind_val[0]: + dim.set('index', str(dim_ind_val[0])) + if len(dim_ind_val) == 2 and dim_ind_val[1]: + dim.set('value', str(dim_ind_val[1])) + dim_list.append(dim) + rm_key_list.append(attr) + rm_key_list.append(line_number) + elif attr == 'dim_parameters' and isinstance(vvalue, dict): + xml_handle_comment(dims_obj, line_number, line_loc) + for kkkey, vvval in vvalue.items(): + if '__line__' in kkkey: + continue + cmnt_number = f'__line__{kkkey}' + cmnt_loc = vvalue[cmnt_number] + if kkkey == 'doc' and dim_list: + # doc comes as list of doc + for i, dim in enumerate(dim_list): + if isinstance(vvval, list) and i < len(vvval): + tmp_val = vvval[i] + xml_handle_doc(dim, vvval[i], cmnt_number, cmnt_loc) + # Check all the dim have doc if not skip + elif isinstance(vvval, list) and i >= len(vvval): + pass + else: + for i, dim in enumerate(dim_list): + # all atribute of dims comes as list + if isinstance(vvval, list) and i < len(vvval): + tmp_val = vvval[i] + dim.set(kkkey, str(tmp_val)) + + # Check all the dim have doc if not skip + elif isinstance(vvval, list) and i >= len(vvval): + pass + # All dim might have the same value for the same attribute + elif not isinstance(vvval, list): + tmp_val = value + dim.set(kkkey, str(tmp_val)) + rm_key_list.append(attr) + rm_key_list.append(line_number) + else: + raise ValueError(f"Got unexpected block except 'dim' and 'dim_parameters'." + f"Please check arround line {line_number}") + + for key in rm_key_list: + del value[key] + + check_for_skiped_attributes('dim', value, possible_dim_attrs, verbose) + + +def xml_handle_enumeration(dct, obj, keyword, value, verbose): + """This function creates an 'enumeration' element instance. + + Two cases are handled: + 1) the items are in a list + 2) the items are dictionaries and may contain a nested doc + """ + line_number = f'__line__{keyword}' + line_loc = dct[line_number] + xml_handle_comment(obj, line_number, line_loc) + enum = ET.SubElement(obj, 'enumeration') + + assert value is not None, f'Line {line_loc}: enumeration must \ +bear at least an argument !' + assert len( + value) >= 1, f'Line {dct[line_number]}: enumeration must not be an empty list!' + if isinstance(value, list): + for element in value: + itm = ET.SubElement(enum, 'item') + itm.set('value', str(element)) + if isinstance(value, dict) and value != {}: + for element in value.keys(): + if '__line__' not in element: + itm = ET.SubElement(enum, 'item') + itm.set('value', str(element)) + if isinstance(value[element], dict): + recursive_build(itm, value[element], verbose) + + +# pylint: disable=unused-argument +def xml_handle_link(dct, obj, keyword, value, verbose): + """ + If we have an NXDL link we decode the name attribute from (link)[:-6] + """ + + line_number = f"__line__{keyword}" + line_loc = dct[line_number] + xml_handle_comment(obj, line_number, line_loc) + possible_attrs = ['name', 'target', 'napimount'] + name = keyword[:-6] + link_obj = ET.SubElement(obj, 'link') + link_obj.set('name', str(name)) + + if value: + rm_key_list = [] + for attr, vval in value.items(): + if '__line__' in attr: + continue + line_number = f"__line__{attr}" + line_loc = value[line_number] + if attr == 'doc': + xml_handle_doc(link_obj, vval, line_number, line_loc) + rm_key_list.append(attr) + rm_key_list.append(line_number) + elif attr in possible_attrs and not isinstance(vval, dict): + if vval: + link_obj.set(attr, str(vval)) + rm_key_list.append(attr) + rm_key_list.append(line_number) + xml_handle_comment(obj, line_number, line_loc, link_obj) + + for key in rm_key_list: + del value[key] + # Check for skipped attrinutes + check_for_skiped_attributes('link', value, possible_attrs, verbose) + + if isinstance(value, dict) and value != {}: + recursive_build(link_obj, value, verbose=None) + + +def xml_handle_choice(dct, obj, keyword, value, verbose=False): + """ + Build choice xml elements. That consists of groups. + """ + line_number = f'__line__{keyword}' + line_loc = dct[line_number] + xml_handle_comment(obj, line_number, line_loc) + # Add attributes in possible if new attributs have been added nexus definition. + possible_attr = [] + choice_obj = ET.SubElement(obj, 'choice') + # take care of special attributes + name = keyword[:-8] + choice_obj.set('name', name) + + if value: + rm_key_list = [] + for attr, vval in value.items(): + if '__line__' in attr: + continue + line_number = f"__line__{attr}" + line_loc = value[line_number] + if attr == 'doc': + xml_handle_doc(choice_obj, vval, line_number, line_loc) + rm_key_list.append(attr) + rm_key_list.append(line_number) + elif attr in possible_attr and not isinstance(vval, dict): + if vval: + choice_obj.set(attr, str(vval)) + rm_key_list.append(attr) + rm_key_list.append(line_number) + xml_handle_comment(obj, line_number, line_loc, choice_obj) + + for key in rm_key_list: + del value[key] + # Check for skipped attrinutes + check_for_skiped_attributes('choice', value, possible_attr, verbose) + + if isinstance(value, dict) and value != {}: + recursive_build(choice_obj, value, verbose=None) + + +def xml_handle_symbols(dct, obj, keyword, value: dict): + """Handle a set of NXDL symbols as a child to obj + + """ + line_number = f'__line__{keyword}' + line_loc = dct[line_number] + assert len(list(value.keys()) + ) >= 1, f'Line {line_loc}: symbols table must not be empty !' + xml_handle_comment(obj, line_number, line_loc) + syms = ET.SubElement(obj, 'symbols') + if 'doc' in value.keys(): + line_number = '__line__doc' + line_loc = value[line_number] + xml_handle_comment(syms, line_number, line_loc) + doctag = ET.SubElement(syms, 'doc') + doctag.text = '\n' + textwrap.fill(value['doc'], width=70) + '\n' + rm_key_list = [] + for kkeyword, vvalue in value.items(): + if '__line__' in kkeyword: + continue + if kkeyword != 'doc': + line_number = f'__line__{kkeyword}' + line_loc = value[line_number] + xml_handle_comment(syms, line_number, line_loc) + assert vvalue is not None and isinstance( + vvalue, str), f'Line {line_loc}: put a comment in doc string !' + sym = ET.SubElement(syms, 'symbol') + sym.set('name', str(kkeyword)) + # sym_doc = ET.SubElement(sym, 'doc') + xml_handle_doc(sym, vvalue) + rm_key_list.append(kkeyword) + rm_key_list.append(line_number) + # sym_doc.text = '\n' + textwrap.fill(vvalue, width=70) + '\n' + for key in rm_key_list: + del value[key] + + +def check_keyword_variable(verbose, dct, keyword, value): + """ + Check whether both keyword_name and keyword_type are empty, + and complains if it is the case + """ + keyword_name, keyword_type = nx_name_type_resolving(keyword) + if verbose: + sys.stdout.write( + f'{keyword_name}({keyword_type}): value type is {type(value)}\n') + if keyword_name == '' and keyword_type == '': + line_number = f'__line__{keyword}' + raise ValueError(f'Line {dct[line_number]}: found an improper yaml key !') + + +def helper_keyword_type(kkeyword_type): + """ + This function is returning a value of keyword_type if it belong to NX_TYPE_KEYS + """ + if kkeyword_type in NX_TYPE_KEYS: + return kkeyword_type + return None + + +def verbose_flag(verbose, keyword, value): + """ + Verbose stdout printing for nested levels of yaml file, if verbose flag is active + """ + if verbose: + sys.stdout.write(f' key:{keyword}; value type is {type(value)}\n') + + +def xml_handle_attributes(dct, obj, keyword, value, verbose): + """Handle the attributes found connected to attribute field""" + + line_number = f"__line__{keyword}" + line_loc = dct[line_number] + xml_handle_comment(obj, line_number, line_loc) + # list of possible attribute of xml attribute elementsa + attr_attr_list = ['name', 'type', 'unit', 'nameType', + 'optional', 'recommended', 'minOccurs', + 'maxOccurs', 'deprecated', 'exists'] + # as an attribute identifier + keyword_name, keyword_typ = nx_name_type_resolving(keyword) + line_number = f'__line__{keyword}' + if verbose: + print(f"__line__ : {dct[line_number]}") + if keyword_name == '' and keyword_typ == '': + raise ValueError(f'Line {dct[line_number]}: found an improper yaml key !') + elemt_obj = ET.SubElement(obj, 'attribute') + elemt_obj.set('name', keyword_name[2:]) + if keyword_typ: + elemt_obj.set('type', keyword_typ) + + rm_key_list = [] + if value and value: + # taking care of attributes of attributes + for attr, attr_val in value.items(): + if '__line__' in attr: + continue + line_number = f"__line__{attr}" + line_loc = value[line_number] + if attr in ['doc', *attr_attr_list] and not isinstance(attr_val, dict): + if attr == 'unit': + elemt_obj.set(f"{attr}s", str(value[attr])) + rm_key_list.append(attr) + rm_key_list.append(line_number) + xml_handle_comment(obj, line_number, line_loc, elemt_obj) + elif attr == 'exists' and attr_val: + xml_handle_exists(value, elemt_obj, attr, attr_val) + rm_key_list.append(attr) + rm_key_list.append(line_number) + xml_handle_comment(obj, line_number, line_loc, elemt_obj) + elif attr == 'doc': + xml_handle_doc(elemt_obj, format_nxdl_doc(attr_val), + line_number, line_loc) + rm_key_list.append(attr) + rm_key_list.append(line_number) + else: + elemt_obj.set(attr, check_for_mapping_char_other(attr_val)) + rm_key_list.append(attr) + rm_key_list.append(line_number) + xml_handle_comment(obj, line_number, line_loc, elemt_obj) + + for key in rm_key_list: + del value[key] + # Check cor skiped attribute + check_for_skiped_attributes('Attribute', value, attr_attr_list, verbose) + if value: + recursive_build(elemt_obj, value, verbose) + + +def validate_field_attribute_and_value(v_attr, vval, allowed_attribute, value): + """ + Check for any attributes that comes with invalid name, + and invalid value. + """ + + # check for empty val + if (not isinstance(vval, dict) + and not str(vval)): # check for empty value + + line_number = f"__line__{v_attr}" + raise ValueError(f"In a field a valid attrbute ('{v_attr}') found that is not stored." + f" Please check arround line {value[line_number]}") + + # The bellow elements might come as child element + skipped_child_name = ['doc', 'dimension', 'enumeration', 'choice', 'exists'] + # check for invalid key or attributes + if (v_attr not in [*skipped_child_name, *allowed_attribute] + and '__line__' not in v_attr + and not isinstance(vval, dict) + and '(' not in v_attr # skip only groups and field that has name and type + and '\\@' not in v_attr): # skip nexus attributes + + line_number = f"__line__{v_attr}" + raise ValueError(f"In a field or group a invalid attribute ('{v_attr}') or child has found." + f" Please check arround line {value[line_number]}.") + + +def xml_handle_fields(obj, keyword, value, line_annot, line_loc, verbose=False): + """ + Handle a field in yaml file. + When a keyword is NOT: + symbol, + NX baseclass member, + attribute (\\@), + doc, + enumerations, + dimension, + exists, + then the not empty keyword_name is a field! + This simple function will define a new node of xml tree + """ + + # List of possible attributes of xml elements + allowed_attr = ['name', 'type', 'nameType', 'unit', 'minOccurs', 'long_name', + 'axis', 'signal', 'deprecated', 'axes', 'exists', + 'data_offset', 'interpretation', 'maxOccurs', + 'primary', 'recommended', 'optional', 'stride'] + + xml_handle_comment(obj, line_annot, line_loc) + l_bracket = -1 + r_bracket = -1 + if keyword.count('(') == 1: + l_bracket = keyword.index('(') + if keyword.count(')') == 1: + r_bracket = keyword.index(')') + + keyword_name, keyword_type = nx_name_type_resolving(keyword) + if not keyword_type and not keyword_name: + raise ValueError("Check for name or type in field.") + elemt_obj = ET.SubElement(obj, 'field') + + # type come first + if l_bracket == 0 and r_bracket > 0: + elemt_obj.set('type', keyword_type) + if keyword_name: + elemt_obj.set('name', keyword_name) + elif l_bracket > 0: + elemt_obj.set('name', keyword_name) + if keyword_type: + elemt_obj.set('type', keyword_type) + else: + elemt_obj.set('name', keyword_name) + + if value: + rm_key_list = [] + # In each each if clause apply xml_handle_comment(), to collect + # comments on that yaml line. + for attr, vval in value.items(): + if '__line__' in attr: + continue + line_number = f"__line__{attr}" + line_loc = value[line_number] + if attr == 'doc': + xml_handle_doc(elemt_obj, vval, line_number, line_loc,) + rm_key_list.append(attr) + rm_key_list.append(line_number) + elif attr == 'exists' and vval: + xml_handle_exists(value, elemt_obj, attr, vval) + rm_key_list.append(attr) + rm_key_list.append(line_number) + xml_handle_comment(obj, + line_number, + line_loc, elemt_obj) + elif attr == 'unit': + xml_handle_units(elemt_obj, vval) + xml_handle_comment(obj, + line_number, + line_loc, elemt_obj) + elif attr in allowed_attr and not isinstance(vval, dict) and vval: + validate_field_attribute_and_value(attr, vval, allowed_attr, value) + elemt_obj.set(attr, check_for_mapping_char_other(vval)) + rm_key_list.append(attr) + rm_key_list.append(line_number) + xml_handle_comment(obj, + line_number, + line_loc, elemt_obj) + + for key in rm_key_list: + del value[key] + # Check for skipped attrinutes + check_for_skiped_attributes('field', value, allowed_attr, verbose) + + if isinstance(value, dict) and value != {}: + recursive_build(elemt_obj, value, verbose) + + +def xml_handle_comment(obj: ET.Element, + line_annotation: str, + line_loc_no: int, + xml_ele: ET.Element = None, + is_def_cmnt: bool = False): + """ + Add xml comment: check for comments that has the same 'line_annotation' + (e.g. __line__data) and the same line_loc_no (e.g. 30). After that, i + does of three tasks: + 1. Returns list of comments texts (multiple members if element has multiple comments) + 2. Rearrange comment element and xml_ele where comment comes first. + 3. Append comment element when no xml_ele will no be provided. + """ + + line_info = (line_annotation, int(line_loc_no)) + if line_info in COMMENT_BLOCKS: + cmnt = COMMENT_BLOCKS.get_coment_by_line_info(line_info) + cmnt_text = cmnt.get_comment_text() + + if is_def_cmnt: + return cmnt_text + if xml_ele is not None: + obj.remove(xml_ele) + for string in cmnt_text: + si_comnt = ET.Comment(string) + obj.append(si_comnt) + obj.append(xml_ele) + elif not is_def_cmnt and xml_ele is None: + for string in cmnt_text: + si_comnt = ET.Comment(string) + obj.append(si_comnt) + else: + raise ValueError("Provied correct parameter values.") + return '' + + +def recursive_build(obj, dct, verbose): + """obj is the current node of the XML tree where we want to append to, + dct is a dictionary object which represents the content of a child to obj + dct may contain further dictionary nests, representing NXDL groups, + which trigger recursive processing + NXDL fields may contain attributes but trigger no recursion so attributes are leafs. + + """ + for keyword, value in iter(dct.items()): + if '__line__' in keyword: + continue + line_number = f"__line__{keyword}" + line_loc = dct[line_number] + keyword_name, keyword_type = nx_name_type_resolving(keyword) + check_keyword_variable(verbose, dct, keyword, value) + if verbose: + sys.stdout.write( + f'keyword_name:{keyword_name} keyword_type {keyword_type}\n') + + if keyword[-6:] == '(link)': + xml_handle_link(dct, obj, keyword, value, verbose) + elif keyword[-8:] == '(choice)': + xml_handle_choice(dct, obj, keyword, value) + # The bellow xml_symbol clause is for the symbols that come ubde filed or attributes + # Root level symbols has been inside nyaml2nxdl() + elif keyword_type == '' and keyword_name == 'symbols': + xml_handle_symbols(dct, obj, keyword, value) + + elif ((keyword_type in NX_CLSS) or (keyword_type not in + [*NX_TYPE_KEYS, '', *NX_NEW_DEFINED_CLASSES])): + # we can be sure we need to instantiate a new group + xml_handle_group(dct, obj, keyword, value, verbose) + + elif keyword_name[0:2] == NX_ATTR_IDNT: # check if obj qualifies + xml_handle_attributes(dct, obj, keyword, value, verbose) + elif keyword == 'doc': + xml_handle_doc(obj, value, line_number, line_loc) + elif keyword == NX_UNIT_IDNT: + xml_handle_units(obj, value) + elif keyword == 'enumeration': + xml_handle_enumeration(dct, obj, keyword, value, verbose) + + elif keyword == 'dimensions': + xml_handle_dimensions(dct, obj, keyword, value) + + elif keyword == 'exists': + xml_handle_exists(dct, obj, keyword, value) + # Handles fileds e.g. AXISNAME + elif keyword_name != '' and '__line__' not in keyword_name: + xml_handle_fields(obj, keyword, + value, line_number, + line_loc, verbose) + else: + raise ValueError(f"An unfamiliar type of element {keyword} has been found which is " + f"not be able to be resolved. Chekc arround line {dct[line_number]}") + + +def pretty_print_xml(xml_root, output_xml, def_comments=None): + """ + Print better human-readable indented and formatted xml file using + built-in libraries and preceding XML processing instruction + """ + dom = minidom.parseString(ET.tostring( + xml_root, encoding='utf-8', method='xml')) + proc_instractionn = dom.createProcessingInstruction( + 'xml-stylesheet', 'type="text/xsl" href="nxdlformat.xsl"') + dom_comment = dom.createComment(DOM_COMMENT) + root = dom.firstChild + dom.insertBefore(proc_instractionn, root) + dom.insertBefore(dom_comment, root) + + if def_comments: + for string in def_comments: + def_comt_ele = dom.createComment(string) + dom.insertBefore(def_comt_ele, root) + + xml_string = dom.toprettyxml(indent=1 * DEPTH_SIZE, newl='\n', encoding='UTF-8') + with open('tmp.xml', "wb") as file_tmp: + file_tmp.write(xml_string) + flag = False + with open('tmp.xml', "r", encoding="utf-8") as file_out: + with open(output_xml, "w", encoding="utf-8") as file_out_mod: + for i in file_out.readlines(): + if '' not in i and '' not in i and flag is False: + file_out_mod.write(i) + elif '' in i and '' in i: + file_out_mod.write(i) + elif '' in i and '' not in i: + flag = True + white_spaces = len(i) - len(i.lstrip()) + file_out_mod.write(i) + elif '' not in i and '' not in i and flag is True: + file_out_mod.write((white_spaces + 5) * ' ' + i) + elif '' not in i and '' in i and flag is True: + file_out_mod.write(white_spaces * ' ' + i) + flag = False + os.remove('tmp.xml') + + +# pylint: disable=too-many-statements +def nyaml2nxdl(input_file: str, out_file, verbose: bool): + """ + Main of the nyaml2nxdl converter, creates XML tree, namespace and + schema, definitions then evaluates a dictionary nest of groups recursively and + fields or (their) attributes as childs of the groups + """ + + def_attributes = ['deprecated', 'ignoreExtraGroups', 'category', 'type', + 'ignoreExtraFields', 'ignoreExtraAttributes', 'restricts'] + yml_appdef = yml_reader(input_file) + def_cmnt_text = [] + if verbose: + sys.stdout.write(f'input-file: {input_file}\n') + sys.stdout.write('application/base contains the following root-level entries:\n') + sys.stdout.write(str(yml_appdef.keys())) + xml_root = ET.Element('definition', {}) + assert 'category' in yml_appdef.keys( + ), 'Required root-level keyword category is missing!' + assert yml_appdef['category'] in ['application', 'base'], 'Only \ +application and base are valid categories!' + assert 'doc' in yml_appdef.keys(), 'Required root-level keyword doc is missing!' + + name_extends = '' + yml_appdef_copy = yml_appdef.copy() + for kkey, vvalue in yml_appdef_copy.items(): + if '__line__' in kkey: + continue + line_number = f"__line__{kkey}" + line_loc_no = yml_appdef[line_number] + if not isinstance(vvalue, dict) and kkey in def_attributes: + xml_root.set(kkey, str(vvalue) or '') + cmnt_text = xml_handle_comment(xml_root, + line_number, line_loc_no, + is_def_cmnt=True) + def_cmnt_text += cmnt_text if cmnt_text else [] + + del yml_appdef[line_number] + del yml_appdef[kkey] + # Taking care or name and extends + elif 'NX' in kkey: + # Tacking the attribute order but the correct value will be stored later + # check for name first or type first if (NXobject)NXname then type first + l_bracket_ind = kkey.rfind('(') + r_bracket_ind = kkey.rfind(')') + if l_bracket_ind == 0: + extend = kkey[1:r_bracket_ind] + name = kkey[r_bracket_ind + 1:] + xml_root.set('extends', extend) + xml_root.set('name', name) + elif l_bracket_ind > 0: + name = kkey[0:l_bracket_ind] + extend = kkey[l_bracket_ind + 1: r_bracket_ind] + xml_root.set('name', name) + xml_root.set('extends', extend) + else: + name = kkey + xml_root.set('name', name) + xml_root.set('extends', 'NXobject') + cmnt_text = xml_handle_comment(xml_root, + line_number, line_loc_no, + is_def_cmnt=True) + def_cmnt_text += cmnt_text if cmnt_text else [] + + name_extends = kkey + + if 'type' not in xml_root.attrib: + xml_root.set('type', "group") + # Taking care of namespaces + namespaces = {'xmlns': 'http://definition.nexusformat.org/nxdl/3.1', + 'xmlns:xsi': 'http://www.w3.org/2001/XMLSchema-instance', + 'xsi:schemaLocation': 'http://definition.nexusformat.org/nxdl/3.1 ../nxdl.xsd'} + for key, ns_ in namespaces.items(): + xml_root.attrib[key] = ns_ + # Taking care of Symbols elements + if 'symbols' in yml_appdef.keys(): + xml_handle_symbols(yml_appdef, + xml_root, + 'symbols', + yml_appdef['symbols']) + + del yml_appdef['symbols'] + del yml_appdef["__line__symbols"] + + assert isinstance(yml_appdef['doc'], str) and yml_appdef['doc'] != '', 'Doc \ +has to be a non-empty string!' + + line_number = '__line__doc' + line_loc_no = yml_appdef[line_number] + xml_handle_doc(xml_root, yml_appdef['doc'], line_number, line_loc_no) + + del yml_appdef['doc'] + + root_keys = 0 + for key in yml_appdef.keys(): + if '__line__' not in key: + root_keys += 1 + extra_key = key + + assert root_keys == 1, (f"Accepting at most keywords: category, doc, symbols, and NX... " + f"at root-level! check key at root level {extra_key}") + + assert ('NX' in name_extends and len(name_extends) > 2), 'NX \ +keyword has an invalid pattern, or is too short!' + # Taking care if definition has empty content + if yml_appdef[name_extends]: + recursive_build(xml_root, yml_appdef[name_extends], verbose) + # Taking care of comments that comes at the end of file that is might not be intended for + # any nxdl elements. + if COMMENT_BLOCKS[-1].has_post_comment: + post_comment = COMMENT_BLOCKS[-1] + (lin_annot, line_loc) = post_comment.get_line_info() + xml_handle_comment(xml_root, lin_annot, line_loc) + + pretty_print_xml(xml_root, out_file, def_cmnt_text) + if verbose: + sys.stdout.write('Parsed YAML to NXDL successfully\n') diff --git a/nyaml2nxdl_helper.py b/nyaml2nxdl_helper.py new file mode 100644 index 0000000000..0fb51237d7 --- /dev/null +++ b/nyaml2nxdl_helper.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +"""Main file of yaml2nxdl tool. +Users create NeXus instances by writing a YAML file +which details a hierarchy of data/metadata elements + +""" +# -*- coding: utf-8 -*- +# +# Copyright The NOMAD Authors. +# +# This file is part of NOMAD. See https://nomad-lab.eu for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +# Yaml library does not except the keys (escapechar "\t" and yaml separator ":") +# So the corresponding value is to skip them and +# and also carefull about this order +from yaml.composer import Composer +from yaml.constructor import Constructor + +from yaml.nodes import ScalarNode +from yaml.resolver import BaseResolver +from yaml.loader import Loader + +ESCAPE_CHAR_DICT = {"\t": " "} + + +class LineLoader(Loader): # pylint: disable=too-many-ancestors + """ + LineLoader parses a yaml into a python dictionary extended with extra items. + The new items have as keys __line__ and as values the yaml file line number + """ + + def compose_node(self, parent, index): + # the line number where the previous token has ended (plus empty lines) + node = Composer.compose_node(self, parent, index) + node.__line__ = self.line + 1 + return node + + def construct_mapping(self, node, deep=False): + node_pair_lst = node.value + node_pair_lst_for_appending = [] + + for key_node in node_pair_lst: + shadow_key_node = ScalarNode( + tag=BaseResolver.DEFAULT_SCALAR_TAG, value='__line__' + key_node[0].value) + shadow_value_node = ScalarNode( + tag=BaseResolver.DEFAULT_SCALAR_TAG, value=key_node[0].__line__) + node_pair_lst_for_appending.append( + (shadow_key_node, shadow_value_node)) + + node.value = node_pair_lst + node_pair_lst_for_appending + return Constructor.construct_mapping(self, node, deep=deep) + + +def get_yaml_escape_char_dict(): + """Get escape char and the way to skip them in yaml.""" + return ESCAPE_CHAR_DICT + + +def get_yaml_escape_char_reverter_dict(): + """To revert yaml escape char in xml constructor from yaml.""" + temp_dict = {} + for key, val in ESCAPE_CHAR_DICT.items(): + temp_dict[val] = key + return temp_dict + + +def type_check(nx_type): + """ + Check for nexus type if type is NX_CHAR get '' or get as it is. + """ + + if nx_type in ['NX_CHAR', '']: + nx_type = '' + else: + nx_type = f"({nx_type})" + return nx_type + + +def get_node_parent_info(tree, node): + """ + Return tuple of (parent, index) where: + parent = node of parent within tree + index = index of node under parent + """ + + parent_map = {c: p for p in tree.iter() for c in p} + parent = parent_map[node] + return parent, list(parent).index(node) + + +def cleaning_empty_lines(line_list): + """ + Cleaning up empty lines on top and bottom. + """ + + if not isinstance(line_list, list): + line_list = line_list.split('\n') if '\n' in line_list else [''] + + # Clining up top empty lines + while True: + if line_list[0].strip(): + break + line_list = line_list[1:] + # Clining bottom empty lines + while True: + if line_list[-1].strip(): + break + line_list = line_list[0:-1] + + return line_list + + +def nx_name_type_resolving(tmp): + """ + extracts the eventually custom name {optional_string} + and type {nexus_type} from a YML section string. + YML section string syntax: optional_string(nexus_type) + """ + if tmp.count('(') == 1 and tmp.count(')') == 1: + # we can safely assume that every valid YML key resolves + # either an nx_ (type, base, candidate) class contains only 1 '(' and ')' + index_start = tmp.index('(') + index_end = tmp.index(')', index_start + 1) + typ = tmp[index_start + 1:index_end] + nam = tmp.replace('(' + typ + ')', '') + return nam, typ + + # or a name for a member + typ = '' + nam = tmp + return nam, typ From d413881a91633b5e7214bfa7fbb7bd78f20911dc Mon Sep 17 00:00:00 2001 From: Florian Dobener Date: Fri, 21 Apr 2023 17:10:43 +0200 Subject: [PATCH 11/93] Renames to pynxtools (#93) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Renames to pynxtools geändert: .github/workflows/pylint.yml geändert: .gitmodules modified: .github/workflows/pylint.yml * Properly adds definitions * Rename left over nexusutils reference from rebase * Merges xps reader.py --- test_nyaml2nxdl.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test_nyaml2nxdl.py b/test_nyaml2nxdl.py index 5254f54640..75072131c1 100755 --- a/test_nyaml2nxdl.py +++ b/test_nyaml2nxdl.py @@ -28,8 +28,8 @@ import xml.etree.ElementTree as ET import pytest from click.testing import CliRunner -import nexusutils.nyaml2nxdl.nyaml2nxdl as nyml2nxdl -from nexusutils.nyaml2nxdl import nyaml2nxdl_forward_tools +import pynxtools.nyaml2nxdl.nyaml2nxdl as nyml2nxdl +from pynxtools.nyaml2nxdl import nyaml2nxdl_forward_tools def delete_duplicates(list_of_matching_string): @@ -293,8 +293,8 @@ def test_yml_consistency_comment_parsing(): """Test comments parsing from yaml. Convert 'yaml' input file to '.nxdl.xml' and '.nxdl.xml' to '.yaml' """ - from nexusutils.nyaml2nxdl.comment_collector import CommentCollector - from nexusutils.nyaml2nxdl.nyaml2nxdl_helper import LineLoader + from pynxtools.nyaml2nxdl.comment_collector import CommentCollector + from pynxtools.nyaml2nxdl.nyaml2nxdl_helper import LineLoader ref_yml_file = 'tests/data/nyaml2nxdl/Ref_NXcomment.yaml' test_yml_file = 'tests/data/nyaml2nxdl/Ref_NXcomment_consistency.yaml' From cb08bb27225e13f2fa6cbbbb242e824d561be141 Mon Sep 17 00:00:00 2001 From: RubelMozumder <32923026+RubelMozumder@users.noreply.github.com> Date: Mon, 8 May 2023 13:59:09 +0200 Subject: [PATCH 12/93] Changes for removing old errors related ':' in yaml and xml file. (#96) * Changes for removing old errors related ':' in yaml and xml file. * To store nxdl file into yaml as comment and generate or retrieve nxdl from provided yaml. Wokring perfectly. * Bug: last line of yaml file was not counted * Passing all the tests, pylint, mypy and pydocstyle. * Fixing copy-right doc, new_line_char at the end of 'deprecated key', version from of nexus. * pylint.yml checked. --------- Co-authored-by: RubelMozumder --- README.md | 3 ++ comment_collector.py | 31 ++++++++++--- nyaml2nxdl.py | 39 +++++++++++++++-- nyaml2nxdl_backward_tools.py | 21 +++++---- nyaml2nxdl_forward_tools.py | 43 +++++++++++++++++- nyaml2nxdl_helper.py | 85 +++++++++++++++++++++++++++++++++--- 6 files changed, 195 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 6a0bbf86a7..ff083e1896 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,9 @@ Options: --input-file TEXT The path to the input data file to read. --append TEXT Parse xml NeXus file and append to specified base class, write the base class name with no extension. + --check-consistency Check consistency by generating another version of the input file. + E.g. for input file: NXexample.nxdl.xml the output file + NXexample_consistency.nxdl.xml. --verbose Addictional std output info is printed to help debugging. --help Show this message and exit. diff --git a/comment_collector.py b/comment_collector.py index a2aa54b782..5f0c5e3bce 100644 --- a/comment_collector.py +++ b/comment_collector.py @@ -89,11 +89,17 @@ def extract_all_comment_blocks(self): for line_num, line in enumerate(lines): if single_comment.is_storing_single_comment(): # If the last comment comes without post nxdl fields, groups and attributes + if '++ SHA HASH ++' in line: + # Handle with stored nxdl.xml file that is not part of yaml + line = '' + single_comment.process_each_line(line + 'post_comment', (line_num + 1)) + self._comment_chain.append(single_comment) + break if line_num < (len(lines) - 1): # Processing file from Line number 1 single_comment.process_each_line(line, (line_num + 1)) else: - # For processing post comment + # For processing last line of file single_comment.process_each_line(line + 'post_comment', (line_num + 1)) self._comment_chain.append(single_comment) else: @@ -122,6 +128,14 @@ def get_coment_by_line_info(self, comment_locs: Tuple[str, Union[int, str]]): self._comment_hash[comment_locs] = cmnt return cmnt + def remove_comment(self, ind): + """Remove a comment from comment list. + """ + if ind < len(self._comment_chain): + del self._comment_chain[ind] + else: + raise ValueError("Oops! Index is out of range.") + def reload_comment(self): """ Update self._comment_tracker after done with last comment. @@ -150,9 +164,15 @@ def __contains__(self, comment_locs: tuple): def __getitem__(self, ind): """Get comment from self.obj._comment_chain by index. """ - if ind >= len(self._comment_chain): - raise IndexError(f'Oops! Coment index {ind} in {__class__} is out of range!') - return self._comment_chain[ind] + if isinstance(ind, int): + if ind >= len(self._comment_chain): + raise IndexError(f'Oops! Comment index {ind} in {__class__} is out of range!') + return self._comment_chain[ind] + + if isinstance(ind, slice): + start_n = ind.start or 0 + end_n = ind.stop or len(self._comment_chain) + return self._comment_chain[start_n:end_n] def __iter__(self): """get comment ieratively @@ -230,6 +250,7 @@ def process_each_line(self, text, line_num): if text and line_num: self.append_comment(text) if self._comnt_end_found and not self._is_elemt_found: + # for multiple comment if exist if self._comnt: self._comnt_list.append(self._comnt) self._comnt = '' @@ -455,7 +476,7 @@ def get_element_location(self): f"{self._elemt}") for key, val in self._elemt.items(): - return key, val + yield key, val def collect_yaml_line_info(self, yaml_dict, line_info_dict): """Collect __line__key and corresponding value from diff --git a/nyaml2nxdl.py b/nyaml2nxdl.py index 3088a25e52..bfd62cfdad 100755 --- a/nyaml2nxdl.py +++ b/nyaml2nxdl.py @@ -26,7 +26,9 @@ import xml.etree.ElementTree as ET import click - +from pynxtools.nyaml2nxdl.nyaml2nxdl_helper import (get_sha256_hash, + extend_yamlfile_with_comment, + separate_hash_yaml_and_nxdl) from pynxtools.nyaml2nxdl.nyaml2nxdl_forward_tools import nyaml2nxdl, pretty_print_xml from pynxtools.nyaml2nxdl.nyaml2nxdl_backward_tools import (Nxdl2yaml, compare_niac_and_my) @@ -35,6 +37,27 @@ DEPTH_SIZE = " " +def generate_nxdl_or_retrieve_nxdl(yaml_file, out_xml_file, verbose): + """ + Generate yaml, nxdl and hash. + if the extracted hash is exactly the same as producd from generated yaml then + retrieve the nxdl part from provided yaml. + Else, generate nxdl from separated yaml with the help of nyaml2nxdl function + """ + pa_path, rel_file = os.path.split(yaml_file) + sep_yaml = pa_path + f'temp_{rel_file}' + hash_found = separate_hash_yaml_and_nxdl(yaml_file, sep_yaml, out_xml_file) + + if hash_found: + gen_hash = get_sha256_hash(sep_yaml) + if hash_found == gen_hash: + os.remove(sep_yaml) + return + + nyaml2nxdl(sep_yaml, out_xml_file, verbose) + os.remove(sep_yaml) + + # pylint: disable=too-many-locals def append_yml(input_file, append, verbose): """Append to an existing NeXus base class new elements provided in YML input file \ @@ -159,7 +182,7 @@ def launch_tool(input_file, verbose, append, check_consistency): if ext == 'yaml': xml_out_file = raw_name + '.nxdl.xml' - nyaml2nxdl(input_file, xml_out_file, verbose) + generate_nxdl_or_retrieve_nxdl(input_file, xml_out_file, verbose) if append: append_yml(raw_name + '.nxdl.xml', append, @@ -176,12 +199,22 @@ def launch_tool(input_file, verbose, append, check_consistency): yaml_out_file = raw_name + '_parsed' + '.yaml' converter = Nxdl2yaml([], []) converter.print_yml(input_file, yaml_out_file, verbose) + # Append nxdl.xml file with yaml output file + yaml_hash = get_sha256_hash(yaml_out_file) + # Lines as divider between yaml and nxdl + top_lines = [('\n# ++++++++++++++++++++++++++++++++++ SHA HASH' + ' ++++++++++++++++++++++++++++++++++\n'), + f'# {yaml_hash}\n'] + + extend_yamlfile_with_comment(yaml_file=yaml_out_file, + file_to_be_appended=input_file, + top_lines_list=top_lines) else: append_yml(input_file, append, verbose) # Taking care of consistency running if check_consistency: xml_out_file = raw_name + '_consistency.' + ext - nyaml2nxdl(yaml_out_file, xml_out_file, verbose) + generate_nxdl_or_retrieve_nxdl(yaml_out_file, xml_out_file, verbose) os.remove(yaml_out_file) else: raise ValueError("Provide correct file with extension '.yaml or '.nxdl.xml") diff --git a/nyaml2nxdl_backward_tools.py b/nyaml2nxdl_backward_tools.py index 0ebc0d5cc1..ffcaa6f07a 100755 --- a/nyaml2nxdl_backward_tools.py +++ b/nyaml2nxdl_backward_tools.py @@ -99,28 +99,27 @@ def parse(filepath): def handle_mapping_char(text, depth=-1, skip_n_line_on_top=False): """Check for ":" char and replace it by "':'". """ - # This escape chars and sepeerator ':' is not allowed in yaml library escape_char = get_yaml_escape_char_dict() for esc_key, val in escape_char.items(): if esc_key in text: text = text.replace(esc_key, val) if not skip_n_line_on_top: if depth > 0: - text = add_new_line_on_top(text, depth) + text = add_new_line_with_pipe_on_top(text, depth) else: raise ValueError("Need depth size to co-ordinate text line in yaml file.") return text -def add_new_line_on_top(text, depth): +def add_new_line_with_pipe_on_top(text, depth): """ - Return char list for what we get error in converter. After adding a - new line at the start of text the eroor is solved. + Return modified text for what we get error in converter, such as ':'. After adding a + new line at the start of text the error is solved. """ char_list_to_add_new_line_on_top_of_text = [":"] for char in char_list_to_add_new_line_on_top_of_text: if char in text: - return '\n' + depth * DEPTH_SIZE + text + return '|' + '\n' + depth * DEPTH_SIZE + text return text @@ -217,10 +216,10 @@ def handle_symbols(self, depth, node): self.handle_not_root_level_doc(depth, tag=child.attrib['name'], text=symbol_doc.text)) - self.stroe_root_level_comments('symbol_doc_comments', sbl_doc_cmnt_list) - self.stroe_root_level_comments('symbol_comments', symbol_cmnt_list) + self.store_root_level_comments('symbol_doc_comments', sbl_doc_cmnt_list) + self.store_root_level_comments('symbol_comments', symbol_cmnt_list) - def stroe_root_level_comments(self, holder, comment): + def store_root_level_comments(self, holder, comment): """Store yaml text or section line and the comments inteded for that lines or section""" self.root_level_comment[holder] = comment @@ -864,7 +863,7 @@ def xmlparse(self, output_yml, xml_tree, depth, verbose): last_comment = self.comvert_to_ymal_comment(depth * DEPTH_SIZE, child.text) remove_cmnt_n = child if tag_tmp == 'doc': - self.stroe_root_level_comments('root_doc', last_comment) + self.store_root_level_comments('root_doc', last_comment) last_comment = '' self.handle_root_level_doc(child) node.remove(child) @@ -872,7 +871,7 @@ def xmlparse(self, output_yml, xml_tree, depth, verbose): node.remove(remove_cmnt_n) remove_cmnt_n = None if tag_tmp == 'symbols': - self.stroe_root_level_comments('symbols', last_comment) + self.store_root_level_comments('symbols', last_comment) last_comment = '' self.handle_symbols(depth, child) node.remove(child) diff --git a/nyaml2nxdl_forward_tools.py b/nyaml2nxdl_forward_tools.py index cee59d1f3d..9a79096a3f 100644 --- a/nyaml2nxdl_forward_tools.py +++ b/nyaml2nxdl_forward_tools.py @@ -45,7 +45,7 @@ "# This library is free software; you can redistribute it and/or\n" "# modify it under the terms of the GNU Lesser General Public\n" "# License as published by the Free Software Foundation; either\n" - "# version 2 of the License, or (at your option) any later version.\n" + "# version 3 of the License, or (at your option) any later version.\n" "#\n" "# This library is distributed in the hope that it will be useful,\n" "# but WITHOUT ANY WARRANTY; without even the implied warranty of\n" @@ -67,6 +67,41 @@ COMMENT_BLOCKS: CommentCollector +def check_for_dom_comment_in_yaml(): + """Check the yaml file has dom comment or dom comment needed to be hard coded. + """ + dignature_keyword_list = ['NeXus', + 'GNU Lesser General Public', + 'Free Software Foundation', + 'Copyright (C)', + 'WITHOUT ANY WARRANTY'] + + # Check for dom comments in first three comments + dom_comment = '' + dom_comment_ind = 1 + for ind, comnt in enumerate(COMMENT_BLOCKS[0:5]): + cmnt_list = comnt.get_comment_text() + if len(cmnt_list) == 1: + text = cmnt_list[0] + else: + continue + dom_comment = text + dom_comment_ind = ind + for keyword in dignature_keyword_list: + if keyword not in text: + dom_comment = '' + break + if dom_comment: + break + + # deactivate the root dom_comment, So that the corresponding comment would not be + # considered as comment for definition xml element. + if dom_comment: + COMMENT_BLOCKS.remove_comment(dom_comment_ind) + + return dom_comment + + def yml_reader(inputfile): """ This function launches the LineLoader class. @@ -78,6 +113,10 @@ def yml_reader(inputfile): loaded_yaml = loader.get_single_data() COMMENT_BLOCKS = CommentCollector(inputfile, loaded_yaml) COMMENT_BLOCKS.extract_all_comment_blocks() + dom_cmnt_frm_yaml = check_for_dom_comment_in_yaml() + global DOM_COMMENT + if dom_cmnt_frm_yaml: + DOM_COMMENT = dom_cmnt_frm_yaml return loaded_yaml @@ -168,7 +207,7 @@ def check_for_mapping_char_other(text): for key, val in escape_reverter.items(): if key in text: text = text.replace(key, val) - return str(text) + return str(text).strip() def xml_handle_doc(obj, value: str, diff --git a/nyaml2nxdl_helper.py b/nyaml2nxdl_helper.py index 0fb51237d7..867eae6612 100644 --- a/nyaml2nxdl_helper.py +++ b/nyaml2nxdl_helper.py @@ -27,6 +27,7 @@ # Yaml library does not except the keys (escapechar "\t" and yaml separator ":") # So the corresponding value is to skip them and # and also carefull about this order +import hashlib from yaml.composer import Composer from yaml.constructor import Constructor @@ -34,7 +35,12 @@ from yaml.resolver import BaseResolver from yaml.loader import Loader -ESCAPE_CHAR_DICT = {"\t": " "} +# NOTE: If any one change one of the bellow dict please change it for both +ESCAPE_CHAR_DICT_IN_YAML = {"\t": " ", + "\':\'": ":"} + +ESCAPE_CHAR_DICT_IN_XML = {" ": "\t", + "\':\'": ":"} class LineLoader(Loader): # pylint: disable=too-many-ancestors @@ -67,15 +73,13 @@ def construct_mapping(self, node, deep=False): def get_yaml_escape_char_dict(): """Get escape char and the way to skip them in yaml.""" - return ESCAPE_CHAR_DICT + return ESCAPE_CHAR_DICT_IN_YAML def get_yaml_escape_char_reverter_dict(): """To revert yaml escape char in xml constructor from yaml.""" - temp_dict = {} - for key, val in ESCAPE_CHAR_DICT.items(): - temp_dict[val] = key - return temp_dict + + return ESCAPE_CHAR_DICT_IN_XML def type_check(nx_type): @@ -143,3 +147,72 @@ def nx_name_type_resolving(tmp): typ = '' nam = tmp return nam, typ + + +def get_sha256_hash(file_name): + """Generate a sha256_hash for a given file. + """ + sha_hash = hashlib.sha256() + + with open(file=file_name, mode='rb',) as file_obj: + # Update hash for each 4k block of bytes + for b_line in iter(lambda: file_obj.read(4096), b""): + sha_hash.update(b_line) + return sha_hash.hexdigest() + + +def extend_yamlfile_with_comment(yaml_file, + file_to_be_appended, + top_lines_list=None): + """Extend yaml file by the file_to_be_appended as comment. + """ + + with open(yaml_file, mode='a+', encoding='utf-8') as f1_obj: + if top_lines_list: + for line in top_lines_list: + f1_obj.write(line) + + with open(file_to_be_appended, mode='r', encoding='utf-8') as f2_obj: + lines = f2_obj.readlines() + for line in lines: + f1_obj.write(f"# {line}") + + +def separate_hash_yaml_and_nxdl(yaml_file, sep_yaml, sep_xml): + """Separate the provided yaml file into yaml, nxdl and hash if yaml was extended with + nxdl at the end of yaml by + '\n# ++++++++++++++++++++++++++++++++++ SHA HASH \ + ++++++++++++++++++++++++++++++++++\n' + # ' + """ + sha_hash = '' + with open(yaml_file, 'r', encoding='utf-8') as inp_file: + lines = inp_file.readlines() + # file to write yaml part + with open(sep_yaml, 'w', encoding='utf-8') as yml_f_ob, \ + open(sep_xml, 'w', encoding='utf-8') as xml_f_ob: + + last_line = '' + write_on_yaml = True + for ind, line in enumerate(lines): + if ind == 0: + last_line = line + # Write in file when ensured that the nest line is not with '++ SHA HASH ++' + elif '++ SHA HASH ++' not in line and write_on_yaml: + yml_f_ob.write(last_line) + last_line = line + elif '++ SHA HASH ++' in line: + write_on_yaml = False + last_line = '' + elif not write_on_yaml and not last_line: + # The first line of xml file has been found. Onward write lines directly + # into xml file. + if not sha_hash: + sha_hash = line.split('# ', 1)[-1].strip() + else: + xml_f_ob.write(line[2:]) + # If the yaml fiile does not contain any hash for nxdl then we may have last line. + if last_line: + yml_f_ob.write(last_line) + + return sha_hash From 53996daca6de688ac5bc4a86273f4164512b9761 Mon Sep 17 00:00:00 2001 From: RubelMozumder <32923026+RubelMozumder@users.noreply.github.com> Date: Mon, 8 May 2023 13:59:09 +0200 Subject: [PATCH 13/93] Changes for removing old errors related ':' in yaml and xml file. (#96) * Changes for removing old errors related ':' in yaml and xml file. * To store nxdl file into yaml as comment and generate or retrieve nxdl from provided yaml. Wokring perfectly. * Bug: last line of yaml file was not counted * Passing all the tests, pylint, mypy and pydocstyle. * Fixing copy-right doc, new_line_char at the end of 'deprecated key', version from of nexus. * pylint.yml checked. --------- Co-authored-by: RubelMozumder --- test_nyaml2nxdl.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/test_nyaml2nxdl.py b/test_nyaml2nxdl.py index 75072131c1..2e044ed6a2 100755 --- a/test_nyaml2nxdl.py +++ b/test_nyaml2nxdl.py @@ -93,9 +93,14 @@ def test_links(): """ Check the correct parsing of links """ + data_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), + '../data/nyaml2nxdl') ref_xml_link_file = 'tests/data/nyaml2nxdl/Ref_NXtest_links.nxdl.xml' test_yml_link_file = 'tests/data/nyaml2nxdl/NXtest_links.yaml' test_xml_link_file = 'tests/data/nyaml2nxdl/NXtest_links.nxdl.xml' + # ref_xml_link_file = os.path.abspath(data_path + '/Ref_NXtest_links.nxdl.xml') + # test_yml_link_file = os.path.abspath(data_path + '/NXtest_links.yaml') + # test_xml_link_file = os.path.abspath(data_path + '/NXtest_links.nxdl.xml') desired_matches = [''] compare_matches( ref_xml_link_file, @@ -123,10 +128,12 @@ def test_docs(): sys.stdout.write('Test on documentation formatting okay.\n') -def test_nxdl2yaml_doc_format(): +def test_nxdl2yaml_doc_format_and_nxdl_part_as_comment(): """ - In this test an nxdl file with all kind of doc formats are translated + This test for two reason: + 1. In test-1 an nxdl file with all kind of doc formats are translated to yaml to check if they are correct. + 2. In test-2: Check the nxdl that comes at the end of yaml file as comment. """ ref_xml_file = 'tests/data/nyaml2nxdl/Ref_NXentry.nxdl.xml' ref_yml_file = 'tests/data/nyaml2nxdl/Ref_NXentry.yaml' From e0e55090901891047fc4c9aeeee1ded6c53f22c9 Mon Sep 17 00:00:00 2001 From: RubelMozumder <32923026+RubelMozumder@users.noreply.github.com> Date: Tue, 9 May 2023 16:09:52 +0200 Subject: [PATCH 14/93] DEBUG: Deleting intermeadiate generated file. (#109) Co-authored-by: RubelMozumder --- nyaml2nxdl.py | 4 ++-- nyaml2nxdl_helper.py | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/nyaml2nxdl.py b/nyaml2nxdl.py index bfd62cfdad..25a7cae2e7 100755 --- a/nyaml2nxdl.py +++ b/nyaml2nxdl.py @@ -45,7 +45,7 @@ def generate_nxdl_or_retrieve_nxdl(yaml_file, out_xml_file, verbose): Else, generate nxdl from separated yaml with the help of nyaml2nxdl function """ pa_path, rel_file = os.path.split(yaml_file) - sep_yaml = pa_path + f'temp_{rel_file}' + sep_yaml = os.path.join(pa_path, f'temp_{rel_file}') hash_found = separate_hash_yaml_and_nxdl(yaml_file, sep_yaml, out_xml_file) if hash_found: @@ -132,7 +132,7 @@ def split_name_and_extension(file_name): Split file name into extension and rest of the file name. return file raw nam and extension """ - parts = file_name.rsplit('.', 2) + parts = file_name.rsplit('.', 3) if len(parts) == 2: raw = parts[0] ext = parts[1] diff --git a/nyaml2nxdl_helper.py b/nyaml2nxdl_helper.py index 867eae6612..2f12098a17 100644 --- a/nyaml2nxdl_helper.py +++ b/nyaml2nxdl_helper.py @@ -28,6 +28,7 @@ # So the corresponding value is to skip them and # and also carefull about this order import hashlib +import os from yaml.composer import Composer from yaml.constructor import Constructor @@ -214,5 +215,7 @@ def separate_hash_yaml_and_nxdl(yaml_file, sep_yaml, sep_xml): # If the yaml fiile does not contain any hash for nxdl then we may have last line. if last_line: yml_f_ob.write(last_line) + if not sha_hash: + os.remove(sep_xml) return sha_hash From b44c9c391c547e562a73fddc6787fd75d5fcbf33 Mon Sep 17 00:00:00 2001 From: RubelMozumder <32923026+RubelMozumder@users.noreply.github.com> Date: Tue, 9 May 2023 16:09:52 +0200 Subject: [PATCH 15/93] DEBUG: Deleting intermeadiate generated file. (#109) Co-authored-by: RubelMozumder --- test_nyaml2nxdl.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test_nyaml2nxdl.py b/test_nyaml2nxdl.py index 2e044ed6a2..c596ea0d64 100755 --- a/test_nyaml2nxdl.py +++ b/test_nyaml2nxdl.py @@ -154,19 +154,25 @@ def test_fileline_error(): In this test the yaml fileline in the error message is tested. """ test_yml_file = 'tests/data/nyaml2nxdl/NXfilelineError1.yaml' + out_nxdl = 'tests/data/nyaml2nxdl/temp_NXfilelineError1.yaml' result = CliRunner().invoke(nyml2nxdl.launch_tool, ['--input-file', test_yml_file]) assert result.exit_code == 1 assert '13' in str(result.exception) + os.remove(out_nxdl) test_yml_file = 'tests/data/nyaml2nxdl/NXfilelineError2.yaml' + out_nxdl = 'tests/data/nyaml2nxdl/temp_NXfilelineError2.yaml' result = CliRunner().invoke(nyml2nxdl.launch_tool, ['--input-file', test_yml_file]) assert result.exit_code == 1 assert '21' in str(result.exception) + os.remove(out_nxdl) test_yml_file = 'tests/data/nyaml2nxdl/NXfilelineError3.yaml' + out_nxdl = 'tests/data/nyaml2nxdl/temp_NXfilelineError3.yaml' result = CliRunner().invoke(nyml2nxdl.launch_tool, ['--input-file', test_yml_file]) assert result.exit_code == 1 assert '25' in str(result.exception) + os.remove(out_nxdl) sys.stdout.write('Test on xml -> yml fileline error handling okay.\n') From b712dd8115dc28bc29ca3f419972cf2465379421 Mon Sep 17 00:00:00 2001 From: RubelMozumder <32923026+RubelMozumder@users.noreply.github.com> Date: Tue, 13 Jun 2023 16:45:01 +0200 Subject: [PATCH 16/93] Optionality check in nyam2nxdl (#126) * temporary changes. * Including some changes for optionality error. * Passing test successfully. Cleaning up some tests. * Removing autogenerated 'optional' attribute from nxdl.xml. * Removing autogenerated 'optional' attribute from nxdl.xml. * Removing autogenerated 'optional' attribute from nxdl.xml. * Removing autogenerated 'optional' attribute from nxdl.xml. * Comments or corrections are resolved. * Corrections from review. --- nyaml2nxdl.py | 5 +- nyaml2nxdl_backward_tools.py | 16 +++--- nyaml2nxdl_forward_tools.py | 106 ++++++++++++++++++++++++++--------- nyaml2nxdl_helper.py | 11 ++-- 4 files changed, 98 insertions(+), 40 deletions(-) diff --git a/nyaml2nxdl.py b/nyaml2nxdl.py index 25a7cae2e7..160b3f830d 100755 --- a/nyaml2nxdl.py +++ b/nyaml2nxdl.py @@ -34,7 +34,10 @@ compare_niac_and_my) -DEPTH_SIZE = " " +DEPTH_SIZE = 4 * " " + +# NOTE: Some handful links for nyaml2nxdl converter: +# https://manual.nexusformat.org/nxdl_desc.html?highlight=optional def generate_nxdl_or_retrieve_nxdl(yaml_file, out_xml_file, verbose): diff --git a/nyaml2nxdl_backward_tools.py b/nyaml2nxdl_backward_tools.py index ffcaa6f07a..72f5a6c426 100755 --- a/nyaml2nxdl_backward_tools.py +++ b/nyaml2nxdl_backward_tools.py @@ -270,7 +270,7 @@ def handle_not_root_level_doc(self, depth, text, tag='doc', file_out=None): Handle docs field along the yaml file. In this function we also tried to keep the track of intended indentation. E.g. the bollow doc block. * Topic name - DEscription of topic + Description of topic """ # Handling empty doc @@ -280,14 +280,16 @@ def handle_not_root_level_doc(self, depth, text, tag='doc', file_out=None): text = handle_mapping_char(text, -1, True) if "\n" in text: # To remove '\n' character as it will be added before text. - text = text.split('\n') - text = cleaning_empty_lines(text) + text = cleaning_empty_lines(text.split('\n')) text_tmp = [] yaml_indent_n = len((depth + 1) * DEPTH_SIZE) - # Find indentaion in the first valid line with alphabet + # Find indentaion in the first text line with alphabet tmp_i = 0 while tmp_i != -1: first_line_indent_n = 0 + # Taking care of empty text whitout any character + if len(text) == 1 and text[0] == '': + break for ch_ in text[tmp_i]: if ch_ == ' ': first_line_indent_n = first_line_indent_n + 1 @@ -538,8 +540,8 @@ def handle_dimension(self, depth, node, file_out): and attributes of dim has been handled inside this function here. """ # pylint: disable=consider-using-f-string - possible_dim_attrs = ['ref', 'optional', 'recommended', - 'required', 'incr', 'refindex'] + possible_dim_attrs = ['ref', 'required', + 'incr', 'refindex'] possible_dimemsion_attrs = ['rank'] # taking care of Dimension tag @@ -851,7 +853,7 @@ def xmlparse(self, output_yml, xml_tree, depth, verbose): sys.stdout.write(f'Attributes: {node.attrib}\n') with open(output_yml, "a", encoding="utf-8") as file_out: tag = remove_namespace_from_tag(node.tag) - if tag == ('definition'): + if tag == 'definition': self.found_definition = True self.handle_definition(node) # Taking care of root level doc and symbols diff --git a/nyaml2nxdl_forward_tools.py b/nyaml2nxdl_forward_tools.py index 9a79096a3f..db4d4c4644 100644 --- a/nyaml2nxdl_forward_tools.py +++ b/nyaml2nxdl_forward_tools.py @@ -31,6 +31,7 @@ from pynxtools.nexus import nexus from pynxtools.nyaml2nxdl.comment_collector import CommentCollector +from pynxtools.dataconverter.helpers import remove_namespace_from_tag from pynxtools.nyaml2nxdl.nyaml2nxdl_helper import (get_yaml_escape_char_reverter_dict, nx_name_type_resolving, cleaning_empty_lines, LineLoader) @@ -65,6 +66,7 @@ DEPTH_SIZE = " " NX_UNIT_TYPES = nexus.get_nx_units() COMMENT_BLOCKS: CommentCollector +CATEGORY = '' # Definition would be either 'base' or 'application' def check_for_dom_comment_in_yaml(): @@ -117,9 +119,60 @@ def yml_reader(inputfile): global DOM_COMMENT if dom_cmnt_frm_yaml: DOM_COMMENT = dom_cmnt_frm_yaml + + if 'category' not in loaded_yaml.keys(): + raise ValueError("All definitions should be either 'base' or 'application' category. " + "No category has been found.") + global CATEGORY + CATEGORY = loaded_yaml['category'] return loaded_yaml +def check_for_default_attribute_and_value(xml_element): + """NeXus Groups, fields and attributes might have xml default attributes and valuesthat must + come. For example: 'optional' which is 'true' by default for base class and false otherwise. + """ + + # base:Default attributes and value for all elements of base class except dimension element + base_attr_to_val = {'optional': 'true'} + + # application: Default attributes and value for all elements of application class except + # dimension element + application_attr_to_val = {'optional': 'false'} + + # Default attributes and value for dimension element + base_dim_attr_to_val = {'required': 'false'} + application_dim_attr_to_val = {'required': 'true'} + + # Eligible tag for default attr and value + elegible_tag = ['group', 'field', 'attribute'] + + def set_default_attribute(xml_elem, default_attr_to_val): + for deflt_attr, deflt_val in default_attr_to_val.items(): + if deflt_attr not in xml_elem.attrib \ + and 'maxOccurs' not in xml_elem.attrib \ + and 'minOccurs' not in xml_elem.attrib \ + and 'recommended' not in xml_elem.attrib: + xml_elem.set(deflt_attr, deflt_val) + + for child in list(xml_element): + # skiping comment 'function' that mainly collect comment from yaml file. + if not isinstance(child.tag, str): + continue + tag = remove_namespace_from_tag(child.tag) + + if tag == 'dim' and CATEGORY == 'base': + set_default_attribute(child, base_dim_attr_to_val) + if tag == 'dim' and CATEGORY == 'application': + set_default_attribute(child, application_dim_attr_to_val) + if tag in elegible_tag and CATEGORY == 'base': + set_default_attribute(child, base_attr_to_val) + if tag in elegible_tag and CATEGORY == 'application': + + set_default_attribute(child, application_attr_to_val) + check_for_default_attribute_and_value(child) + + def yml_reader_nolinetag(inputfile): """ pyyaml based parsing of yaml file in python dict @@ -132,7 +185,7 @@ def yml_reader_nolinetag(inputfile): def check_for_skiped_attributes(component, value, allowed_attr=None, verbose=False): """ Check for any attributes have been skipped or not. - NOTE: We should we should keep in mind about 'doc' + NOTE: We should keep in mind about 'doc' """ block_tag = ['enumeration'] if value: @@ -154,20 +207,6 @@ def check_for_skiped_attributes(component, value, allowed_attr=None, verbose=Fal f"moment. The allowed attrbutes are {allowed_attr}") -def check_optionality_and_write(obj, opl_key, opl_val): - """ - Taking care of optinality. - """ - if opl_key == 'optional': - if opl_val == 'false': - obj.set('required', 'true') - elif opl_key == 'minOccurs': - if opl_val == '0': - pass - else: - obj.set(opl_key, str(opl_val)) - - def format_nxdl_doc(string): """NeXus format for doc string """ @@ -237,20 +276,19 @@ def xml_handle_exists(dct, obj, keyword, value): """ This function creates an 'exists' element instance, and appends it to an existing element """ - line_number = f'__line__{keyword}' assert value is not None, f'Line {dct[line_number]}: exists argument must not be None !' if isinstance(value, list): - if len(value) == 2 and value[0] == 'min': - obj.set('minOccurs', str(value[1])) - elif len(value) == 2 and value[0] == 'max': - obj.set('maxOccurs', str(value[1])) - elif len(value) == 4 and value[0] == 'min' and value[2] == 'max': + if len(value) == 4 and value[0] == 'min' and value[2] == 'max': obj.set('minOccurs', str(value[1])) if str(value[3]) != 'infty': obj.set('maxOccurs', str(value[3])) else: obj.set('maxOccurs', 'unbounded') + elif len(value) == 2 and value[0] == 'min': + obj.set('minOccurs', str(value[1])) + elif len(value) == 2 and value[0] == 'max': + obj.set('maxOccurs', str(value[1])) elif len(value) == 4 and value[0] == 'max' and value[2] == 'min': obj.set('minOccurs', str(value[3])) if str(value[1]) != 'infty': @@ -268,12 +306,14 @@ def xml_handle_exists(dct, obj, keyword, value): f'entries either [min, ] or [max, ], or a list of four ' f'entries [min, , max, ] !') else: + # This clause take optional in all concept except dimension where 'required' key is allowed + # not the 'optional' key. if value == 'optional': obj.set('optional', 'true') elif value == 'recommended': obj.set('recommended', 'true') elif value == 'required': - obj.set('required', 'true') + obj.set('optional', 'false') else: obj.set('minOccurs', '0') @@ -300,7 +340,6 @@ def xml_handle_group(dct, obj, keyword, value, verbose=False): raise ValueError("A group must have both value and name. Check for group.") grp = ET.SubElement(obj, 'group') - # type come first if l_bracket == 0 and r_bracket > 0: grp.set('type', keyword_type) if keyword_name: @@ -364,7 +403,7 @@ def xml_handle_dimensions(dct, obj, keyword, value: dict): incr:[...]' """ - possible_dimension_attrs = ['rank'] + possible_dimension_attrs = ['rank'] # nxdl attributes line_number = f'__line__{keyword}' line_loc = dct[line_number] assert 'dim' in value.keys(), (f"Line {line_loc}: No dim as child of dimension has " @@ -373,7 +412,7 @@ def xml_handle_dimensions(dct, obj, keyword, value: dict): dims = ET.SubElement(obj, 'dimensions') # Consider all the childs under dimension is dim element and # its attributes -# val_attrs = list(value.keys()) + rm_key_list = [] rank = '' for key, val in value.items(): @@ -418,7 +457,11 @@ def xml_handle_dim_from_dimension_dict(dct, dims_obj, keyword, value, rank, verb function. please also read note in xml_handle_dimensions. """ - possible_dim_attrs = ['ref', 'optional', 'recommended', 'required', 'incr', 'refindex'] + possible_dim_attrs = ['ref', 'incr', 'refindex', 'required'] + + # Some attributes might have equivalent name e.g. 'required' is correct one and + # 'optional' could be another name. Then change attribute to the correct one. + wrong_to_correct_attr = [('optional', 'required')] header_line_number = f"__line__{keyword}" dim_list = [] rm_key_list = [] @@ -431,7 +474,6 @@ def xml_handle_dim_from_dimension_dict(dct, dims_obj, keyword, value, rank, verb for attr, vvalue in value.items(): if '__line__' in attr: continue - line_number = f"__line__{attr}" line_loc = value[line_number] # dim comes in precedence @@ -466,6 +508,11 @@ def xml_handle_dim_from_dimension_dict(dct, dims_obj, keyword, value, rank, verb continue cmnt_number = f'__line__{kkkey}' cmnt_loc = vvalue[cmnt_number] + # Check whether any optional attributes added + for tuple_wng_crt in wrong_to_correct_attr: + if kkkey == tuple_wng_crt[0]: + raise ValueError(f"{cmnt_loc}: Attribute '{kkkey}' is prohibited, use " + f"'{tuple_wng_crt[1]}") if kkkey == 'doc' and dim_list: # doc comes as list of doc for i, dim in enumerate(dim_list): @@ -782,7 +829,6 @@ def xml_handle_fields(obj, keyword, value, line_annot, line_loc, verbose=False): then the not empty keyword_name is a field! This simple function will define a new node of xml tree """ - # List of possible attributes of xml elements allowed_attr = ['name', 'type', 'nameType', 'unit', 'minOccurs', 'long_name', 'axis', 'signal', 'deprecated', 'axes', 'exists', @@ -1106,6 +1152,10 @@ def nyaml2nxdl(input_file: str, out_file, verbose: bool): (lin_annot, line_loc) = post_comment.get_line_info() xml_handle_comment(xml_root, lin_annot, line_loc) + # Note: Just to keep the functionality if we need this functionality later. + default_attr = False + if default_attr: + check_for_default_attribute_and_value(xml_root) pretty_print_xml(xml_root, out_file, def_cmnt_text) if verbose: sys.stdout.write('Parsed YAML to NXDL successfully\n') diff --git a/nyaml2nxdl_helper.py b/nyaml2nxdl_helper.py index 2f12098a17..9583b375d2 100644 --- a/nyaml2nxdl_helper.py +++ b/nyaml2nxdl_helper.py @@ -28,7 +28,6 @@ # So the corresponding value is to skip them and # and also carefull about this order import hashlib -import os from yaml.composer import Composer from yaml.constructor import Constructor @@ -111,7 +110,6 @@ def cleaning_empty_lines(line_list): """ Cleaning up empty lines on top and bottom. """ - if not isinstance(line_list, list): line_list = line_list.split('\n') if '\n' in line_list else [''] @@ -120,11 +118,18 @@ def cleaning_empty_lines(line_list): if line_list[0].strip(): break line_list = line_list[1:] + if len(line_list) == 0: + line_list.append('') + return line_list + # Clining bottom empty lines while True: if line_list[-1].strip(): break line_list = line_list[0:-1] + if len(line_list) == 0: + line_list.append('') + return line_list return line_list @@ -215,7 +220,5 @@ def separate_hash_yaml_and_nxdl(yaml_file, sep_yaml, sep_xml): # If the yaml fiile does not contain any hash for nxdl then we may have last line. if last_line: yml_f_ob.write(last_line) - if not sha_hash: - os.remove(sep_xml) return sha_hash From 9da44eee5a440d24bd72240e967bae0faf9a83d8 Mon Sep 17 00:00:00 2001 From: RubelMozumder <32923026+RubelMozumder@users.noreply.github.com> Date: Tue, 13 Jun 2023 16:45:01 +0200 Subject: [PATCH 17/93] Optionality check in nyam2nxdl (#126) * temporary changes. * Including some changes for optionality error. * Passing test successfully. Cleaning up some tests. * Removing autogenerated 'optional' attribute from nxdl.xml. * Removing autogenerated 'optional' attribute from nxdl.xml. * Removing autogenerated 'optional' attribute from nxdl.xml. * Removing autogenerated 'optional' attribute from nxdl.xml. * Comments or corrections are resolved. * Corrections from review. --- test_nyaml2nxdl.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/test_nyaml2nxdl.py b/test_nyaml2nxdl.py index c596ea0d64..d0c9f875a5 100755 --- a/test_nyaml2nxdl.py +++ b/test_nyaml2nxdl.py @@ -154,25 +154,31 @@ def test_fileline_error(): In this test the yaml fileline in the error message is tested. """ test_yml_file = 'tests/data/nyaml2nxdl/NXfilelineError1.yaml' - out_nxdl = 'tests/data/nyaml2nxdl/temp_NXfilelineError1.yaml' + out_nxdl = 'tests/data/nyaml2nxdl/NXfilelineError1.nxdl.xml' + out_yaml = 'tests/data/nyaml2nxdl/temp_NXfilelineError1.yaml' result = CliRunner().invoke(nyml2nxdl.launch_tool, ['--input-file', test_yml_file]) assert result.exit_code == 1 assert '13' in str(result.exception) os.remove(out_nxdl) + os.remove(out_yaml) test_yml_file = 'tests/data/nyaml2nxdl/NXfilelineError2.yaml' - out_nxdl = 'tests/data/nyaml2nxdl/temp_NXfilelineError2.yaml' + out_nxdl = 'tests/data/nyaml2nxdl/NXfilelineError2.nxdl.xml' + out_yaml = 'tests/data/nyaml2nxdl/temp_NXfilelineError2.yaml' result = CliRunner().invoke(nyml2nxdl.launch_tool, ['--input-file', test_yml_file]) assert result.exit_code == 1 assert '21' in str(result.exception) os.remove(out_nxdl) + os.remove(out_yaml) test_yml_file = 'tests/data/nyaml2nxdl/NXfilelineError3.yaml' - out_nxdl = 'tests/data/nyaml2nxdl/temp_NXfilelineError3.yaml' + out_nxdl = 'tests/data/nyaml2nxdl/NXfilelineError3.nxdl.xml' + out_yaml = 'tests/data/nyaml2nxdl/temp_NXfilelineError3.yaml' result = CliRunner().invoke(nyml2nxdl.launch_tool, ['--input-file', test_yml_file]) assert result.exit_code == 1 assert '25' in str(result.exception) os.remove(out_nxdl) + os.remove(out_yaml) sys.stdout.write('Test on xml -> yml fileline error handling okay.\n') @@ -196,13 +202,14 @@ def test_symbols(): def test_attributes(): """ - Check the correct handling of empty attributes - or attributes fields, e.g. doc + Check expected attributes in NeXus fields, groups, and attributes. + Check proper doc elements. """ ref_xml_attribute_file = 'tests/data/nyaml2nxdl/Ref_NXattributes.nxdl.xml' test_yml_attribute_file = 'tests/data/nyaml2nxdl/NXattributes.yaml' test_xml_attribute_file = 'tests/data/nyaml2nxdl/NXattributes.nxdl.xml' - desired_matches = ['', '', ''] + desired_matches = ['', '', '', + '', ''] compare_matches( ref_xml_attribute_file, test_yml_attribute_file, From 0233d91a87fd8bb20bf680f48674a6bbacf454fd Mon Sep 17 00:00:00 2001 From: Sandor Brockhauser Date: Thu, 15 Jun 2023 10:48:34 +0200 Subject: [PATCH 18/93] adding links to first references of the vocabulary items --- Makefile | 1 + dev_tools/docs/nxdl.py | 47 ++++++++++++++++++++++++++++++++++++++++-- requirements.txt | 3 +++ 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index ae556d7339..44c076c341 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,7 @@ PYTHON = python3 SPHINX = sphinx-build BUILD_DIR = "build" +export NEXUS_DEF_PATH = $(shell pwd) .PHONY: help install style autoformat test clean prepare html pdf impatient-guide all local diff --git a/dev_tools/docs/nxdl.py b/dev_tools/docs/nxdl.py index 8da3ebbc05..0e8d646f22 100644 --- a/dev_tools/docs/nxdl.py +++ b/dev_tools/docs/nxdl.py @@ -7,6 +7,7 @@ from typing import Optional import lxml +from pynxtools.nexus import nexus as pynxtools_nxlib from ..globals.directories import get_nxdl_root from ..globals.errors import NXDLParseError @@ -506,7 +507,7 @@ def _print_attribute(self, ns, kind, node, optional, indent, parent_path): ) self._print(f"{indent}.. index:: {index_name} ({kind} attribute)\n") self._print( - f"{indent}**@{name}**: {optional}{self._format_type(node)}{self._format_units(node)}\n" + f"{indent}**@{name}**: {optional}{self._format_type(node)}{self._format_units(node)} {self.get_first_parent_ref(f'{parent_path}/{name}', 'attribute')}\n" ) self._print_doc(indent + self._INDENTATION_UNIT, ns, node) node_list = node.xpath("nx:enumeration", namespaces=ns) @@ -549,6 +550,7 @@ def _print_full_tree(self, ns, parent, name, indent, parent_path): f"{self._format_type(node)}" f"{dims}" f"{self._format_units(node)}" + f" {self.get_first_parent_ref(f'{parent_path}/{name}', 'field')}" "\n" ) @@ -585,7 +587,9 @@ def _print_full_tree(self, ns, parent, name, indent, parent_path): # target = hTarget.replace(".. _", "").replace(":\n", "") # TODO: https://github.com/nexusformat/definitions/issues/1057 self._print(f"{indent}{hTarget}") - self._print(f"{indent}**{name}**: {optional_text}{typ}\n") + self._print( + f"{indent}**{name}**: {optional_text}{typ} {self.get_first_parent_ref(f'{parent_path}/{name}', 'group')}\n" + ) self._print_if_deprecated(ns, node, indent + self._INDENTATION_UNIT) self._print_doc(indent + self._INDENTATION_UNIT, ns, node) @@ -624,3 +628,42 @@ def _print_full_tree(self, ns, parent, name, indent, parent_path): def _print(self, *args, end="\n"): # TODO: change instances of \t to proper indentation self._rst_lines.append(" ".join(args) + end) + + def get_first_parent_ref(self, path, tag): + nx_name = path[1 : path.find("/", 1)] + path = path[path.find("/", 1) :] + + try: + parents = pynxtools_nxlib.get_inherited_nodes(path, nx_name)[2] + except: + return "" + if len(parents) > 1: + parent = parents[1] + parent_path = parent_display_name = parent.attrib["nxdlpath"] + parent_path_segments = parent_path[1:].split("/") + parent_def_name = parent.attrib["nxdlbase"][ + parent.attrib["nxdlbase"] + .rfind("/") : parent.attrib["nxdlbase"] + .rfind(".nxdl") + ] + + # Case where the first parent is a base_class + if parent_path_segments[0] == "": + return f":ref:`<{parent_def_name[1:]}> <{parent_def_name[1:]}>`" + + parent_display_name = ( + f"{parent_def_name[1:]}/.../{parent_path_segments[-1]}" + if len(parent_path_segments) > 1 + else f"{parent_def_name[1:]}/{parent_path_segments[-1]}" + ) + if tag == "attribute": + pos_of_right_slash = parent_path.rfind("/") + parent_path = ( + parent_path[:pos_of_right_slash] + + "@" + + parent_path[pos_of_right_slash + 1 :] + ) + return ( + f":ref:`<{parent_display_name}> <{parent_def_name}{parent_path}-{tag}>`" + ) + return "" diff --git a/requirements.txt b/requirements.txt index 6d024bda3a..8b22819ff4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,7 @@ pyyaml # Documentation building sphinx>=5 sphinx-tabs +sphinx-comments # Testing pytest @@ -13,3 +14,5 @@ pytest black>=22.3 flake8>=4 isort>=5.10 + +pynxtools>=0.0.3 \ No newline at end of file From a671d15a7da7dba7842de3aa853ea34fd22129d9 Mon Sep 17 00:00:00 2001 From: Sandor Brockhauser Date: Thu, 15 Jun 2023 13:34:56 +0200 Subject: [PATCH 19/93] do not display first reference redundantly if it is the only reference --- dev_tools/docs/nxdl.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/dev_tools/docs/nxdl.py b/dev_tools/docs/nxdl.py index 0e8d646f22..fdff6a8258 100644 --- a/dev_tools/docs/nxdl.py +++ b/dev_tools/docs/nxdl.py @@ -635,7 +635,7 @@ def get_first_parent_ref(self, path, tag): try: parents = pynxtools_nxlib.get_inherited_nodes(path, nx_name)[2] - except: + except FileNotFoundError: return "" if len(parents) > 1: parent = parents[1] @@ -649,7 +649,11 @@ def get_first_parent_ref(self, path, tag): # Case where the first parent is a base_class if parent_path_segments[0] == "": - return f":ref:`<{parent_def_name[1:]}> <{parent_def_name[1:]}>`" + return "" + + #special treatment for NXnote@type + if tag == "attribute" and parent_def_name == "/NXnote" and parent_path == "/type": + return "" parent_display_name = ( f"{parent_def_name[1:]}/.../{parent_path_segments[-1]}" From 124b41eedeb44d530629bc95cfea46f89dd874f5 Mon Sep 17 00:00:00 2001 From: Sandor Brockhauser Date: Thu, 15 Jun 2023 13:41:29 +0200 Subject: [PATCH 20/93] reformatting --- dev_tools/docs/nxdl.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/dev_tools/docs/nxdl.py b/dev_tools/docs/nxdl.py index fdff6a8258..fedfe02785 100644 --- a/dev_tools/docs/nxdl.py +++ b/dev_tools/docs/nxdl.py @@ -650,9 +650,13 @@ def get_first_parent_ref(self, path, tag): # Case where the first parent is a base_class if parent_path_segments[0] == "": return "" - - #special treatment for NXnote@type - if tag == "attribute" and parent_def_name == "/NXnote" and parent_path == "/type": + + # special treatment for NXnote@type + if ( + tag == "attribute" + and parent_def_name == "/NXnote" + and parent_path == "/type" + ): return "" parent_display_name = ( From 8351366eacbb24aa33665ab23604371112aaaa52 Mon Sep 17 00:00:00 2001 From: Sandor Brockhauser Date: Thu, 15 Jun 2023 22:07:01 +0200 Subject: [PATCH 21/93] changing to shorter link with tooltip --- dev_tools/docs/nxdl.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dev_tools/docs/nxdl.py b/dev_tools/docs/nxdl.py index fedfe02785..6c7b092337 100644 --- a/dev_tools/docs/nxdl.py +++ b/dev_tools/docs/nxdl.py @@ -672,6 +672,7 @@ def get_first_parent_ref(self, path, tag): + parent_path[pos_of_right_slash + 1 :] ) return ( - f":ref:`<{parent_display_name}> <{parent_def_name}{parent_path}-{tag}>`" + f"<:abbr:`parent (parent definition: {parent_def_name[1:]}" + + f"/{parent_path_segments[-1]})`:ref:`🔗 <{parent_def_name}{parent_path}-{tag}>`>" ) return "" From 1af3a42452c2506f7e303f8a4b73022d7dcccae7 Mon Sep 17 00:00:00 2001 From: Sandor Brockhauser Date: Thu, 15 Jun 2023 22:12:47 +0200 Subject: [PATCH 22/93] linting --- dev_tools/docs/nxdl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev_tools/docs/nxdl.py b/dev_tools/docs/nxdl.py index 6c7b092337..d76abfbd22 100644 --- a/dev_tools/docs/nxdl.py +++ b/dev_tools/docs/nxdl.py @@ -672,7 +672,7 @@ def get_first_parent_ref(self, path, tag): + parent_path[pos_of_right_slash + 1 :] ) return ( - f"<:abbr:`parent (parent definition: {parent_def_name[1:]}" + - f"/{parent_path_segments[-1]})`:ref:`🔗 <{parent_def_name}{parent_path}-{tag}>`>" + f"<:abbr:`parent (parent definition: {parent_def_name[1:]}" + + f"/{parent_path_segments[-1]})`:ref:`🔗 <{parent_def_name}{parent_path}-{tag}>`>" ) return "" From 8c0881d3311c75a17828d2ba86eed2cea6b399aa Mon Sep 17 00:00:00 2001 From: Sandor Brockhauser Date: Thu, 15 Jun 2023 22:33:18 +0200 Subject: [PATCH 23/93] linting --- dev_tools/docs/nxdl.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/dev_tools/docs/nxdl.py b/dev_tools/docs/nxdl.py index d76abfbd22..086e3aa761 100644 --- a/dev_tools/docs/nxdl.py +++ b/dev_tools/docs/nxdl.py @@ -659,11 +659,6 @@ def get_first_parent_ref(self, path, tag): ): return "" - parent_display_name = ( - f"{parent_def_name[1:]}/.../{parent_path_segments[-1]}" - if len(parent_path_segments) > 1 - else f"{parent_def_name[1:]}/{parent_path_segments[-1]}" - ) if tag == "attribute": pos_of_right_slash = parent_path.rfind("/") parent_path = ( @@ -671,8 +666,9 @@ def get_first_parent_ref(self, path, tag): + "@" + parent_path[pos_of_right_slash + 1 :] ) + parent_display_name = f"{parent_def_name[1:]}{parent_path}" return ( - f"<:abbr:`parent (parent definition: {parent_def_name[1:]}" - + f"/{parent_path_segments[-1]})`:ref:`🔗 <{parent_def_name}{parent_path}-{tag}>`>" + f"<:abbr:`parent (parent definition: {parent_display_name})" + + f"`:ref:`🔗 `>" ) return "" From d5faa6b0c1bbd289c1b6ee654e613535ca909187 Mon Sep 17 00:00:00 2001 From: Sandor Brockhauser Date: Thu, 15 Jun 2023 22:54:42 +0200 Subject: [PATCH 24/93] supporting unicode char for latex --- manual/source/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/manual/source/conf.py b/manual/source/conf.py index 51b35e4bb3..19a52d7078 100644 --- a/manual/source/conf.py +++ b/manual/source/conf.py @@ -95,4 +95,5 @@ latex_elements = { 'maxlistdepth':7, # some application definitions are deeply nested 'preamble': '\\usepackage{amsbsy}\n' + \DeclareUnicodeCharacter{1F517}{X} } From 1138ebfe0088cf56e8505244e5419646db9ee76e Mon Sep 17 00:00:00 2001 From: Sandor Brockhauser Date: Thu, 15 Jun 2023 23:56:12 +0200 Subject: [PATCH 25/93] short tooltip and link --- build/manual/source/conf.py | 100 ++++++++++++++++++++++++++++++++++++ dev_tools/docs/nxdl.py | 4 +- 2 files changed, 102 insertions(+), 2 deletions(-) create mode 100644 build/manual/source/conf.py diff --git a/build/manual/source/conf.py b/build/manual/source/conf.py new file mode 100644 index 0000000000..f6d05e2971 --- /dev/null +++ b/build/manual/source/conf.py @@ -0,0 +1,100 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +import sys, os, datetime + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- Project information ----------------------------------------------------- + +project = 'nexus' +author = 'NIAC, https://www.nexusformat.org' +copyright = u'1996-{}, {}'.format(datetime.datetime.now().year, author) +description = u'NeXus: A Common Data Format for Neutron, X-ray, and Muon Science' + +# The full version, including alpha/beta/rc tags +version = u'unknown NXDL version' +release = u'unknown NXDL release' +nxdl_version = open('../../NXDL_VERSION').read().strip() +if nxdl_version is not None: + version = nxdl_version.split('.')[0] + release = nxdl_version + + +# -- General configuration --------------------------------------------------- + +# https://github.com/nexusformat/definitions/issues/659#issuecomment-577438319 +needs_sphinx = '2.3' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.mathjax', + 'sphinx.ext.ifconfig', + 'sphinx.ext.viewcode', + 'sphinx.ext.githubpages', + 'sphinx.ext.todo', + 'sphinx_tabs.tabs' +] + +# Show `.. todo` directives in the output +# todo_include_todos = True + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = [] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +# html_theme = 'alabaster' +html_theme = 'sphinxdoc' + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Add extra files +html_extra_path = ['CNAME'] + +html_sidebars = { + '**': [ + 'localtoc.html', + 'relations.html', + 'sourcelink.html', + 'searchbox.html', + 'google_search.html' + ], +} + +# Output file base name for HTML help builder. +htmlhelp_basename = 'NeXusManualdoc' + +# -- Options for Latex output ------------------------------------------------- +latex_elements = { + 'maxlistdepth':25, # some application definitions are deeply nested + 'preamble': r''' + \usepackage{amsbsy} + \usepackage[utf8]{inputenc}''' +} diff --git a/dev_tools/docs/nxdl.py b/dev_tools/docs/nxdl.py index 086e3aa761..d0b4cb9c14 100644 --- a/dev_tools/docs/nxdl.py +++ b/dev_tools/docs/nxdl.py @@ -668,7 +668,7 @@ def get_first_parent_ref(self, path, tag): ) parent_display_name = f"{parent_def_name[1:]}{parent_path}" return ( - f"<:abbr:`parent (parent definition: {parent_display_name})" - + f"`:ref:`🔗 `>" + f":abbr:`... (override: {parent_display_name})" + + f"`:ref:`🔗 `" ) return "" From cb92872532a11368c962e0bcfcf9c2ad6b8478b8 Mon Sep 17 00:00:00 2001 From: Sandor Brockhauser Date: Fri, 16 Jun 2023 00:21:04 +0200 Subject: [PATCH 26/93] adjusted conf.py --- build/manual/source/conf.py | 100 ------------------------------------ manual/source/conf.py | 5 +- 2 files changed, 3 insertions(+), 102 deletions(-) delete mode 100644 build/manual/source/conf.py diff --git a/build/manual/source/conf.py b/build/manual/source/conf.py deleted file mode 100644 index f6d05e2971..0000000000 --- a/build/manual/source/conf.py +++ /dev/null @@ -1,100 +0,0 @@ -# Configuration file for the Sphinx documentation builder. -# -# This file only contains a selection of the most common options. For a full -# list see the documentation: -# https://www.sphinx-doc.org/en/master/usage/configuration.html - -# -- Path setup -------------------------------------------------------------- - -import sys, os, datetime - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) - - -# -- Project information ----------------------------------------------------- - -project = 'nexus' -author = 'NIAC, https://www.nexusformat.org' -copyright = u'1996-{}, {}'.format(datetime.datetime.now().year, author) -description = u'NeXus: A Common Data Format for Neutron, X-ray, and Muon Science' - -# The full version, including alpha/beta/rc tags -version = u'unknown NXDL version' -release = u'unknown NXDL release' -nxdl_version = open('../../NXDL_VERSION').read().strip() -if nxdl_version is not None: - version = nxdl_version.split('.')[0] - release = nxdl_version - - -# -- General configuration --------------------------------------------------- - -# https://github.com/nexusformat/definitions/issues/659#issuecomment-577438319 -needs_sphinx = '2.3' - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [ - 'sphinx.ext.mathjax', - 'sphinx.ext.ifconfig', - 'sphinx.ext.viewcode', - 'sphinx.ext.githubpages', - 'sphinx.ext.todo', - 'sphinx_tabs.tabs' -] - -# Show `.. todo` directives in the output -# todo_include_todos = True - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This pattern also affects html_static_path and html_extra_path. -exclude_patterns = [] - - -# -- Options for HTML output ------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -# html_theme = 'alabaster' -html_theme = 'sphinxdoc' - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] - -# Add extra files -html_extra_path = ['CNAME'] - -html_sidebars = { - '**': [ - 'localtoc.html', - 'relations.html', - 'sourcelink.html', - 'searchbox.html', - 'google_search.html' - ], -} - -# Output file base name for HTML help builder. -htmlhelp_basename = 'NeXusManualdoc' - -# -- Options for Latex output ------------------------------------------------- -latex_elements = { - 'maxlistdepth':25, # some application definitions are deeply nested - 'preamble': r''' - \usepackage{amsbsy} - \usepackage[utf8]{inputenc}''' -} diff --git a/manual/source/conf.py b/manual/source/conf.py index 19a52d7078..346e664d95 100644 --- a/manual/source/conf.py +++ b/manual/source/conf.py @@ -94,6 +94,7 @@ # -- Options for Latex output ------------------------------------------------- latex_elements = { 'maxlistdepth':7, # some application definitions are deeply nested - 'preamble': '\\usepackage{amsbsy}\n' - \DeclareUnicodeCharacter{1F517}{X} + 'preamble': r''' + \usepackage{amsbsy} + \DeclareUnicodeCharacter{1F517}{X}''' } From c117842cfc088b9e3babb8f368c9d113e5890642 Mon Sep 17 00:00:00 2001 From: Sandor Brockhauser Date: Fri, 16 Jun 2023 00:42:05 +0200 Subject: [PATCH 27/93] removing pynxtools as dependecy --- dev_tools/docs/nxdl.py | 2 +- dev_tools/utils/nexus.py | 1394 ++++++++++++++++++++++++++++++++++++++ requirements.txt | 3 - 3 files changed, 1395 insertions(+), 4 deletions(-) create mode 100644 dev_tools/utils/nexus.py diff --git a/dev_tools/docs/nxdl.py b/dev_tools/docs/nxdl.py index d0b4cb9c14..1316e230ac 100644 --- a/dev_tools/docs/nxdl.py +++ b/dev_tools/docs/nxdl.py @@ -7,7 +7,7 @@ from typing import Optional import lxml -from pynxtools.nexus import nexus as pynxtools_nxlib +from ..utils import nexus as pynxtools_nxlib from ..globals.directories import get_nxdl_root from ..globals.errors import NXDLParseError diff --git a/dev_tools/utils/nexus.py b/dev_tools/utils/nexus.py new file mode 100644 index 0000000000..ac1d8b36cf --- /dev/null +++ b/dev_tools/utils/nexus.py @@ -0,0 +1,1394 @@ +# pylint: disable=too-many-lines +"""Read files from different format and print it in a standard NeXus format +""" + +import os +import xml.etree.ElementTree as ET +from functools import lru_cache +from glob import glob +import sys +import logging +import textwrap +import h5py +import click + + +class NxdlAttributeError(Exception): + """An exception for throwing an error when an Nxdl attribute is not found.""" + + +def get_app_defs_names(): + """Returns all the AppDef names without their extension: .nxdl.xml""" + app_def_path_glob = f"{get_nexus_definitions_path()}{os.sep}applications{os.sep}*.nxdl*" + contrib_def_path_glob = (f"{get_nexus_definitions_path()}{os.sep}" + f"contributed_definitions{os.sep}*.nxdl*") + files = sorted(glob(app_def_path_glob)) + sorted(glob(contrib_def_path_glob)) + return [os.path.basename(file).split(".")[0] for file in files] + ["NXroot"] + + +@lru_cache(maxsize=None) +def get_xml_root(file_path): + """Reducing I/O time by caching technique""" + + return ET.parse(file_path).getroot() + + +def get_nexus_definitions_path(): + """Check NEXUS_DEF_PATH variable. +If it is empty, this function is filling it""" + try: # either given by sys env + return os.environ['NEXUS_DEF_PATH'] + except KeyError: # or it should be available locally under the dir 'definitions' + local_dir = os.path.abspath(os.path.dirname(__file__)) + return os.path.join(local_dir, f"..{os.sep}definitions") + + +def get_hdf_root(hdf_node): + """Get the root HDF5 node""" + node = hdf_node + while node.name != '/': + node = node.parent + return node + + +def get_hdf_parent(hdf_info): + """Get the parent of an hdf_node in an hdf_info""" + if 'hdf_path' not in hdf_info: + return hdf_info['hdf_node'].parent + node = get_hdf_root(hdf_info['hdf_node']) if 'hdf_root' not in hdf_info \ + else hdf_info['hdf_root'] + for child_name in hdf_info['hdf_path'].split('/'): + node = node[child_name] + return node + + +def get_parent_path(hdf_name): + """Get parent path""" + return '/'.join(hdf_name.split('/')[:-1]) + + +def get_hdf_info_parent(hdf_info): + """Get the hdf_info for the parent of an hdf_node in an hdf_info""" + if 'hdf_path' not in hdf_info: + return {'hdf_node': hdf_info['hdf_node'].parent} + node = get_hdf_root(hdf_info['hdf_node']) if 'hdf_root' not in hdf_info \ + else hdf_info['hdf_root'] + for child_name in hdf_info['hdf_path'].split('/')[1:-1]: + node = node[child_name] + return {'hdf_node': node, 'hdf_path': get_parent_path(hdf_info['hdf_path'])} + + +def get_nx_class_path(hdf_info): + """Get the full path of an HDF5 node using nexus classes +in case of a field, end with the field name""" + hdf_node = hdf_info['hdf_node'] + if hdf_node.name == '/': + return '' + if isinstance(hdf_node, h5py.Group): + return get_nx_class_path(get_hdf_info_parent(hdf_info)) + '/' + \ + (hdf_node.attrs['NX_class'] if 'NX_class' in hdf_node.attrs.keys() else + hdf_node.name.split('/')[-1]) + if isinstance(hdf_node, h5py.Dataset): + return get_nx_class_path( + get_hdf_info_parent(hdf_info)) + '/' + hdf_node.name.split('/')[-1] + return '' + + +def get_nxdl_entry(hdf_info): + """Get the nxdl application definition for an HDF5 node""" + entry = hdf_info + while isinstance(entry['hdf_node'], h5py.Dataset) or \ + 'NX_class' not in entry['hdf_node'].attrs.keys() or \ + entry['hdf_node'].attrs['NX_class'] != 'NXentry': + entry = get_hdf_info_parent(entry) + if entry['hdf_node'].name == '/': + return 'NO NXentry found' + try: + nxdef = entry['hdf_node']['definition'][()] + return nxdef.decode() + except KeyError: # 'NO Definition referenced' + return "NXentry" + + +def get_nx_class(nxdl_elem): + """Get the nexus class for a NXDL node""" + if 'category' in nxdl_elem.attrib.keys(): + return None + try: + return nxdl_elem.attrib['type'] + except KeyError: + return 'NX_CHAR' + + +def get_nx_namefit(hdf_name, name, name_any=False): + """Checks if an HDF5 node name corresponds to a child of the NXDL element +uppercase letters in front can be replaced by arbitraty name, but +uppercase to lowercase match is preferred, +so such match is counted as a measure of the fit""" + if name == hdf_name: + return len(name) * 2 + # count leading capitals + counting = 0 + while counting < len(name) and name[counting].upper() == name[counting]: + counting += 1 + if name_any or counting == len(name) or \ + (counting > 0 and hdf_name.endswith(name[counting:])): # if potential fit + # count the matching chars + fit = 0 + for i in range(min(counting, len(hdf_name))): + if hdf_name[i].upper() == name[i]: + fit += 1 + else: + break + if fit == min(counting, len(hdf_name)): # accept only full fits as better fits + return fit + return 0 + return -1 # no fit + + +def get_nx_classes(): + """Read base classes from the NeXus definition folder. +Check each file in base_classes, applications, contributed_definitions. +If its category attribute is 'base', then it is added to the list. """ + base_classes = sorted(glob(os.path.join(get_nexus_definitions_path(), + 'base_classes', '*.nxdl.xml'))) + applications = sorted(glob(os.path.join(get_nexus_definitions_path(), + 'applications', '*.nxdl.xml'))) + contributed = sorted(glob(os.path.join(get_nexus_definitions_path(), + 'contributed_definitions', '*.nxdl.xml'))) + nx_clss = [] + for nexus_file in base_classes + applications + contributed: + root = get_xml_root(nexus_file) + if root.attrib['category'] == 'base': + nx_clss.append(str(nexus_file[nexus_file.rindex(os.sep) + 1:])[:-9]) + nx_clss = sorted(nx_clss) + return nx_clss + + +def get_nx_units(): + """Read unit kinds from the NeXus definition/nxdlTypes.xsd file""" + filepath = f"{get_nexus_definitions_path()}{os.sep}nxdlTypes.xsd" + root = get_xml_root(filepath) + units_and_type_list = [] + for child in root: + for i in child.attrib.values(): + units_and_type_list.append(i) + flag = False + for line in units_and_type_list: + if line == 'anyUnitsAttr': + flag = True + nx_units = [] + elif 'NX' in line and flag is True: + nx_units.append(line) + elif line == 'primitiveType': + flag = False + else: + pass + return nx_units + + +def get_nx_attribute_type(): + """Read attribute types from the NeXus definition/nxdlTypes.xsd file""" + filepath = get_nexus_definitions_path() + '/nxdlTypes.xsd' + root = get_xml_root(filepath) + units_and_type_list = [] + for child in root: + for i in child.attrib.values(): + units_and_type_list.append(i) + flag = False + for line in units_and_type_list: + if line == 'primitiveType': + flag = True + nx_types = [] + elif 'NX' in line and flag is True: + nx_types.append(line) + elif line == 'anyUnitsAttr': + flag = False + else: + pass + return nx_types + + +def get_node_name(node): + '''Node - xml node. Returns html documentation name. + Either as specified by the 'name' or taken from the type (nx_class). + Note that if only class name is available, the NX prefix is removed and + the string is converted to UPPER case.''' + if 'name' in node.attrib.keys(): + name = node.attrib['name'] + else: + name = node.attrib['type'] + if name.startswith('NX'): + name = name[2:].upper() + return name + + +def belongs_to(nxdl_elem, child, name, class_type=None, hdf_name=None): + """Checks if an HDF5 node name corresponds to a child of the NXDL element +uppercase letters in front can be replaced by arbitraty name, but +uppercase to lowercase match is preferred""" + if class_type and get_nx_class(child) != class_type: + return False + act_htmlname = get_node_name(child) + chk_name = hdf_name or name + if act_htmlname == chk_name: + return True + if not hdf_name: # search for name fits is only allowed for hdf_nodes + return False + try: # check if nameType allows different name + name_any = bool(child.attrib['nameType'] == "any") + except KeyError: + name_any = False + params = [act_htmlname, chk_name, name_any, nxdl_elem, child, name] + return belongs_to_capital(params) + + +def belongs_to_capital(params): + """Checking continues for Upper case""" + (act_htmlname, chk_name, name_any, nxdl_elem, child, name) = params + # or starts with capital and no reserved words used + if (name_any or 'A' <= act_htmlname[0] <= 'Z') and \ + name != 'doc' and name != 'enumeration': + fit = get_nx_namefit(chk_name, act_htmlname, name_any) # check if name fits + if fit < 0: + return False + for child2 in nxdl_elem: + if get_local_name_from_xml(child) != \ + get_local_name_from_xml(child2) or get_node_name(child2) == act_htmlname: + continue + # check if the name of another sibling fits better + name_any2 = "nameType" in child2.attrib.keys() and child2.attrib["nameType"] == "any" + fit2 = get_nx_namefit(chk_name, get_node_name(child2), name_any2) + if fit2 > fit: + return False + # accept this fit + return True + return False + + +def get_local_name_from_xml(element): + """Helper function to extract the element tag without the namespace.""" + return element.tag[element.tag.rindex("}") + 1:] + + +def get_own_nxdl_child_reserved_elements(child, name, nxdl_elem): + """checking reserved elements, like doc, enumeration""" + if get_local_name_from_xml(child) == 'doc' and name == 'doc': + if nxdl_elem.get('nxdlbase'): + child.set('nxdlbase', nxdl_elem.get('nxdlbase')) + child.set('nxdlbase_class', nxdl_elem.get('nxdlbase_class')) + child.set('nxdlpath', nxdl_elem.get('nxdlpath') + '/doc') + return child + if get_local_name_from_xml(child) == 'enumeration' and name == 'enumeration': + if nxdl_elem.get('nxdlbase'): + child.set('nxdlbase', nxdl_elem.get('nxdlbase')) + child.set('nxdlbase_class', nxdl_elem.get('nxdlbase_class')) + child.set('nxdlpath', nxdl_elem.get('nxdlpath') + '/enumeration') + return child + return False + + +def get_own_nxdl_child_base_types(child, class_type, nxdl_elem, name, hdf_name): + """checking base types of group, field,m attribute""" + if get_local_name_from_xml(child) == 'group': + if (class_type is None or (class_type and get_nx_class(child) == class_type)) and \ + belongs_to(nxdl_elem, child, name, class_type, hdf_name): + if nxdl_elem.get('nxdlbase'): + child.set('nxdlbase', nxdl_elem.get('nxdlbase')) + child.set('nxdlbase_class', nxdl_elem.get('nxdlbase_class')) + child.set('nxdlpath', nxdl_elem.get('nxdlpath') + '/' + get_node_name(child)) + return child + if get_local_name_from_xml(child) == 'field' and \ + belongs_to(nxdl_elem, child, name, None, hdf_name): + if nxdl_elem.get('nxdlbase'): + child.set('nxdlbase', nxdl_elem.get('nxdlbase')) + child.set('nxdlbase_class', nxdl_elem.get('nxdlbase_class')) + child.set('nxdlpath', nxdl_elem.get('nxdlpath') + '/' + get_node_name(child)) + return child + if get_local_name_from_xml(child) == 'attribute' and \ + belongs_to(nxdl_elem, child, name, None, hdf_name): + if nxdl_elem.get('nxdlbase'): + child.set('nxdlbase', nxdl_elem.get('nxdlbase')) + child.set('nxdlbase_class', nxdl_elem.get('nxdlbase_class')) + child.set('nxdlpath', nxdl_elem.get('nxdlpath') + '/' + get_node_name(child)) + return child + return False + + +def get_own_nxdl_child(nxdl_elem, name, class_type=None, hdf_name=None, nexus_type=None): + """Checks if an NXDL child node fits to the specific name (either nxdl or hdf) + name - nxdl name + class_type - nxdl type or hdf classname (for groups, it is obligatory) + hdf_name - hdf name""" + for child in nxdl_elem: + if 'name' in child.attrib and child.attrib['name'] == name: + if nxdl_elem.get('nxdlbase'): + child.set('nxdlbase', nxdl_elem.get('nxdlbase')) + child.set('nxdlbase_class', nxdl_elem.get('nxdlbase_class')) + child.set('nxdlpath', nxdl_elem.get('nxdlpath') + '/' + get_node_name(child)) + return child + for child in nxdl_elem: + if "name" in child.attrib and child.attrib["name"] == name: + child.set('nxdlbase', nxdl_elem.get('nxdlbase')) + return child + + for child in nxdl_elem: + result = get_own_nxdl_child_reserved_elements(child, name, nxdl_elem) + if result is not False: + return result + if nexus_type and get_local_name_from_xml(child) != nexus_type: + continue + result = get_own_nxdl_child_base_types(child, class_type, nxdl_elem, name, hdf_name) + if result is not False: + return result + return None + + +def find_definition_file(bc_name): + """find the nxdl file corresponding to the name. + Note that it first checks in contributed and goes beyond only if no contributed found""" + bc_filename = None + for nxdl_folder in ['contributed_definitions', 'base_classes', 'applications']: + if os.path.exists(f"{get_nexus_definitions_path()}{os.sep}" + f"{nxdl_folder}{os.sep}{bc_name}.nxdl.xml"): + bc_filename = f"{get_nexus_definitions_path()}{os.sep}" \ + f"{nxdl_folder}{os.sep}{bc_name}.nxdl.xml" + break + return bc_filename + + +def get_nxdl_child(nxdl_elem, name, class_type=None, hdf_name=None, nexus_type=None, go_base=True): # pylint: disable=too-many-arguments + """Get the NXDL child node corresponding to a specific name +(e.g. of an HDF5 node,or of a documentation) note that if child is not found in application +definition, it also checks for the base classes""" + # search for possible fits for hdf_nodes : skipped + # only exact hits are returned when searching an nxdl child + own_child = get_own_nxdl_child(nxdl_elem, name, class_type, hdf_name, nexus_type) + if own_child is not None: + return own_child + if not go_base: + return None + bc_name = get_nx_class(nxdl_elem) # check in the base class, app def or contributed + if bc_name[2] == '_': # filter primitive types + return None + if bc_name == "group": # Check if it is the root element. Then send to NXroot.nxdl.xml + bc_name = "NXroot" + bc_filename = find_definition_file(bc_name) + if not bc_filename: + raise ValueError('nxdl file not found in definitions folder!') + bc_obj = ET.parse(bc_filename).getroot() + bc_obj.set('nxdlbase', bc_filename) + if 'category' in bc_obj.attrib: + bc_obj.set('nxdlbase_class', bc_obj.attrib['category']) + bc_obj.set('nxdlpath', '') + return get_own_nxdl_child(bc_obj, name, class_type, hdf_name, nexus_type) + + +def get_required_string(nxdl_elem): + """Check for being REQUIRED, RECOMMENDED, OPTIONAL, NOT IN SCHEMA""" + if nxdl_elem is None: + return "<>" + is_optional = 'optional' in nxdl_elem.attrib.keys() \ + and nxdl_elem.attrib['optional'] == "true" + is_minoccurs = 'minOccurs' in nxdl_elem.attrib.keys() \ + and nxdl_elem.attrib['minOccurs'] == "0" + is_recommended = 'recommended' in nxdl_elem.attrib.keys() \ + and nxdl_elem.attrib['recommended'] == "true" + + if is_recommended: + return "<>" + if is_optional or is_minoccurs: + return "<>" + # default optionality: in BASE CLASSES is true; in APPLICATIONS is false + try: + if nxdl_elem.get('nxdlbase_class') == 'base': + return "<>" + except TypeError: + return "<>" + return "<>" + + +def chk_nxdataaxis_v2(hdf_node, name, logger): + """Check if dataset is an axis""" + own_signal = hdf_node.attrs.get('signal') # check for being a Signal + if own_signal is str and own_signal == "1": + logger.debug("Dataset referenced (v2) as NXdata SIGNAL") + own_axes = hdf_node.attrs.get('axes') # check for being an axis + if own_axes is str: + axes = own_axes.split(':') + for i in len(axes): + if axes[i] and name == axes[i]: + logger.debug("Dataset referenced (v2) as NXdata AXIS #%d", i) + return None + ownpaxis = hdf_node.attrs.get('primary') + own_axis = hdf_node.attrs.get('axis') + if own_axis is int: + # also convention v1 + if ownpaxis is int and ownpaxis == 1: + logger.debug("Dataset referenced (v2) as NXdata AXIS #%d", own_axis - 1) + else: + logger.debug( + "Dataset referenced (v2) as NXdata (primary/alternative) AXIS #%d", own_axis - 1) + return None + + +def chk_nxdataaxis(hdf_node, name, logger): + """NEXUS Data Plotting Standard v3: new version from 2014""" + if not isinstance(hdf_node, h5py.Dataset): # check if it is a field in an NXdata node + return None + parent = hdf_node.parent + if not parent or (parent and not parent.attrs.get('NX_class') == "NXdata"): + return None + signal = parent.attrs.get('signal') # chk for Signal + if signal and name == signal: + logger.debug("Dataset referenced as NXdata SIGNAL") + return None + axes = parent.attrs.get('axes') # check for default Axes + if axes is str: + if name == axes: + logger.debug("Dataset referenced as NXdata AXIS") + return None + elif axes is not None: + for i, j in enumerate(axes): + if name == j: + indices = parent.attrs.get(j + '_indices') + if indices is int: + logger.debug(f"Dataset referenced as NXdata AXIS #{indices}") + else: + logger.debug(f"Dataset referenced as NXdata AXIS #{i}") + return None + indices = parent.attrs.get(name + '_indices') # check for alternative Axes + if indices is int: + logger.debug(f"Dataset referenced as NXdata alternative AXIS #{indices}") + return chk_nxdataaxis_v2(hdf_node, name, logger) # check for older conventions + + +# below there are some functions used in get_nxdl_doc function: +def write_doc_string(logger, doc, attr): + """Simple function that prints a line in the logger if doc exists""" + if doc: + logger.debug("@" + attr + ' [NX_CHAR]') + return logger, doc, attr + + +def try_find_units(logger, elem, nxdl_path, doc, attr): + """Try to find if units is defined inside the field in the NXDL element, + otherwise try to find if units is defined as a child of the NXDL element.""" + try: # try to find if units is defined inside the field in the NXDL element + unit = elem.attrib[attr] + if doc: + logger.debug(get_node_concept_path(elem) + "@" + attr + ' [' + unit + ']') + elem = None + nxdl_path.append(attr) + except KeyError: # otherwise try to find if units is defined as a child of the NXDL element + orig_elem = elem + elem = get_nxdl_child(elem, attr, nexus_type='attribute') + if elem is not None: + if doc: + logger.debug(get_node_concept_path(orig_elem) + + "@" + attr + ' - [' + get_nx_class(elem) + ']') + nxdl_path.append(elem) + else: # if no units category were defined in NXDL: + if doc: + logger.debug(get_node_concept_path(orig_elem) + + "@" + attr + " - REQUIRED, but undefined unit category") + nxdl_path.append(attr) + return logger, elem, nxdl_path, doc, attr + + +def check_attr_name_nxdl(param): + """Check for ATTRIBUTENAME_units in NXDL (normal). +If not defined, check for ATTRIBUTENAME to see if the ATTRIBUTE +is in the SCHEMA, but no units category were defined. """ + (logger, elem, nxdl_path, doc, attr, req_str) = param + orig_elem = elem + elem2 = get_nxdl_child(elem, attr, nexus_type='attribute') + if elem2 is not None: # check for ATTRIBUTENAME_units in NXDL (normal) + elem = elem2 + if doc: + logger.debug(get_node_concept_path(orig_elem) + + "@" + attr + ' - [' + get_nx_class(elem) + ']') + nxdl_path.append(elem) + else: + # if not defined, check for ATTRIBUTENAME to see if the ATTRIBUTE + # is in the SCHEMA, but no units category were defined + elem2 = get_nxdl_child(elem, attr[:-6], nexus_type='attribute') + if elem2 is not None: + req_str = '<>' + if doc: + logger.debug(get_node_concept_path(orig_elem) + + "@" + attr + " - RECOMMENDED, but undefined unit category") + nxdl_path.append(attr) + else: # otherwise: NOT IN SCHEMA + elem = elem2 + if doc: + logger.debug(get_node_concept_path(orig_elem) + "@" + attr + " - IS NOT IN SCHEMA") + return logger, elem, nxdl_path, doc, attr, req_str + + +def try_find_default(logger, orig_elem, elem, nxdl_path, doc, attr): # pylint: disable=too-many-arguments + """Try to find if default is defined as a child of the NXDL element """ + if elem is not None: + if doc: + logger.debug(get_node_concept_path(orig_elem) + + "@" + attr + ' - [' + get_nx_class(elem) + ']') + nxdl_path.append(elem) + else: # if no default category were defined in NXDL: + if doc: + logger.debug(get_node_concept_path(orig_elem) + "@" + attr + " - [NX_CHAR]") + nxdl_path.append(attr) + return logger, elem, nxdl_path, doc, attr + + +def other_attrs(logger, orig_elem, elem, nxdl_path, doc, attr): # pylint: disable=too-many-arguments + """Handle remaining attributes """ + if elem is not None: + if doc: + logger.debug(get_node_concept_path(orig_elem) + + "@" + attr + ' - [' + get_nx_class(elem) + ']') + nxdl_path.append(elem) + else: + if doc: + logger.debug(get_node_concept_path(orig_elem) + "@" + attr + " - IS NOT IN SCHEMA") + return logger, elem, nxdl_path, doc, attr + + +def check_deprecation_enum_axis(variables, doc, elist, attr, hdf_node): + """Check for several attributes. - deprecation - enums - nxdataaxis """ + logger, elem, path = variables + dep_str = elem.attrib.get('deprecated') # check for deprecation + if dep_str: + if doc: + logger.debug("DEPRECATED - " + dep_str) + for base_elem in elist if not attr else [elem]: # check for enums + sdoc = get_nxdl_child(base_elem, 'enumeration', go_base=False) + if sdoc is not None: + if doc: + logger.debug("enumeration (" + get_node_concept_path(base_elem) + "):") + for item in sdoc: + if get_local_name_from_xml(item) == 'item': + if doc: + logger.debug("-> " + item.attrib['value']) + chk_nxdataaxis(hdf_node, path.split('/')[-1], logger) # look for NXdata reference (axes/signal) + for base_elem in elist if not attr else [elem]: # check for doc + sdoc = get_nxdl_child(base_elem, 'doc', go_base=False) + if doc: + logger.debug("documentation (" + get_node_concept_path(base_elem) + "):") + logger.debug(sdoc.text if sdoc is not None else "") + return logger, elem, path, doc, elist, attr, hdf_node + + +def get_node_concept_path(elem): + """get the short version of nxdlbase:nxdlpath""" + return str(elem.get('nxdlbase').split('/')[-1] + ":" + elem.get('nxdlpath')) + + +def get_nxdl_attr_doc( # pylint: disable=too-many-arguments,too-many-locals + elem, elist, attr, hdf_node, logger, doc, nxdl_path, req_str, path, hdf_info): + """Get nxdl documentation for an attribute""" + new_elem = [] + old_elem = elem + for elem_index, act_elem1 in enumerate(elist): + act_elem = act_elem1 + # NX_class is a compulsory attribute for groups in a nexus file + # which should match the type of the corresponding NXDL element + if attr == 'NX_class' and not isinstance(hdf_node, h5py.Dataset) and elem_index == 0: + elem = None + logger, doc, attr = write_doc_string(logger, doc, attr) + new_elem = elem + break + # units category is a compulsory attribute for any fields + if attr == 'units' and isinstance(hdf_node, h5py.Dataset): + req_str = "<>" + logger, act_elem, nxdl_path, doc, attr = try_find_units(logger, + act_elem, + nxdl_path, + doc, + attr) + # units for attributes can be given as ATTRIBUTENAME_units + elif attr.endswith('_units'): + logger, act_elem, nxdl_path, doc, attr, req_str = check_attr_name_nxdl((logger, + act_elem, + nxdl_path, + doc, + attr, + req_str)) + # default is allowed for groups + elif attr == 'default' and not isinstance(hdf_node, h5py.Dataset): + req_str = "<>" + # try to find if default is defined as a child of the NXDL element + act_elem = get_nxdl_child(act_elem, attr, nexus_type='attribute', go_base=False) + logger, act_elem, nxdl_path, doc, attr = try_find_default(logger, + act_elem1, + act_elem, + nxdl_path, + doc, + attr) + else: # other attributes + act_elem = get_nxdl_child(act_elem, attr, nexus_type='attribute', go_base=False) + if act_elem is not None: + logger, act_elem, nxdl_path, doc, attr = \ + other_attrs(logger, act_elem1, act_elem, nxdl_path, doc, attr) + if act_elem is not None: + new_elem.append(act_elem) + if req_str is None: + req_str = get_required_string(act_elem) # check for being required + if doc: + logger.debug(req_str) + variables = [logger, act_elem, path] + logger, elem, path, doc, elist, attr, hdf_node = check_deprecation_enum_axis(variables, + doc, + elist, + attr, + hdf_node) + elem = old_elem + if req_str is None and doc: + if attr != 'NX_class': + logger.debug("@" + attr + " - IS NOT IN SCHEMA") + logger.debug("") + return (req_str, get_nxdl_entry(hdf_info), nxdl_path) + + +def get_nxdl_doc(hdf_info, logger, doc, attr=False): + """Get nxdl documentation for an HDF5 node (or its attribute)""" + hdf_node = hdf_info['hdf_node'] + # new way: retrieve multiple inherited base classes + (class_path, nxdl_path, elist) = \ + get_inherited_nodes(None, nx_name=get_nxdl_entry(hdf_info), hdf_node=hdf_node, + hdf_path=hdf_info['hdf_path'] if 'hdf_path' in hdf_info else None, + hdf_root=hdf_info['hdf_root'] if 'hdf_root' in hdf_info else None) + elem = elist[0] if class_path and elist else None + if doc: + logger.debug("classpath: " + str(class_path)) + logger.debug("NOT IN SCHEMA" if elem is None else + "classes:\n" + "\n".join + (get_node_concept_path(e) for e in elist)) + # old solution with a single elem instead of using elist + path = get_nx_class_path(hdf_info) + req_str = None + if elem is None: + if doc: + logger.debug("") + return ('None', None, None) + if attr: + return get_nxdl_attr_doc(elem, elist, attr, hdf_node, logger, doc, nxdl_path, + req_str, path, hdf_info) + req_str = get_required_string(elem) # check for being required + if doc: + logger.debug(req_str) + variables = [logger, elem, path] + logger, elem, path, doc, elist, attr, hdf_node = check_deprecation_enum_axis(variables, + doc, + elist, + attr, + hdf_node) + return (req_str, get_nxdl_entry(hdf_info), nxdl_path) + + +def get_doc(node, ntype, nxhtml, nxpath): + """Get documentation""" + # URL for html documentation + anchor = '' + for n_item in nxpath: + anchor += n_item.lower() + "-" + anchor = ('https://manual.nexusformat.org/classes/', + nxhtml + "#" + anchor.replace('_', '-') + ntype) + if not ntype: + anchor = anchor[:-1] + doc = "" # RST documentation from the field 'doc' + doc_field = node.find("doc") + if doc_field is not None: + doc = doc_field.text + (index, enums) = get_enums(node) # enums + if index: + enum_str = "\n " + ("Possible values:" + if len(enums.split(',')) > 1 + else "Obligatory value:") + "\n " + enums + "\n" + else: + enum_str = "" + return anchor, doc + enum_str + + +def print_doc(node, ntype, level, nxhtml, nxpath): + """Print documentation""" + anchor, doc = get_doc(node, ntype, nxhtml, nxpath) + print(" " * (level + 1) + anchor) + preferred_width = 80 + level * 2 + wrapper = textwrap.TextWrapper(initial_indent=' ' * (level + 1), width=preferred_width, + subsequent_indent=' ' * (level + 1), expand_tabs=False, + tabsize=0) + if doc is not None: + for par in doc.split('\n'): + print(wrapper.fill(par)) + + +def get_namespace(element): + """Extracts the namespace for elements in the NXDL""" + return element.tag[element.tag.index("{"):element.tag.rindex("}") + 1] + + +def get_enums(node): + """Makes list of enumerations, if node contains any. + Returns comma separated STRING of enumeration values, if there are enum tag, + otherwise empty string.""" + # collect item values from enumeration tag, if any + namespace = get_namespace(node) + enums = [] + for enumeration in node.findall(f"{namespace}enumeration"): + for item in enumeration.findall(f"{namespace}item"): + enums.append(item.attrib["value"]) + enums = ','.join(enums) + if enums != "": + return (True, '[' + enums + ']') + return (False, "") # if there is no enumeration tag, returns empty string + + +def add_base_classes(elist, nx_name=None, elem: ET.Element = None): + """Add the base classes corresponding to the last eleme in elist to the list. Note that if +elist is empty, a nxdl file with the name of nx_name or a rather room elem is used if provided""" + if elist and nx_name is None: + nx_name = get_nx_class(elist[-1]) + # to support recursive defintions, like NXsample in NXsample, the following test is removed + # if elist and nx_name and f"{nx_name}.nxdl.xml" in (e.get('nxdlbase') for e in elist): + # return + if elem is None: + if not nx_name: + return + nxdl_file_path = find_definition_file(nx_name) + if nxdl_file_path is None: + nxdl_file_path = f"{nx_name}.nxdl.xml" + elem = ET.parse(nxdl_file_path).getroot() + elem.set('nxdlbase', nxdl_file_path) + else: + elem.set('nxdlbase', '') + if 'category' in elem.attrib: + elem.set('nxdlbase_class', elem.attrib['category']) + elem.set('nxdlpath', '') + elist.append(elem) + # add inherited base class + if 'extends' in elem.attrib and elem.attrib['extends'] != 'NXobject': + add_base_classes(elist, elem.attrib['extends']) + else: + add_base_classes(elist) + + +def set_nxdlpath(child, nxdl_elem): + """ + Setting up child nxdlbase, nxdlpath and nxdlbase_class from nxdl_element. + """ + if nxdl_elem.get('nxdlbase'): + child.set('nxdlbase', nxdl_elem.get('nxdlbase')) + child.set('nxdlbase_class', nxdl_elem.get('nxdlbase_class')) + child.set('nxdlpath', nxdl_elem.get('nxdlpath') + '/' + get_node_name(child)) + return child + + +def get_direct_child(nxdl_elem, html_name): + """ returns the child of nxdl_elem which has a name + corresponding to the the html documentation name html_name""" + for child in nxdl_elem: + if get_local_name_from_xml(child) in ('group', 'field', 'attribute') and \ + html_name == get_node_name(child): + decorated_child = set_nxdlpath(child, nxdl_elem) + return decorated_child + return None + + +def get_field_child(nxdl_elem, html_name): + """ returns the child of nxdl_elem which has a name + corresponding to the html documentation name html_name""" + data_child = None + for child in nxdl_elem: + if get_local_name_from_xml(child) != 'field': + continue + if get_node_name(child) == html_name: + data_child = set_nxdlpath(child, nxdl_elem) + break + return data_child + + +def get_best_nxdata_child(nxdl_elem, hdf_node, hdf_name): + """ returns the child of an NXdata nxdl_elem which has a name + corresponding to the hdf_name""" + nxdata = hdf_node.parent + signals = [] + if 'signal' in nxdata.attrs.keys(): + signals.append(nxdata.attrs.get("signal")) + if "auxiliary_signals" in nxdata.attrs.keys(): + for aux_signal in nxdata.attrs.get("auxiliary_signals"): + signals.append(aux_signal) + data_child = get_field_child(nxdl_elem, 'DATA') + data_error_child = get_field_child(nxdl_elem, 'FIELDNAME_errors') + for signal in signals: + if signal == hdf_name: + return (data_child, 100) + if hdf_name.endswith('_errors') and signal == hdf_name[:-7]: + return (data_error_child, 100) + axes = [] + if "axes" in nxdata.attrs.keys(): + for axis in nxdata.attrs.get("axes"): + axes.append(axis) + axis_child = get_field_child(nxdl_elem, 'AXISNAME') + for axis in axes: + if axis == hdf_name: + return (axis_child, 100) + return (None, 0) + + +def get_best_child(nxdl_elem, hdf_node, hdf_name, hdf_class_name, nexus_type): + """ returns the child of nxdl_elem which has a name + corresponding to the the html documentation name html_name""" + bestfit = -1 + bestchild = None + if 'name' in nxdl_elem.attrib.keys() and nxdl_elem.attrib['name'] == 'NXdata' and \ + hdf_node is not None and hdf_node.parent is not None and \ + hdf_node.parent.attrs.get('NX_class') == 'NXdata': + (fnd_child, fit) = get_best_nxdata_child(nxdl_elem, hdf_node, hdf_name) + if fnd_child is not None: + return (fnd_child, fit) + for child in nxdl_elem: + fit = -2 + if get_local_name_from_xml(child) == nexus_type and \ + (nexus_type != 'group' or get_nx_class(child) == hdf_class_name): + name_any = "nameType" in nxdl_elem.attrib.keys() and \ + nxdl_elem.attrib["nameType"] == "any" + fit = get_nx_namefit(hdf_name, get_node_name(child), name_any) + if fit > bestfit: + bestfit = fit + bestchild = set_nxdlpath(child, nxdl_elem) + return (bestchild, bestfit) + + +def walk_elist(elist, html_name): + """Handle elist from low priority inheritance classes to higher""" + for ind in range(len(elist) - 1, -1, -1): + child = get_direct_child(elist[ind], html_name) + if child is None: + # check for names fitting to a superclas definition + main_child = None + for potential_direct_parent in elist: + main_child = get_direct_child(potential_direct_parent, html_name) + if main_child is not None: + (fitting_child, _) = get_best_child(elist[ind], None, html_name, + get_nx_class(main_child), + get_local_name_from_xml(main_child)) + if fitting_child is not None: + child = fitting_child + break + elist[ind] = child + if elist[ind] is None: + del elist[ind] + continue + # override: remove low priority inheritance classes if class_type is overriden + if len(elist) > ind + 1 and get_nx_class(elist[ind]) != get_nx_class(elist[ind + 1]): + del elist[ind + 1:] + # add new base class(es) if new element brings such (and not a primitive type) + if len(elist) == ind + 1 and get_nx_class(elist[ind])[0:3] != 'NX_': + add_base_classes(elist) + return elist, html_name + + +def helper_get_inherited_nodes(hdf_info2, elist, pind, attr): + """find the best fitting name in all children""" + hdf_path, hdf_node, hdf_class_path = hdf_info2 + hdf_name = hdf_path[pind] + hdf_class_name = hdf_class_path[pind] + if pind < len(hdf_path) - (2 if attr else 1): + act_nexus_type = 'group' + elif pind == len(hdf_path) - 1 and attr: + act_nexus_type = 'attribute' + else: + act_nexus_type = 'field' if isinstance(hdf_node, h5py.Dataset) else 'group' + # find the best fitting name in all children + bestfit = -1 + html_name = None + for ind in range(len(elist) - 1, -1, -1): + newelem, fit = get_best_child(elist[ind], + hdf_node, + hdf_name, + hdf_class_name, + act_nexus_type) + if fit >= bestfit and newelem is not None: + html_name = get_node_name(newelem) + return hdf_path, hdf_node, hdf_class_path, elist, pind, attr, html_name + + +def get_hdf_path(hdf_info): + """Get the hdf_path from an hdf_info""" + if 'hdf_path' in hdf_info: + return hdf_info['hdf_path'].split('/')[1:] + return hdf_info['hdf_node'].name.split('/')[1:] + + +@lru_cache(maxsize=None) +def get_inherited_nodes(nxdl_path: str = None, # pylint: disable=too-many-arguments,too-many-locals + nx_name: str = None, elem: ET.Element = None, + hdf_node=None, hdf_path=None, hdf_root=None, attr=False): + """Returns a list of ET.Element for the given path.""" + # let us start with the given definition file + elist = [] # type: ignore[var-annotated] + add_base_classes(elist, nx_name, elem) + nxdl_elem_path = [elist[0]] + + class_path = [] # type: ignore[var-annotated] + if hdf_node is not None: + hdf_info = {'hdf_node': hdf_node} + if hdf_path: + hdf_info['hdf_path'] = hdf_path + if hdf_root: + hdf_root['hdf_root'] = hdf_root + hdf_node = hdf_info['hdf_node'] + hdf_path = get_hdf_path(hdf_info) + hdf_class_path = get_nx_class_path(hdf_info).split('/')[1:] + if attr: + hdf_path.append(attr) + hdf_class_path.append(attr) + path = hdf_path + else: + html_path = nxdl_path.split('/')[1:] + path = html_path + for pind in range(len(path)): + if hdf_node is not None: + hdf_info2 = [hdf_path, hdf_node, hdf_class_path] + [hdf_path, hdf_node, hdf_class_path, elist, + pind, attr, html_name] = helper_get_inherited_nodes(hdf_info2, elist, + pind, attr) + if html_name is None: # return if NOT IN SCHEMA + return (class_path, nxdl_elem_path, None) + else: + html_name = html_path[pind] + elist, html_name = walk_elist(elist, html_name) + if elist: + class_path.append(get_nx_class(elist[0])) + nxdl_elem_path.append(elist[0]) + return (class_path, nxdl_elem_path, elist) + + +def get_node_at_nxdl_path(nxdl_path: str = None, + nx_name: str = None, elem: ET.Element = None, + exc: bool = True): + """Returns an ET.Element for the given path. + This function either takes the name for the NeXus Application Definition + we are looking for or the root elem from a previously loaded NXDL file + and finds the corresponding XML element with the needed attributes.""" + try: + (class_path, nxdlpath, elist) = get_inherited_nodes(nxdl_path, nx_name, elem) + except ValueError as value_error: + if exc: + raise NxdlAttributeError(f"Attributes were not found for {nxdl_path}. " + "Please check this entry in the template dictionary.") \ + from value_error + return None + if class_path and nxdlpath and elist: + elem = elist[0] + else: + elem = None + if exc: + raise NxdlAttributeError(f"Attributes were not found for {nxdl_path}. " + "Please check this entry in the template dictionary.") + return elem + + +def process_node(hdf_node, hdf_path, parser, logger, doc=True): + """Processes an hdf5 node. +- it logs the node found and also checks for its attributes +- retrieves the corresponding nxdl documentation +TODO: +- follow variants +- NOMAD parser: store in NOMAD """ + hdf_info = {'hdf_path': hdf_path, 'hdf_node': hdf_node} + if isinstance(hdf_node, h5py.Dataset): + logger.debug(f'===== FIELD (/{hdf_path}): {hdf_node}') + val = str(hdf_node[()]).split('\n') if len(hdf_node.shape) <= 1 else str( + hdf_node[0]).split('\n') + logger.debug(f'value: {val[0]} {"..." if len(val) > 1 else ""}') + else: + logger.debug( + f"===== GROUP (/{hdf_path} " + f"[{get_nxdl_entry(hdf_info)}" + f"::{get_nx_class_path(hdf_info)}]): {hdf_node}" + ) + (req_str, nxdef, nxdl_path) = get_nxdl_doc(hdf_info, logger, doc) + if parser is not None and isinstance(hdf_node, h5py.Dataset): + parser({"hdf_info": hdf_info, + "nxdef": nxdef, + "nxdl_path": nxdl_path, + "val": val, + "logger": logger}) + for key, value in hdf_node.attrs.items(): + logger.debug(f'===== ATTRS (/{hdf_path}@{key})') + val = str(value).split('\n') + logger.debug(f'value: {val[0]} {"..." if len(val) > 1 else ""}') + (req_str, nxdef, nxdl_path) = \ + get_nxdl_doc(hdf_info, logger, doc, attr=key) + if ( + parser is not None + and req_str is not None + and 'NOT IN SCHEMA' not in req_str + and 'None' not in req_str + ): + parser({"hdf_info": hdf_info, + "nxdef": nxdef, + "nxdl_path": nxdl_path, + "val": val, + "logger": logger}, attr=key) + + +def logger_auxiliary_signal(logger, nxdata): + """Handle the presence of auxiliary signal""" + aux = nxdata.attrs.get('auxiliary_signals') + if aux is not None: + if isinstance(aux, str): + aux = [aux] + for asig in aux: + logger.debug(f'Further auxiliary signal has been identified: {asig}') + return logger + + +def print_default_plotable_header(logger): + """Print a three-lines header""" + logger.debug('========================') + logger.debug('=== Default Plotable ===') + logger.debug('========================') + + +def get_default_plotable(root, logger): + """Get default plotable""" + print_default_plotable_header(logger) + # v3 from 2014 + # nxentry + nxentry = None + default_nxentry_group_name = root.attrs.get("default") + if default_nxentry_group_name: + try: + nxentry = root[default_nxentry_group_name] + except KeyError: + nxentry = None + if not nxentry: + nxentry = entry_helper(root) + if not nxentry: + logger.debug('No NXentry has been found') + return + logger.debug('') + logger.debug('NXentry has been identified: ' + nxentry.name) + # nxdata + nxdata = None + nxgroup = nxentry + default_group_name = nxgroup.attrs.get("default") + while default_group_name: + try: + nxgroup = nxgroup[default_group_name] + default_group_name = nxgroup.attrs.get("default") + except KeyError: + pass + if nxgroup == nxentry: + nxdata = nxdata_helper(nxentry) + else: + nxdata = nxgroup + if not nxdata: + logger.debug('No NXdata group has been found') + return + logger.debug('') + logger.debug('NXdata group has been identified: ' + nxdata.name) + process_node(nxdata, nxdata.name, None, logger, False) + # signal + signal = None + signal_dataset_name = nxdata.attrs.get("signal") + try: + signal = nxdata[signal_dataset_name] + except (TypeError, KeyError): + signal = None + if not signal: + signal = signal_helper(nxdata) + if not signal: + logger.debug('No Signal has been found') + return + logger.debug('') + logger.debug('Signal has been identified: ' + signal.name) + process_node(signal, signal.name, None, logger, False) + logger = logger_auxiliary_signal(logger, nxdata) # check auxiliary_signals + dim = len(signal.shape) + axes = [] # axes + axis_helper(dim, nxdata, signal, axes, logger) + + +def entry_helper(root): + """Check entry related data""" + nxentries = [] + for key in root.keys(): + if isinstance(root[key], h5py.Group) and root[key].attrs.get('NX_class') and \ + root[key].attrs['NX_class'] == "NXentry": + nxentries.append(root[key]) + if len(nxentries) >= 1: + return nxentries[0] + return None + + +def nxdata_helper(nxentry): + """Check if nxentry hdf5 object has a NX_class and, if it contains NXdata, +return its value""" + lnxdata = [] + for key in nxentry.keys(): + if isinstance(nxentry[key], h5py.Group) and nxentry[key].attrs.get('NX_class') and \ + nxentry[key].attrs['NX_class'] == "NXdata": + lnxdata.append(nxentry[key]) + if len(lnxdata) >= 1: + return lnxdata[0] + return None + + +def signal_helper(nxdata): + """Check signal related data""" + signals = [] + for key in nxdata.keys(): + if isinstance(nxdata[key], h5py.Dataset): + signals.append(nxdata[key]) + if len(signals) == 1: # v3: as there was no selection given, only 1 data field shall exists + return signals[0] + if len(signals) > 1: # v2: select the one with an attribute signal="1" attribute + for sig in signals: + if sig.attrs.get("signal") and sig.attrs.get("signal") is str and \ + sig.attrs.get("signal") == "1": + return sig + return None + + +def find_attrib_axis_actual_dim_num(nxdata, a_item, ax_list): + """Finds axis that have defined dimensions""" + # find those with attribute axis= actual dimension number + lax = [] + for key in nxdata.keys(): + if isinstance(nxdata[key], h5py.Dataset): + try: + if nxdata[key].attrs['axis'] == a_item + 1: + lax.append(nxdata[key]) + except KeyError: + pass + if len(lax) == 1: + ax_list.append(lax[0]) + # if there are more alternatives, prioritise the one with an attribute primary="1" + elif len(lax) > 1: + for sax in lax: + if sax.attrs.get('primary') and sax.attrs.get('primary') == 1: + ax_list.insert(0, sax) + else: + ax_list.append(sax) + + +def get_single_or_multiple_axes(nxdata, ax_datasets, a_item, ax_list): + """Gets either single or multiple axes from the NXDL""" + try: + if isinstance(ax_datasets, str): # single axis is defined + # explicite definition of dimension number + ind = nxdata.attrs.get(ax_datasets + '_indices') + if ind and ind is int: + if ind == a_item: + ax_list.append(nxdata[ax_datasets]) + elif a_item == 0: # positional determination of the dimension number + ax_list.append(nxdata[ax_datasets]) + else: # multiple axes are listed + # explicite definition of dimension number + for aax in ax_datasets: + ind = nxdata.attrs.get(aax + '_indices') + if ind and isinstance(ind, int): + if ind == a_item: + ax_list.append(nxdata[aax]) + if not ax_list: # positional determination of the dimension number + ax_list.append(nxdata[ax_datasets[a_item]]) + except KeyError: + pass + return ax_list + + +def axis_helper(dim, nxdata, signal, axes, logger): + """Check axis related data""" + for a_item in range(dim): + ax_list = [] + ax_datasets = nxdata.attrs.get("axes") # primary axes listed in attribute axes + ax_list = get_single_or_multiple_axes(nxdata, ax_datasets, a_item, ax_list) + for attr in nxdata.attrs.keys(): # check for corresponding AXISNAME_indices + if attr.endswith('_indices') and nxdata.attrs[attr] == a_item and \ + nxdata[attr.split('_indices')[0]] not in ax_list: + ax_list.append(nxdata[attr.split('_indices')[0]]) + # v2 # check for ':' separated axes defined in Signal + if not ax_list: + try: + ax_datasets = signal.attrs.get("axes").split(':') + ax_list.append(nxdata[ax_datasets[a_item]]) + except (KeyError, AttributeError): + pass + if not ax_list: # check for axis/primary specifications + find_attrib_axis_actual_dim_num(nxdata, a_item, ax_list) + axes.append(ax_list) + logger.debug('') + logger.debug( + f'For Axis #{a_item}, {len(ax_list)} axes have been identified: {str(ax_list)}' + ) + + +def get_all_is_a_rel_from_hdf_node(hdf_node, hdf_path): + """Return list of nxdl concept paths for a nxdl element which corresponds to + hdf node. + """ + hdf_info = {'hdf_path': hdf_path, 'hdf_node': hdf_node} + (_, _, elist) = \ + get_inherited_nodes(None, nx_name=get_nxdl_entry(hdf_info), hdf_node=hdf_node, + hdf_path=hdf_info['hdf_path'] if 'hdf_path' in hdf_info else None, + hdf_root=hdf_info['hdf_root'] if 'hdf_root' in hdf_info else None) + return elist + + +def hdf_node_to_self_concept_path(hdf_info, logger): + """ Get concept or nxdl path from given hdf_node. + """ + # The bellow logger is for deactivatine unnecessary debug message above + if logger is None: + logger = logging.getLogger(__name__) + logger.setLevel(logging.INFO) + (_, _, nxdl_path) = get_nxdl_doc(hdf_info, logger, None) + con_path = '' + if nxdl_path: + for nd_ in nxdl_path: + con_path = con_path + '/' + get_node_name(nd_) + return con_path + + +class HandleNexus: + """documentation""" + def __init__(self, logger, nexus_file, + d_inq_nd=None, c_inq_nd=None): + self.logger = logger + local_dir = os.path.abspath(os.path.dirname(__file__)) + + self.input_file_name = nexus_file if nexus_file is not None else \ + os.path.join(local_dir, '../../tests/data/nexus/201805_WSe2_arpes.nxs') + self.parser = None + self.in_file = None + self.d_inq_nd = d_inq_nd + self.c_inq_nd = c_inq_nd + # Aggregating hdf path corresponds to concept query node + self.hdf_path_list_for_c_inq_nd = [] + + def visit_node(self, hdf_name, hdf_node): + """Function called by h5py that iterates on each node of hdf5file. + It allows h5py visititems function to visit nodes.""" + if self.d_inq_nd is None and self.c_inq_nd is None: + process_node(hdf_node, '/' + hdf_name, self.parser, self.logger) + elif (self.d_inq_nd is not None + and hdf_name in (self.d_inq_nd, self.d_inq_nd[1:])): + process_node(hdf_node, '/' + hdf_name, self.parser, self.logger) + elif self.c_inq_nd is not None: + attributed_concept = self.c_inq_nd.split('@') + attr = attributed_concept[1] if len(attributed_concept) > 1 else None + elist = get_all_is_a_rel_from_hdf_node(hdf_node, '/' + hdf_name) + if elist is None: + return + fnd_superclass = False + fnd_superclass_attr = False + for elem in reversed(elist): + tmp_path = elem.get('nxdlbase').split('.nxdl')[0] + con_path = '/NX' + tmp_path.split('NX')[-1] + elem.get('nxdlpath') + if fnd_superclass or con_path == attributed_concept[0]: + fnd_superclass = True + if attr is None: + self.hdf_path_list_for_c_inq_nd.append(hdf_name) + break + for attribute in hdf_node.attrs.keys(): + attr_concept = get_nxdl_child(elem, attribute, nexus_type='attribute', + go_base=False) + if attr_concept is not None and \ + attr_concept.get('nxdlpath').endswith(attr): + fnd_superclass_attr = True + con_path = '/NX' + tmp_path.split('NX')[-1] \ + + attr_concept.get('nxdlpath') + self.hdf_path_list_for_c_inq_nd.append(hdf_name + "@" + attribute) + break + if fnd_superclass_attr: + break + + def not_yet_visited(self, root, name): + """checking if a new node has already been visited in its path""" + path = name.split('/') + for i in range(1, len(path)): + act_path = '/'.join(path[:i]) + # print(act_path+' - '+name) + if root['/' + act_path] == root['/' + name]: + return False + return True + + def full_visit(self, root, hdf_node, name, func): + """visiting recursivly all children, but avoiding endless cycles""" + # print(name) + if len(name) > 0: + func(name, hdf_node) + if isinstance(hdf_node, h5py.Group): + for ch_name, child in hdf_node.items(): + full_name = ch_name if len(name) == 0 else name + '/' + ch_name + if self.not_yet_visited(root, full_name): + self.full_visit(root, child, full_name, func) + + def process_nexus_master_file(self, parser): + """Process a nexus master file by processing all its nodes and their attributes""" + self.parser = parser + self.in_file = h5py.File( + self.input_file_name[0] + if isinstance(self.input_file_name, list) + else self.input_file_name, 'r' + ) + self.full_visit(self.in_file, self.in_file, '', self.visit_node) + if self.d_inq_nd is None and self.c_inq_nd is None: + get_default_plotable(self.in_file, self.logger) + # To log the provided concept and concepts founded + if self.c_inq_nd is not None: + for hdf_path in self.hdf_path_list_for_c_inq_nd: + self.logger.info(hdf_path) + self.in_file.close() + + +@click.command() +@click.option( + '-f', + '--nexus-file', + required=False, + default=None, + help=('NeXus file with extension .nxs to learn NeXus different concept' + ' documentation and concept.') +) +@click.option( + '-d', + '--documentation', + required=False, + default=None, + help=("Definition path in nexus output (.nxs) file. Returns debug" + "log relavent with that definition path. Example: /entry/data/delays") +) +@click.option( + '-c', + '--concept', + required=False, + default=None, + help=("Concept path from application definition file (.nxdl,xml). Finds out" + "all the available concept definition (IS-A realation) for rendered" + "concept path. Example: /NXarpes/ENTRY/INSTRUMENT/analyser") +) +def main(nexus_file, documentation, concept): + """The main function to call when used as a script.""" + logging_format = "%(levelname)s: %(message)s" + stdout_handler = logging.StreamHandler(sys.stdout) + stdout_handler.setLevel(logging.DEBUG) + logging.basicConfig(level=logging.INFO, format=logging_format, handlers=[stdout_handler]) + logger = logging.getLogger(__name__) + logger.addHandler(stdout_handler) + logger.setLevel(logging.DEBUG) + logger.propagate = False + if documentation and concept: + raise ValueError("Only one option either documentation (-d) or is_a relation " + "with a concept (-c) can be requested.") + nexus_helper = HandleNexus(logger, nexus_file, + d_inq_nd=documentation, + c_inq_nd=concept) + nexus_helper.process_nexus_master_file(None) + + +if __name__ == '__main__': + main() # pylint: disable=no-value-for-parameter diff --git a/requirements.txt b/requirements.txt index 8b22819ff4..6d024bda3a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,6 @@ pyyaml # Documentation building sphinx>=5 sphinx-tabs -sphinx-comments # Testing pytest @@ -14,5 +13,3 @@ pytest black>=22.3 flake8>=4 isort>=5.10 - -pynxtools>=0.0.3 \ No newline at end of file From 185f6497ab02c7fc1a819568fa0cea00de1334c6 Mon Sep 17 00:00:00 2001 From: Sandor Brockhauser Date: Fri, 16 Jun 2023 00:45:17 +0200 Subject: [PATCH 28/93] linting --- dev_tools/utils/nexus.py | 1069 +++++++++++++++++++++++--------------- 1 file changed, 645 insertions(+), 424 deletions(-) diff --git a/dev_tools/utils/nexus.py b/dev_tools/utils/nexus.py index ac1d8b36cf..7b09e30f33 100644 --- a/dev_tools/utils/nexus.py +++ b/dev_tools/utils/nexus.py @@ -19,9 +19,13 @@ class NxdlAttributeError(Exception): def get_app_defs_names(): """Returns all the AppDef names without their extension: .nxdl.xml""" - app_def_path_glob = f"{get_nexus_definitions_path()}{os.sep}applications{os.sep}*.nxdl*" - contrib_def_path_glob = (f"{get_nexus_definitions_path()}{os.sep}" - f"contributed_definitions{os.sep}*.nxdl*") + app_def_path_glob = ( + f"{get_nexus_definitions_path()}{os.sep}applications{os.sep}*.nxdl*" + ) + contrib_def_path_glob = ( + f"{get_nexus_definitions_path()}{os.sep}" + f"contributed_definitions{os.sep}*.nxdl*" + ) files = sorted(glob(app_def_path_glob)) + sorted(glob(contrib_def_path_glob)) return [os.path.basename(file).split(".")[0] for file in files] + ["NXroot"] @@ -35,9 +39,9 @@ def get_xml_root(file_path): def get_nexus_definitions_path(): """Check NEXUS_DEF_PATH variable. -If it is empty, this function is filling it""" + If it is empty, this function is filling it""" try: # either given by sys env - return os.environ['NEXUS_DEF_PATH'] + return os.environ["NEXUS_DEF_PATH"] except KeyError: # or it should be available locally under the dir 'definitions' local_dir = os.path.abspath(os.path.dirname(__file__)) return os.path.join(local_dir, f"..{os.sep}definitions") @@ -46,65 +50,82 @@ def get_nexus_definitions_path(): def get_hdf_root(hdf_node): """Get the root HDF5 node""" node = hdf_node - while node.name != '/': + while node.name != "/": node = node.parent return node def get_hdf_parent(hdf_info): """Get the parent of an hdf_node in an hdf_info""" - if 'hdf_path' not in hdf_info: - return hdf_info['hdf_node'].parent - node = get_hdf_root(hdf_info['hdf_node']) if 'hdf_root' not in hdf_info \ - else hdf_info['hdf_root'] - for child_name in hdf_info['hdf_path'].split('/'): + if "hdf_path" not in hdf_info: + return hdf_info["hdf_node"].parent + node = ( + get_hdf_root(hdf_info["hdf_node"]) + if "hdf_root" not in hdf_info + else hdf_info["hdf_root"] + ) + for child_name in hdf_info["hdf_path"].split("/"): node = node[child_name] return node def get_parent_path(hdf_name): """Get parent path""" - return '/'.join(hdf_name.split('/')[:-1]) + return "/".join(hdf_name.split("/")[:-1]) def get_hdf_info_parent(hdf_info): """Get the hdf_info for the parent of an hdf_node in an hdf_info""" - if 'hdf_path' not in hdf_info: - return {'hdf_node': hdf_info['hdf_node'].parent} - node = get_hdf_root(hdf_info['hdf_node']) if 'hdf_root' not in hdf_info \ - else hdf_info['hdf_root'] - for child_name in hdf_info['hdf_path'].split('/')[1:-1]: + if "hdf_path" not in hdf_info: + return {"hdf_node": hdf_info["hdf_node"].parent} + node = ( + get_hdf_root(hdf_info["hdf_node"]) + if "hdf_root" not in hdf_info + else hdf_info["hdf_root"] + ) + for child_name in hdf_info["hdf_path"].split("/")[1:-1]: node = node[child_name] - return {'hdf_node': node, 'hdf_path': get_parent_path(hdf_info['hdf_path'])} + return {"hdf_node": node, "hdf_path": get_parent_path(hdf_info["hdf_path"])} def get_nx_class_path(hdf_info): """Get the full path of an HDF5 node using nexus classes -in case of a field, end with the field name""" - hdf_node = hdf_info['hdf_node'] - if hdf_node.name == '/': - return '' + in case of a field, end with the field name""" + hdf_node = hdf_info["hdf_node"] + if hdf_node.name == "/": + return "" if isinstance(hdf_node, h5py.Group): - return get_nx_class_path(get_hdf_info_parent(hdf_info)) + '/' + \ - (hdf_node.attrs['NX_class'] if 'NX_class' in hdf_node.attrs.keys() else - hdf_node.name.split('/')[-1]) + return ( + get_nx_class_path(get_hdf_info_parent(hdf_info)) + + "/" + + ( + hdf_node.attrs["NX_class"] + if "NX_class" in hdf_node.attrs.keys() + else hdf_node.name.split("/")[-1] + ) + ) if isinstance(hdf_node, h5py.Dataset): - return get_nx_class_path( - get_hdf_info_parent(hdf_info)) + '/' + hdf_node.name.split('/')[-1] - return '' + return ( + get_nx_class_path(get_hdf_info_parent(hdf_info)) + + "/" + + hdf_node.name.split("/")[-1] + ) + return "" def get_nxdl_entry(hdf_info): """Get the nxdl application definition for an HDF5 node""" entry = hdf_info - while isinstance(entry['hdf_node'], h5py.Dataset) or \ - 'NX_class' not in entry['hdf_node'].attrs.keys() or \ - entry['hdf_node'].attrs['NX_class'] != 'NXentry': + while ( + isinstance(entry["hdf_node"], h5py.Dataset) + or "NX_class" not in entry["hdf_node"].attrs.keys() + or entry["hdf_node"].attrs["NX_class"] != "NXentry" + ): entry = get_hdf_info_parent(entry) - if entry['hdf_node'].name == '/': - return 'NO NXentry found' + if entry["hdf_node"].name == "/": + return "NO NXentry found" try: - nxdef = entry['hdf_node']['definition'][()] + nxdef = entry["hdf_node"]["definition"][()] return nxdef.decode() except KeyError: # 'NO Definition referenced' return "NXentry" @@ -112,27 +133,30 @@ def get_nxdl_entry(hdf_info): def get_nx_class(nxdl_elem): """Get the nexus class for a NXDL node""" - if 'category' in nxdl_elem.attrib.keys(): + if "category" in nxdl_elem.attrib.keys(): return None try: - return nxdl_elem.attrib['type'] + return nxdl_elem.attrib["type"] except KeyError: - return 'NX_CHAR' + return "NX_CHAR" def get_nx_namefit(hdf_name, name, name_any=False): """Checks if an HDF5 node name corresponds to a child of the NXDL element -uppercase letters in front can be replaced by arbitraty name, but -uppercase to lowercase match is preferred, -so such match is counted as a measure of the fit""" + uppercase letters in front can be replaced by arbitraty name, but + uppercase to lowercase match is preferred, + so such match is counted as a measure of the fit""" if name == hdf_name: return len(name) * 2 # count leading capitals counting = 0 while counting < len(name) and name[counting].upper() == name[counting]: counting += 1 - if name_any or counting == len(name) or \ - (counting > 0 and hdf_name.endswith(name[counting:])): # if potential fit + if ( + name_any + or counting == len(name) + or (counting > 0 and hdf_name.endswith(name[counting:])) + ): # if potential fit # count the matching chars fit = 0 for i in range(min(counting, len(hdf_name))): @@ -143,24 +167,31 @@ def get_nx_namefit(hdf_name, name, name_any=False): if fit == min(counting, len(hdf_name)): # accept only full fits as better fits return fit return 0 - return -1 # no fit + return -1 # no fit def get_nx_classes(): """Read base classes from the NeXus definition folder. -Check each file in base_classes, applications, contributed_definitions. -If its category attribute is 'base', then it is added to the list. """ - base_classes = sorted(glob(os.path.join(get_nexus_definitions_path(), - 'base_classes', '*.nxdl.xml'))) - applications = sorted(glob(os.path.join(get_nexus_definitions_path(), - 'applications', '*.nxdl.xml'))) - contributed = sorted(glob(os.path.join(get_nexus_definitions_path(), - 'contributed_definitions', '*.nxdl.xml'))) + Check each file in base_classes, applications, contributed_definitions. + If its category attribute is 'base', then it is added to the list.""" + base_classes = sorted( + glob(os.path.join(get_nexus_definitions_path(), "base_classes", "*.nxdl.xml")) + ) + applications = sorted( + glob(os.path.join(get_nexus_definitions_path(), "applications", "*.nxdl.xml")) + ) + contributed = sorted( + glob( + os.path.join( + get_nexus_definitions_path(), "contributed_definitions", "*.nxdl.xml" + ) + ) + ) nx_clss = [] for nexus_file in base_classes + applications + contributed: root = get_xml_root(nexus_file) - if root.attrib['category'] == 'base': - nx_clss.append(str(nexus_file[nexus_file.rindex(os.sep) + 1:])[:-9]) + if root.attrib["category"] == "base": + nx_clss.append(str(nexus_file[nexus_file.rindex(os.sep) + 1 :])[:-9]) nx_clss = sorted(nx_clss) return nx_clss @@ -175,12 +206,12 @@ def get_nx_units(): units_and_type_list.append(i) flag = False for line in units_and_type_list: - if line == 'anyUnitsAttr': + if line == "anyUnitsAttr": flag = True nx_units = [] - elif 'NX' in line and flag is True: + elif "NX" in line and flag is True: nx_units.append(line) - elif line == 'primitiveType': + elif line == "primitiveType": flag = False else: pass @@ -189,7 +220,7 @@ def get_nx_units(): def get_nx_attribute_type(): """Read attribute types from the NeXus definition/nxdlTypes.xsd file""" - filepath = get_nexus_definitions_path() + '/nxdlTypes.xsd' + filepath = get_nexus_definitions_path() + "/nxdlTypes.xsd" root = get_xml_root(filepath) units_and_type_list = [] for child in root: @@ -197,12 +228,12 @@ def get_nx_attribute_type(): units_and_type_list.append(i) flag = False for line in units_and_type_list: - if line == 'primitiveType': + if line == "primitiveType": flag = True nx_types = [] - elif 'NX' in line and flag is True: + elif "NX" in line and flag is True: nx_types.append(line) - elif line == 'anyUnitsAttr': + elif line == "anyUnitsAttr": flag = False else: pass @@ -210,23 +241,23 @@ def get_nx_attribute_type(): def get_node_name(node): - '''Node - xml node. Returns html documentation name. + """Node - xml node. Returns html documentation name. Either as specified by the 'name' or taken from the type (nx_class). Note that if only class name is available, the NX prefix is removed and - the string is converted to UPPER case.''' - if 'name' in node.attrib.keys(): - name = node.attrib['name'] + the string is converted to UPPER case.""" + if "name" in node.attrib.keys(): + name = node.attrib["name"] else: - name = node.attrib['type'] - if name.startswith('NX'): + name = node.attrib["type"] + if name.startswith("NX"): name = name[2:].upper() return name def belongs_to(nxdl_elem, child, name, class_type=None, hdf_name=None): """Checks if an HDF5 node name corresponds to a child of the NXDL element -uppercase letters in front can be replaced by arbitraty name, but -uppercase to lowercase match is preferred""" + uppercase letters in front can be replaced by arbitraty name, but + uppercase to lowercase match is preferred""" if class_type and get_nx_class(child) != class_type: return False act_htmlname = get_node_name(child) @@ -236,7 +267,7 @@ def belongs_to(nxdl_elem, child, name, class_type=None, hdf_name=None): if not hdf_name: # search for name fits is only allowed for hdf_nodes return False try: # check if nameType allows different name - name_any = bool(child.attrib['nameType'] == "any") + name_any = bool(child.attrib["nameType"] == "any") except KeyError: name_any = False params = [act_htmlname, chk_name, name_any, nxdl_elem, child, name] @@ -247,17 +278,25 @@ def belongs_to_capital(params): """Checking continues for Upper case""" (act_htmlname, chk_name, name_any, nxdl_elem, child, name) = params # or starts with capital and no reserved words used - if (name_any or 'A' <= act_htmlname[0] <= 'Z') and \ - name != 'doc' and name != 'enumeration': + if ( + (name_any or "A" <= act_htmlname[0] <= "Z") + and name != "doc" + and name != "enumeration" + ): fit = get_nx_namefit(chk_name, act_htmlname, name_any) # check if name fits if fit < 0: return False for child2 in nxdl_elem: - if get_local_name_from_xml(child) != \ - get_local_name_from_xml(child2) or get_node_name(child2) == act_htmlname: + if ( + get_local_name_from_xml(child) != get_local_name_from_xml(child2) + or get_node_name(child2) == act_htmlname + ): continue # check if the name of another sibling fits better - name_any2 = "nameType" in child2.attrib.keys() and child2.attrib["nameType"] == "any" + name_any2 = ( + "nameType" in child2.attrib.keys() + and child2.attrib["nameType"] == "any" + ) fit2 = get_nx_namefit(chk_name, get_node_name(child2), name_any2) if fit2 > fit: return False @@ -268,68 +307,81 @@ def belongs_to_capital(params): def get_local_name_from_xml(element): """Helper function to extract the element tag without the namespace.""" - return element.tag[element.tag.rindex("}") + 1:] + return element.tag[element.tag.rindex("}") + 1 :] def get_own_nxdl_child_reserved_elements(child, name, nxdl_elem): """checking reserved elements, like doc, enumeration""" - if get_local_name_from_xml(child) == 'doc' and name == 'doc': - if nxdl_elem.get('nxdlbase'): - child.set('nxdlbase', nxdl_elem.get('nxdlbase')) - child.set('nxdlbase_class', nxdl_elem.get('nxdlbase_class')) - child.set('nxdlpath', nxdl_elem.get('nxdlpath') + '/doc') + if get_local_name_from_xml(child) == "doc" and name == "doc": + if nxdl_elem.get("nxdlbase"): + child.set("nxdlbase", nxdl_elem.get("nxdlbase")) + child.set("nxdlbase_class", nxdl_elem.get("nxdlbase_class")) + child.set("nxdlpath", nxdl_elem.get("nxdlpath") + "/doc") return child - if get_local_name_from_xml(child) == 'enumeration' and name == 'enumeration': - if nxdl_elem.get('nxdlbase'): - child.set('nxdlbase', nxdl_elem.get('nxdlbase')) - child.set('nxdlbase_class', nxdl_elem.get('nxdlbase_class')) - child.set('nxdlpath', nxdl_elem.get('nxdlpath') + '/enumeration') + if get_local_name_from_xml(child) == "enumeration" and name == "enumeration": + if nxdl_elem.get("nxdlbase"): + child.set("nxdlbase", nxdl_elem.get("nxdlbase")) + child.set("nxdlbase_class", nxdl_elem.get("nxdlbase_class")) + child.set("nxdlpath", nxdl_elem.get("nxdlpath") + "/enumeration") return child return False def get_own_nxdl_child_base_types(child, class_type, nxdl_elem, name, hdf_name): """checking base types of group, field,m attribute""" - if get_local_name_from_xml(child) == 'group': - if (class_type is None or (class_type and get_nx_class(child) == class_type)) and \ - belongs_to(nxdl_elem, child, name, class_type, hdf_name): - if nxdl_elem.get('nxdlbase'): - child.set('nxdlbase', nxdl_elem.get('nxdlbase')) - child.set('nxdlbase_class', nxdl_elem.get('nxdlbase_class')) - child.set('nxdlpath', nxdl_elem.get('nxdlpath') + '/' + get_node_name(child)) + if get_local_name_from_xml(child) == "group": + if ( + class_type is None or (class_type and get_nx_class(child) == class_type) + ) and belongs_to(nxdl_elem, child, name, class_type, hdf_name): + if nxdl_elem.get("nxdlbase"): + child.set("nxdlbase", nxdl_elem.get("nxdlbase")) + child.set("nxdlbase_class", nxdl_elem.get("nxdlbase_class")) + child.set( + "nxdlpath", nxdl_elem.get("nxdlpath") + "/" + get_node_name(child) + ) return child - if get_local_name_from_xml(child) == 'field' and \ - belongs_to(nxdl_elem, child, name, None, hdf_name): - if nxdl_elem.get('nxdlbase'): - child.set('nxdlbase', nxdl_elem.get('nxdlbase')) - child.set('nxdlbase_class', nxdl_elem.get('nxdlbase_class')) - child.set('nxdlpath', nxdl_elem.get('nxdlpath') + '/' + get_node_name(child)) + if get_local_name_from_xml(child) == "field" and belongs_to( + nxdl_elem, child, name, None, hdf_name + ): + if nxdl_elem.get("nxdlbase"): + child.set("nxdlbase", nxdl_elem.get("nxdlbase")) + child.set("nxdlbase_class", nxdl_elem.get("nxdlbase_class")) + child.set( + "nxdlpath", nxdl_elem.get("nxdlpath") + "/" + get_node_name(child) + ) return child - if get_local_name_from_xml(child) == 'attribute' and \ - belongs_to(nxdl_elem, child, name, None, hdf_name): - if nxdl_elem.get('nxdlbase'): - child.set('nxdlbase', nxdl_elem.get('nxdlbase')) - child.set('nxdlbase_class', nxdl_elem.get('nxdlbase_class')) - child.set('nxdlpath', nxdl_elem.get('nxdlpath') + '/' + get_node_name(child)) + if get_local_name_from_xml(child) == "attribute" and belongs_to( + nxdl_elem, child, name, None, hdf_name + ): + if nxdl_elem.get("nxdlbase"): + child.set("nxdlbase", nxdl_elem.get("nxdlbase")) + child.set("nxdlbase_class", nxdl_elem.get("nxdlbase_class")) + child.set( + "nxdlpath", nxdl_elem.get("nxdlpath") + "/" + get_node_name(child) + ) return child return False -def get_own_nxdl_child(nxdl_elem, name, class_type=None, hdf_name=None, nexus_type=None): +def get_own_nxdl_child( + nxdl_elem, name, class_type=None, hdf_name=None, nexus_type=None +): """Checks if an NXDL child node fits to the specific name (either nxdl or hdf) - name - nxdl name - class_type - nxdl type or hdf classname (for groups, it is obligatory) - hdf_name - hdf name""" + name - nxdl name + class_type - nxdl type or hdf classname (for groups, it is obligatory) + hdf_name - hdf name""" for child in nxdl_elem: - if 'name' in child.attrib and child.attrib['name'] == name: - if nxdl_elem.get('nxdlbase'): - child.set('nxdlbase', nxdl_elem.get('nxdlbase')) - child.set('nxdlbase_class', nxdl_elem.get('nxdlbase_class')) - child.set('nxdlpath', nxdl_elem.get('nxdlpath') + '/' + get_node_name(child)) + if "name" in child.attrib and child.attrib["name"] == name: + if nxdl_elem.get("nxdlbase"): + child.set("nxdlbase", nxdl_elem.get("nxdlbase")) + child.set("nxdlbase_class", nxdl_elem.get("nxdlbase_class")) + child.set( + "nxdlpath", nxdl_elem.get("nxdlpath") + "/" + get_node_name(child) + ) return child for child in nxdl_elem: if "name" in child.attrib and child.attrib["name"] == name: - child.set('nxdlbase', nxdl_elem.get('nxdlbase')) + child.set("nxdlbase", nxdl_elem.get("nxdlbase")) return child for child in nxdl_elem: @@ -338,7 +390,9 @@ def get_own_nxdl_child(nxdl_elem, name, class_type=None, hdf_name=None, nexus_ty return result if nexus_type and get_local_name_from_xml(child) != nexus_type: continue - result = get_own_nxdl_child_base_types(child, class_type, nxdl_elem, name, hdf_name) + result = get_own_nxdl_child_base_types( + child, class_type, nxdl_elem, name, hdf_name + ) if result is not False: return result return None @@ -346,21 +400,28 @@ def get_own_nxdl_child(nxdl_elem, name, class_type=None, hdf_name=None, nexus_ty def find_definition_file(bc_name): """find the nxdl file corresponding to the name. - Note that it first checks in contributed and goes beyond only if no contributed found""" + Note that it first checks in contributed and goes beyond only if no contributed found + """ bc_filename = None - for nxdl_folder in ['contributed_definitions', 'base_classes', 'applications']: - if os.path.exists(f"{get_nexus_definitions_path()}{os.sep}" - f"{nxdl_folder}{os.sep}{bc_name}.nxdl.xml"): - bc_filename = f"{get_nexus_definitions_path()}{os.sep}" \ - f"{nxdl_folder}{os.sep}{bc_name}.nxdl.xml" + for nxdl_folder in ["contributed_definitions", "base_classes", "applications"]: + if os.path.exists( + f"{get_nexus_definitions_path()}{os.sep}" + f"{nxdl_folder}{os.sep}{bc_name}.nxdl.xml" + ): + bc_filename = ( + f"{get_nexus_definitions_path()}{os.sep}" + f"{nxdl_folder}{os.sep}{bc_name}.nxdl.xml" + ) break return bc_filename -def get_nxdl_child(nxdl_elem, name, class_type=None, hdf_name=None, nexus_type=None, go_base=True): # pylint: disable=too-many-arguments +def get_nxdl_child( + nxdl_elem, name, class_type=None, hdf_name=None, nexus_type=None, go_base=True +): # pylint: disable=too-many-arguments """Get the NXDL child node corresponding to a specific name -(e.g. of an HDF5 node,or of a documentation) note that if child is not found in application -definition, it also checks for the base classes""" + (e.g. of an HDF5 node,or of a documentation) note that if child is not found in application + definition, it also checks for the base classes""" # search for possible fits for hdf_nodes : skipped # only exact hits are returned when searching an nxdl child own_child = get_own_nxdl_child(nxdl_elem, name, class_type, hdf_name, nexus_type) @@ -369,18 +430,20 @@ def get_nxdl_child(nxdl_elem, name, class_type=None, hdf_name=None, nexus_type=N if not go_base: return None bc_name = get_nx_class(nxdl_elem) # check in the base class, app def or contributed - if bc_name[2] == '_': # filter primitive types + if bc_name[2] == "_": # filter primitive types return None - if bc_name == "group": # Check if it is the root element. Then send to NXroot.nxdl.xml + if ( + bc_name == "group" + ): # Check if it is the root element. Then send to NXroot.nxdl.xml bc_name = "NXroot" bc_filename = find_definition_file(bc_name) if not bc_filename: - raise ValueError('nxdl file not found in definitions folder!') + raise ValueError("nxdl file not found in definitions folder!") bc_obj = ET.parse(bc_filename).getroot() - bc_obj.set('nxdlbase', bc_filename) - if 'category' in bc_obj.attrib: - bc_obj.set('nxdlbase_class', bc_obj.attrib['category']) - bc_obj.set('nxdlpath', '') + bc_obj.set("nxdlbase", bc_filename) + if "category" in bc_obj.attrib: + bc_obj.set("nxdlbase_class", bc_obj.attrib["category"]) + bc_obj.set("nxdlpath", "") return get_own_nxdl_child(bc_obj, name, class_type, hdf_name, nexus_type) @@ -388,12 +451,16 @@ def get_required_string(nxdl_elem): """Check for being REQUIRED, RECOMMENDED, OPTIONAL, NOT IN SCHEMA""" if nxdl_elem is None: return "<>" - is_optional = 'optional' in nxdl_elem.attrib.keys() \ - and nxdl_elem.attrib['optional'] == "true" - is_minoccurs = 'minOccurs' in nxdl_elem.attrib.keys() \ - and nxdl_elem.attrib['minOccurs'] == "0" - is_recommended = 'recommended' in nxdl_elem.attrib.keys() \ - and nxdl_elem.attrib['recommended'] == "true" + is_optional = ( + "optional" in nxdl_elem.attrib.keys() and nxdl_elem.attrib["optional"] == "true" + ) + is_minoccurs = ( + "minOccurs" in nxdl_elem.attrib.keys() and nxdl_elem.attrib["minOccurs"] == "0" + ) + is_recommended = ( + "recommended" in nxdl_elem.attrib.keys() + and nxdl_elem.attrib["recommended"] == "true" + ) if is_recommended: return "<>" @@ -401,7 +468,7 @@ def get_required_string(nxdl_elem): return "<>" # default optionality: in BASE CLASSES is true; in APPLICATIONS is false try: - if nxdl_elem.get('nxdlbase_class') == 'base': + if nxdl_elem.get("nxdlbase_class") == "base": return "<>" except TypeError: return "<>" @@ -410,40 +477,44 @@ def get_required_string(nxdl_elem): def chk_nxdataaxis_v2(hdf_node, name, logger): """Check if dataset is an axis""" - own_signal = hdf_node.attrs.get('signal') # check for being a Signal + own_signal = hdf_node.attrs.get("signal") # check for being a Signal if own_signal is str and own_signal == "1": logger.debug("Dataset referenced (v2) as NXdata SIGNAL") - own_axes = hdf_node.attrs.get('axes') # check for being an axis + own_axes = hdf_node.attrs.get("axes") # check for being an axis if own_axes is str: - axes = own_axes.split(':') + axes = own_axes.split(":") for i in len(axes): if axes[i] and name == axes[i]: logger.debug("Dataset referenced (v2) as NXdata AXIS #%d", i) return None - ownpaxis = hdf_node.attrs.get('primary') - own_axis = hdf_node.attrs.get('axis') + ownpaxis = hdf_node.attrs.get("primary") + own_axis = hdf_node.attrs.get("axis") if own_axis is int: # also convention v1 if ownpaxis is int and ownpaxis == 1: logger.debug("Dataset referenced (v2) as NXdata AXIS #%d", own_axis - 1) else: logger.debug( - "Dataset referenced (v2) as NXdata (primary/alternative) AXIS #%d", own_axis - 1) + "Dataset referenced (v2) as NXdata (primary/alternative) AXIS #%d", + own_axis - 1, + ) return None def chk_nxdataaxis(hdf_node, name, logger): """NEXUS Data Plotting Standard v3: new version from 2014""" - if not isinstance(hdf_node, h5py.Dataset): # check if it is a field in an NXdata node + if not isinstance( + hdf_node, h5py.Dataset + ): # check if it is a field in an NXdata node return None parent = hdf_node.parent - if not parent or (parent and not parent.attrs.get('NX_class') == "NXdata"): + if not parent or (parent and not parent.attrs.get("NX_class") == "NXdata"): return None - signal = parent.attrs.get('signal') # chk for Signal + signal = parent.attrs.get("signal") # chk for Signal if signal and name == signal: logger.debug("Dataset referenced as NXdata SIGNAL") return None - axes = parent.attrs.get('axes') # check for default Axes + axes = parent.attrs.get("axes") # check for default Axes if axes is str: if name == axes: logger.debug("Dataset referenced as NXdata AXIS") @@ -451,13 +522,13 @@ def chk_nxdataaxis(hdf_node, name, logger): elif axes is not None: for i, j in enumerate(axes): if name == j: - indices = parent.attrs.get(j + '_indices') + indices = parent.attrs.get(j + "_indices") if indices is int: logger.debug(f"Dataset referenced as NXdata AXIS #{indices}") else: logger.debug(f"Dataset referenced as NXdata AXIS #{i}") return None - indices = parent.attrs.get(name + '_indices') # check for alternative Axes + indices = parent.attrs.get(name + "_indices") # check for alternative Axes if indices is int: logger.debug(f"Dataset referenced as NXdata alternative AXIS #{indices}") return chk_nxdataaxis_v2(hdf_node, name, logger) # check for older conventions @@ -467,7 +538,7 @@ def chk_nxdataaxis(hdf_node, name, logger): def write_doc_string(logger, doc, attr): """Simple function that prints a line in the logger if doc exists""" if doc: - logger.debug("@" + attr + ' [NX_CHAR]') + logger.debug("@" + attr + " [NX_CHAR]") return logger, doc, attr @@ -477,61 +548,96 @@ def try_find_units(logger, elem, nxdl_path, doc, attr): try: # try to find if units is defined inside the field in the NXDL element unit = elem.attrib[attr] if doc: - logger.debug(get_node_concept_path(elem) + "@" + attr + ' [' + unit + ']') + logger.debug(get_node_concept_path(elem) + "@" + attr + " [" + unit + "]") elem = None nxdl_path.append(attr) - except KeyError: # otherwise try to find if units is defined as a child of the NXDL element + except ( + KeyError + ): # otherwise try to find if units is defined as a child of the NXDL element orig_elem = elem - elem = get_nxdl_child(elem, attr, nexus_type='attribute') + elem = get_nxdl_child(elem, attr, nexus_type="attribute") if elem is not None: if doc: - logger.debug(get_node_concept_path(orig_elem) - + "@" + attr + ' - [' + get_nx_class(elem) + ']') + logger.debug( + get_node_concept_path(orig_elem) + + "@" + + attr + + " - [" + + get_nx_class(elem) + + "]" + ) nxdl_path.append(elem) else: # if no units category were defined in NXDL: if doc: - logger.debug(get_node_concept_path(orig_elem) - + "@" + attr + " - REQUIRED, but undefined unit category") + logger.debug( + get_node_concept_path(orig_elem) + + "@" + + attr + + " - REQUIRED, but undefined unit category" + ) nxdl_path.append(attr) return logger, elem, nxdl_path, doc, attr def check_attr_name_nxdl(param): """Check for ATTRIBUTENAME_units in NXDL (normal). -If not defined, check for ATTRIBUTENAME to see if the ATTRIBUTE -is in the SCHEMA, but no units category were defined. """ + If not defined, check for ATTRIBUTENAME to see if the ATTRIBUTE + is in the SCHEMA, but no units category were defined.""" (logger, elem, nxdl_path, doc, attr, req_str) = param orig_elem = elem - elem2 = get_nxdl_child(elem, attr, nexus_type='attribute') + elem2 = get_nxdl_child(elem, attr, nexus_type="attribute") if elem2 is not None: # check for ATTRIBUTENAME_units in NXDL (normal) elem = elem2 if doc: - logger.debug(get_node_concept_path(orig_elem) - + "@" + attr + ' - [' + get_nx_class(elem) + ']') + logger.debug( + get_node_concept_path(orig_elem) + + "@" + + attr + + " - [" + + get_nx_class(elem) + + "]" + ) nxdl_path.append(elem) else: # if not defined, check for ATTRIBUTENAME to see if the ATTRIBUTE # is in the SCHEMA, but no units category were defined - elem2 = get_nxdl_child(elem, attr[:-6], nexus_type='attribute') + elem2 = get_nxdl_child(elem, attr[:-6], nexus_type="attribute") if elem2 is not None: - req_str = '<>' + req_str = "<>" if doc: - logger.debug(get_node_concept_path(orig_elem) - + "@" + attr + " - RECOMMENDED, but undefined unit category") + logger.debug( + get_node_concept_path(orig_elem) + + "@" + + attr + + " - RECOMMENDED, but undefined unit category" + ) nxdl_path.append(attr) else: # otherwise: NOT IN SCHEMA elem = elem2 if doc: - logger.debug(get_node_concept_path(orig_elem) + "@" + attr + " - IS NOT IN SCHEMA") + logger.debug( + get_node_concept_path(orig_elem) + + "@" + + attr + + " - IS NOT IN SCHEMA" + ) return logger, elem, nxdl_path, doc, attr, req_str -def try_find_default(logger, orig_elem, elem, nxdl_path, doc, attr): # pylint: disable=too-many-arguments - """Try to find if default is defined as a child of the NXDL element """ +def try_find_default( + logger, orig_elem, elem, nxdl_path, doc, attr +): # pylint: disable=too-many-arguments + """Try to find if default is defined as a child of the NXDL element""" if elem is not None: if doc: - logger.debug(get_node_concept_path(orig_elem) - + "@" + attr + ' - [' + get_nx_class(elem) + ']') + logger.debug( + get_node_concept_path(orig_elem) + + "@" + + attr + + " - [" + + get_nx_class(elem) + + "]" + ) nxdl_path.append(elem) else: # if no default category were defined in NXDL: if doc: @@ -540,38 +646,50 @@ def try_find_default(logger, orig_elem, elem, nxdl_path, doc, attr): # pylint: return logger, elem, nxdl_path, doc, attr -def other_attrs(logger, orig_elem, elem, nxdl_path, doc, attr): # pylint: disable=too-many-arguments - """Handle remaining attributes """ +def other_attrs( + logger, orig_elem, elem, nxdl_path, doc, attr +): # pylint: disable=too-many-arguments + """Handle remaining attributes""" if elem is not None: if doc: - logger.debug(get_node_concept_path(orig_elem) - + "@" + attr + ' - [' + get_nx_class(elem) + ']') + logger.debug( + get_node_concept_path(orig_elem) + + "@" + + attr + + " - [" + + get_nx_class(elem) + + "]" + ) nxdl_path.append(elem) else: if doc: - logger.debug(get_node_concept_path(orig_elem) + "@" + attr + " - IS NOT IN SCHEMA") + logger.debug( + get_node_concept_path(orig_elem) + "@" + attr + " - IS NOT IN SCHEMA" + ) return logger, elem, nxdl_path, doc, attr def check_deprecation_enum_axis(variables, doc, elist, attr, hdf_node): - """Check for several attributes. - deprecation - enums - nxdataaxis """ + """Check for several attributes. - deprecation - enums - nxdataaxis""" logger, elem, path = variables - dep_str = elem.attrib.get('deprecated') # check for deprecation + dep_str = elem.attrib.get("deprecated") # check for deprecation if dep_str: if doc: logger.debug("DEPRECATED - " + dep_str) for base_elem in elist if not attr else [elem]: # check for enums - sdoc = get_nxdl_child(base_elem, 'enumeration', go_base=False) + sdoc = get_nxdl_child(base_elem, "enumeration", go_base=False) if sdoc is not None: if doc: logger.debug("enumeration (" + get_node_concept_path(base_elem) + "):") for item in sdoc: - if get_local_name_from_xml(item) == 'item': + if get_local_name_from_xml(item) == "item": if doc: - logger.debug("-> " + item.attrib['value']) - chk_nxdataaxis(hdf_node, path.split('/')[-1], logger) # look for NXdata reference (axes/signal) + logger.debug("-> " + item.attrib["value"]) + chk_nxdataaxis( + hdf_node, path.split("/")[-1], logger + ) # look for NXdata reference (axes/signal) for base_elem in elist if not attr else [elem]: # check for doc - sdoc = get_nxdl_child(base_elem, 'doc', go_base=False) + sdoc = get_nxdl_child(base_elem, "doc", go_base=False) if doc: logger.debug("documentation (" + get_node_concept_path(base_elem) + "):") logger.debug(sdoc.text if sdoc is not None else "") @@ -580,11 +698,12 @@ def check_deprecation_enum_axis(variables, doc, elist, attr, hdf_node): def get_node_concept_path(elem): """get the short version of nxdlbase:nxdlpath""" - return str(elem.get('nxdlbase').split('/')[-1] + ":" + elem.get('nxdlpath')) + return str(elem.get("nxdlbase").split("/")[-1] + ":" + elem.get("nxdlpath")) def get_nxdl_attr_doc( # pylint: disable=too-many-arguments,too-many-locals - elem, elist, attr, hdf_node, logger, doc, nxdl_path, req_str, path, hdf_info): + elem, elist, attr, hdf_node, logger, doc, nxdl_path, req_str, path, hdf_info +): """Get nxdl documentation for an attribute""" new_elem = [] old_elem = elem @@ -592,43 +711,44 @@ def get_nxdl_attr_doc( # pylint: disable=too-many-arguments,too-many-locals act_elem = act_elem1 # NX_class is a compulsory attribute for groups in a nexus file # which should match the type of the corresponding NXDL element - if attr == 'NX_class' and not isinstance(hdf_node, h5py.Dataset) and elem_index == 0: + if ( + attr == "NX_class" + and not isinstance(hdf_node, h5py.Dataset) + and elem_index == 0 + ): elem = None logger, doc, attr = write_doc_string(logger, doc, attr) new_elem = elem break # units category is a compulsory attribute for any fields - if attr == 'units' and isinstance(hdf_node, h5py.Dataset): + if attr == "units" and isinstance(hdf_node, h5py.Dataset): req_str = "<>" - logger, act_elem, nxdl_path, doc, attr = try_find_units(logger, - act_elem, - nxdl_path, - doc, - attr) + logger, act_elem, nxdl_path, doc, attr = try_find_units( + logger, act_elem, nxdl_path, doc, attr + ) # units for attributes can be given as ATTRIBUTENAME_units - elif attr.endswith('_units'): - logger, act_elem, nxdl_path, doc, attr, req_str = check_attr_name_nxdl((logger, - act_elem, - nxdl_path, - doc, - attr, - req_str)) + elif attr.endswith("_units"): + logger, act_elem, nxdl_path, doc, attr, req_str = check_attr_name_nxdl( + (logger, act_elem, nxdl_path, doc, attr, req_str) + ) # default is allowed for groups - elif attr == 'default' and not isinstance(hdf_node, h5py.Dataset): + elif attr == "default" and not isinstance(hdf_node, h5py.Dataset): req_str = "<>" # try to find if default is defined as a child of the NXDL element - act_elem = get_nxdl_child(act_elem, attr, nexus_type='attribute', go_base=False) - logger, act_elem, nxdl_path, doc, attr = try_find_default(logger, - act_elem1, - act_elem, - nxdl_path, - doc, - attr) + act_elem = get_nxdl_child( + act_elem, attr, nexus_type="attribute", go_base=False + ) + logger, act_elem, nxdl_path, doc, attr = try_find_default( + logger, act_elem1, act_elem, nxdl_path, doc, attr + ) else: # other attributes - act_elem = get_nxdl_child(act_elem, attr, nexus_type='attribute', go_base=False) + act_elem = get_nxdl_child( + act_elem, attr, nexus_type="attribute", go_base=False + ) if act_elem is not None: - logger, act_elem, nxdl_path, doc, attr = \ - other_attrs(logger, act_elem1, act_elem, nxdl_path, doc, attr) + logger, act_elem, nxdl_path, doc, attr = other_attrs( + logger, act_elem1, act_elem, nxdl_path, doc, attr + ) if act_elem is not None: new_elem.append(act_elem) if req_str is None: @@ -636,14 +756,18 @@ def get_nxdl_attr_doc( # pylint: disable=too-many-arguments,too-many-locals if doc: logger.debug(req_str) variables = [logger, act_elem, path] - logger, elem, path, doc, elist, attr, hdf_node = check_deprecation_enum_axis(variables, - doc, - elist, - attr, - hdf_node) + ( + logger, + elem, + path, + doc, + elist, + attr, + hdf_node, + ) = check_deprecation_enum_axis(variables, doc, elist, attr, hdf_node) elem = old_elem if req_str is None and doc: - if attr != 'NX_class': + if attr != "NX_class": logger.debug("@" + attr + " - IS NOT IN SCHEMA") logger.debug("") return (req_str, get_nxdl_entry(hdf_info), nxdl_path) @@ -651,48 +775,54 @@ def get_nxdl_attr_doc( # pylint: disable=too-many-arguments,too-many-locals def get_nxdl_doc(hdf_info, logger, doc, attr=False): """Get nxdl documentation for an HDF5 node (or its attribute)""" - hdf_node = hdf_info['hdf_node'] + hdf_node = hdf_info["hdf_node"] # new way: retrieve multiple inherited base classes - (class_path, nxdl_path, elist) = \ - get_inherited_nodes(None, nx_name=get_nxdl_entry(hdf_info), hdf_node=hdf_node, - hdf_path=hdf_info['hdf_path'] if 'hdf_path' in hdf_info else None, - hdf_root=hdf_info['hdf_root'] if 'hdf_root' in hdf_info else None) + (class_path, nxdl_path, elist) = get_inherited_nodes( + None, + nx_name=get_nxdl_entry(hdf_info), + hdf_node=hdf_node, + hdf_path=hdf_info["hdf_path"] if "hdf_path" in hdf_info else None, + hdf_root=hdf_info["hdf_root"] if "hdf_root" in hdf_info else None, + ) elem = elist[0] if class_path and elist else None if doc: logger.debug("classpath: " + str(class_path)) - logger.debug("NOT IN SCHEMA" if elem is None else - "classes:\n" + "\n".join - (get_node_concept_path(e) for e in elist)) + logger.debug( + "NOT IN SCHEMA" + if elem is None + else "classes:\n" + "\n".join(get_node_concept_path(e) for e in elist) + ) # old solution with a single elem instead of using elist path = get_nx_class_path(hdf_info) req_str = None if elem is None: if doc: logger.debug("") - return ('None', None, None) + return ("None", None, None) if attr: - return get_nxdl_attr_doc(elem, elist, attr, hdf_node, logger, doc, nxdl_path, - req_str, path, hdf_info) + return get_nxdl_attr_doc( + elem, elist, attr, hdf_node, logger, doc, nxdl_path, req_str, path, hdf_info + ) req_str = get_required_string(elem) # check for being required if doc: logger.debug(req_str) variables = [logger, elem, path] - logger, elem, path, doc, elist, attr, hdf_node = check_deprecation_enum_axis(variables, - doc, - elist, - attr, - hdf_node) + logger, elem, path, doc, elist, attr, hdf_node = check_deprecation_enum_axis( + variables, doc, elist, attr, hdf_node + ) return (req_str, get_nxdl_entry(hdf_info), nxdl_path) def get_doc(node, ntype, nxhtml, nxpath): """Get documentation""" # URL for html documentation - anchor = '' + anchor = "" for n_item in nxpath: anchor += n_item.lower() + "-" - anchor = ('https://manual.nexusformat.org/classes/', - nxhtml + "#" + anchor.replace('_', '-') + ntype) + anchor = ( + "https://manual.nexusformat.org/classes/", + nxhtml + "#" + anchor.replace("_", "-") + ntype, + ) if not ntype: anchor = anchor[:-1] doc = "" # RST documentation from the field 'doc' @@ -701,9 +831,13 @@ def get_doc(node, ntype, nxhtml, nxpath): doc = doc_field.text (index, enums) = get_enums(node) # enums if index: - enum_str = "\n " + ("Possible values:" - if len(enums.split(',')) > 1 - else "Obligatory value:") + "\n " + enums + "\n" + enum_str = ( + "\n " + + ("Possible values:" if len(enums.split(",")) > 1 else "Obligatory value:") + + "\n " + + enums + + "\n" + ) else: enum_str = "" return anchor, doc + enum_str @@ -714,17 +848,21 @@ def print_doc(node, ntype, level, nxhtml, nxpath): anchor, doc = get_doc(node, ntype, nxhtml, nxpath) print(" " * (level + 1) + anchor) preferred_width = 80 + level * 2 - wrapper = textwrap.TextWrapper(initial_indent=' ' * (level + 1), width=preferred_width, - subsequent_indent=' ' * (level + 1), expand_tabs=False, - tabsize=0) + wrapper = textwrap.TextWrapper( + initial_indent=" " * (level + 1), + width=preferred_width, + subsequent_indent=" " * (level + 1), + expand_tabs=False, + tabsize=0, + ) if doc is not None: - for par in doc.split('\n'): + for par in doc.split("\n"): print(wrapper.fill(par)) def get_namespace(element): """Extracts the namespace for elements in the NXDL""" - return element.tag[element.tag.index("{"):element.tag.rindex("}") + 1] + return element.tag[element.tag.index("{") : element.tag.rindex("}") + 1] def get_enums(node): @@ -737,15 +875,16 @@ def get_enums(node): for enumeration in node.findall(f"{namespace}enumeration"): for item in enumeration.findall(f"{namespace}item"): enums.append(item.attrib["value"]) - enums = ','.join(enums) + enums = ",".join(enums) if enums != "": - return (True, '[' + enums + ']') + return (True, "[" + enums + "]") return (False, "") # if there is no enumeration tag, returns empty string def add_base_classes(elist, nx_name=None, elem: ET.Element = None): """Add the base classes corresponding to the last eleme in elist to the list. Note that if -elist is empty, a nxdl file with the name of nx_name or a rather room elem is used if provided""" + elist is empty, a nxdl file with the name of nx_name or a rather room elem is used if provided + """ if elist and nx_name is None: nx_name = get_nx_class(elist[-1]) # to support recursive defintions, like NXsample in NXsample, the following test is removed @@ -758,48 +897,51 @@ def add_base_classes(elist, nx_name=None, elem: ET.Element = None): if nxdl_file_path is None: nxdl_file_path = f"{nx_name}.nxdl.xml" elem = ET.parse(nxdl_file_path).getroot() - elem.set('nxdlbase', nxdl_file_path) + elem.set("nxdlbase", nxdl_file_path) else: - elem.set('nxdlbase', '') - if 'category' in elem.attrib: - elem.set('nxdlbase_class', elem.attrib['category']) - elem.set('nxdlpath', '') + elem.set("nxdlbase", "") + if "category" in elem.attrib: + elem.set("nxdlbase_class", elem.attrib["category"]) + elem.set("nxdlpath", "") elist.append(elem) # add inherited base class - if 'extends' in elem.attrib and elem.attrib['extends'] != 'NXobject': - add_base_classes(elist, elem.attrib['extends']) + if "extends" in elem.attrib and elem.attrib["extends"] != "NXobject": + add_base_classes(elist, elem.attrib["extends"]) else: add_base_classes(elist) def set_nxdlpath(child, nxdl_elem): """ - Setting up child nxdlbase, nxdlpath and nxdlbase_class from nxdl_element. + Setting up child nxdlbase, nxdlpath and nxdlbase_class from nxdl_element. """ - if nxdl_elem.get('nxdlbase'): - child.set('nxdlbase', nxdl_elem.get('nxdlbase')) - child.set('nxdlbase_class', nxdl_elem.get('nxdlbase_class')) - child.set('nxdlpath', nxdl_elem.get('nxdlpath') + '/' + get_node_name(child)) + if nxdl_elem.get("nxdlbase"): + child.set("nxdlbase", nxdl_elem.get("nxdlbase")) + child.set("nxdlbase_class", nxdl_elem.get("nxdlbase_class")) + child.set("nxdlpath", nxdl_elem.get("nxdlpath") + "/" + get_node_name(child)) return child def get_direct_child(nxdl_elem, html_name): - """ returns the child of nxdl_elem which has a name - corresponding to the the html documentation name html_name""" + """returns the child of nxdl_elem which has a name + corresponding to the the html documentation name html_name""" for child in nxdl_elem: - if get_local_name_from_xml(child) in ('group', 'field', 'attribute') and \ - html_name == get_node_name(child): + if get_local_name_from_xml(child) in ( + "group", + "field", + "attribute", + ) and html_name == get_node_name(child): decorated_child = set_nxdlpath(child, nxdl_elem) return decorated_child return None def get_field_child(nxdl_elem, html_name): - """ returns the child of nxdl_elem which has a name - corresponding to the html documentation name html_name""" + """returns the child of nxdl_elem which has a name + corresponding to the html documentation name html_name""" data_child = None for child in nxdl_elem: - if get_local_name_from_xml(child) != 'field': + if get_local_name_from_xml(child) != "field": continue if get_node_name(child) == html_name: data_child = set_nxdlpath(child, nxdl_elem) @@ -808,27 +950,27 @@ def get_field_child(nxdl_elem, html_name): def get_best_nxdata_child(nxdl_elem, hdf_node, hdf_name): - """ returns the child of an NXdata nxdl_elem which has a name - corresponding to the hdf_name""" + """returns the child of an NXdata nxdl_elem which has a name + corresponding to the hdf_name""" nxdata = hdf_node.parent signals = [] - if 'signal' in nxdata.attrs.keys(): + if "signal" in nxdata.attrs.keys(): signals.append(nxdata.attrs.get("signal")) if "auxiliary_signals" in nxdata.attrs.keys(): for aux_signal in nxdata.attrs.get("auxiliary_signals"): signals.append(aux_signal) - data_child = get_field_child(nxdl_elem, 'DATA') - data_error_child = get_field_child(nxdl_elem, 'FIELDNAME_errors') + data_child = get_field_child(nxdl_elem, "DATA") + data_error_child = get_field_child(nxdl_elem, "FIELDNAME_errors") for signal in signals: if signal == hdf_name: return (data_child, 100) - if hdf_name.endswith('_errors') and signal == hdf_name[:-7]: + if hdf_name.endswith("_errors") and signal == hdf_name[:-7]: return (data_error_child, 100) axes = [] if "axes" in nxdata.attrs.keys(): for axis in nxdata.attrs.get("axes"): axes.append(axis) - axis_child = get_field_child(nxdl_elem, 'AXISNAME') + axis_child = get_field_child(nxdl_elem, "AXISNAME") for axis in axes: if axis == hdf_name: return (axis_child, 100) @@ -836,22 +978,29 @@ def get_best_nxdata_child(nxdl_elem, hdf_node, hdf_name): def get_best_child(nxdl_elem, hdf_node, hdf_name, hdf_class_name, nexus_type): - """ returns the child of nxdl_elem which has a name - corresponding to the the html documentation name html_name""" + """returns the child of nxdl_elem which has a name + corresponding to the the html documentation name html_name""" bestfit = -1 bestchild = None - if 'name' in nxdl_elem.attrib.keys() and nxdl_elem.attrib['name'] == 'NXdata' and \ - hdf_node is not None and hdf_node.parent is not None and \ - hdf_node.parent.attrs.get('NX_class') == 'NXdata': + if ( + "name" in nxdl_elem.attrib.keys() + and nxdl_elem.attrib["name"] == "NXdata" + and hdf_node is not None + and hdf_node.parent is not None + and hdf_node.parent.attrs.get("NX_class") == "NXdata" + ): (fnd_child, fit) = get_best_nxdata_child(nxdl_elem, hdf_node, hdf_name) if fnd_child is not None: return (fnd_child, fit) for child in nxdl_elem: fit = -2 - if get_local_name_from_xml(child) == nexus_type and \ - (nexus_type != 'group' or get_nx_class(child) == hdf_class_name): - name_any = "nameType" in nxdl_elem.attrib.keys() and \ - nxdl_elem.attrib["nameType"] == "any" + if get_local_name_from_xml(child) == nexus_type and ( + nexus_type != "group" or get_nx_class(child) == hdf_class_name + ): + name_any = ( + "nameType" in nxdl_elem.attrib.keys() + and nxdl_elem.attrib["nameType"] == "any" + ) fit = get_nx_namefit(hdf_name, get_node_name(child), name_any) if fit > bestfit: bestfit = fit @@ -869,9 +1018,13 @@ def walk_elist(elist, html_name): for potential_direct_parent in elist: main_child = get_direct_child(potential_direct_parent, html_name) if main_child is not None: - (fitting_child, _) = get_best_child(elist[ind], None, html_name, - get_nx_class(main_child), - get_local_name_from_xml(main_child)) + (fitting_child, _) = get_best_child( + elist[ind], + None, + html_name, + get_nx_class(main_child), + get_local_name_from_xml(main_child), + ) if fitting_child is not None: child = fitting_child break @@ -880,10 +1033,12 @@ def walk_elist(elist, html_name): del elist[ind] continue # override: remove low priority inheritance classes if class_type is overriden - if len(elist) > ind + 1 and get_nx_class(elist[ind]) != get_nx_class(elist[ind + 1]): - del elist[ind + 1:] + if len(elist) > ind + 1 and get_nx_class(elist[ind]) != get_nx_class( + elist[ind + 1] + ): + del elist[ind + 1 :] # add new base class(es) if new element brings such (and not a primitive type) - if len(elist) == ind + 1 and get_nx_class(elist[ind])[0:3] != 'NX_': + if len(elist) == ind + 1 and get_nx_class(elist[ind])[0:3] != "NX_": add_base_classes(elist) return elist, html_name @@ -894,20 +1049,18 @@ def helper_get_inherited_nodes(hdf_info2, elist, pind, attr): hdf_name = hdf_path[pind] hdf_class_name = hdf_class_path[pind] if pind < len(hdf_path) - (2 if attr else 1): - act_nexus_type = 'group' + act_nexus_type = "group" elif pind == len(hdf_path) - 1 and attr: - act_nexus_type = 'attribute' + act_nexus_type = "attribute" else: - act_nexus_type = 'field' if isinstance(hdf_node, h5py.Dataset) else 'group' + act_nexus_type = "field" if isinstance(hdf_node, h5py.Dataset) else "group" # find the best fitting name in all children bestfit = -1 html_name = None for ind in range(len(elist) - 1, -1, -1): - newelem, fit = get_best_child(elist[ind], - hdf_node, - hdf_name, - hdf_class_name, - act_nexus_type) + newelem, fit = get_best_child( + elist[ind], hdf_node, hdf_name, hdf_class_name, act_nexus_type + ) if fit >= bestfit and newelem is not None: html_name = get_node_name(newelem) return hdf_path, hdf_node, hdf_class_path, elist, pind, attr, html_name @@ -915,15 +1068,21 @@ def helper_get_inherited_nodes(hdf_info2, elist, pind, attr): def get_hdf_path(hdf_info): """Get the hdf_path from an hdf_info""" - if 'hdf_path' in hdf_info: - return hdf_info['hdf_path'].split('/')[1:] - return hdf_info['hdf_node'].name.split('/')[1:] + if "hdf_path" in hdf_info: + return hdf_info["hdf_path"].split("/")[1:] + return hdf_info["hdf_node"].name.split("/")[1:] @lru_cache(maxsize=None) -def get_inherited_nodes(nxdl_path: str = None, # pylint: disable=too-many-arguments,too-many-locals - nx_name: str = None, elem: ET.Element = None, - hdf_node=None, hdf_path=None, hdf_root=None, attr=False): +def get_inherited_nodes( + nxdl_path: str = None, # pylint: disable=too-many-arguments,too-many-locals + nx_name: str = None, + elem: ET.Element = None, + hdf_node=None, + hdf_path=None, + hdf_root=None, + attr=False, +): """Returns a list of ET.Element for the given path.""" # let us start with the given definition file elist = [] # type: ignore[var-annotated] @@ -932,27 +1091,33 @@ def get_inherited_nodes(nxdl_path: str = None, # pylint: disable=too-many-argum class_path = [] # type: ignore[var-annotated] if hdf_node is not None: - hdf_info = {'hdf_node': hdf_node} + hdf_info = {"hdf_node": hdf_node} if hdf_path: - hdf_info['hdf_path'] = hdf_path + hdf_info["hdf_path"] = hdf_path if hdf_root: - hdf_root['hdf_root'] = hdf_root - hdf_node = hdf_info['hdf_node'] + hdf_root["hdf_root"] = hdf_root + hdf_node = hdf_info["hdf_node"] hdf_path = get_hdf_path(hdf_info) - hdf_class_path = get_nx_class_path(hdf_info).split('/')[1:] + hdf_class_path = get_nx_class_path(hdf_info).split("/")[1:] if attr: hdf_path.append(attr) hdf_class_path.append(attr) path = hdf_path else: - html_path = nxdl_path.split('/')[1:] + html_path = nxdl_path.split("/")[1:] path = html_path for pind in range(len(path)): if hdf_node is not None: hdf_info2 = [hdf_path, hdf_node, hdf_class_path] - [hdf_path, hdf_node, hdf_class_path, elist, - pind, attr, html_name] = helper_get_inherited_nodes(hdf_info2, elist, - pind, attr) + [ + hdf_path, + hdf_node, + hdf_class_path, + elist, + pind, + attr, + html_name, + ] = helper_get_inherited_nodes(hdf_info2, elist, pind, attr) if html_name is None: # return if NOT IN SCHEMA return (class_path, nxdl_elem_path, None) else: @@ -964,9 +1129,12 @@ def get_inherited_nodes(nxdl_path: str = None, # pylint: disable=too-many-argum return (class_path, nxdl_elem_path, elist) -def get_node_at_nxdl_path(nxdl_path: str = None, - nx_name: str = None, elem: ET.Element = None, - exc: bool = True): +def get_node_at_nxdl_path( + nxdl_path: str = None, + nx_name: str = None, + elem: ET.Element = None, + exc: bool = True, +): """Returns an ET.Element for the given path. This function either takes the name for the NeXus Application Definition we are looking for or the root elem from a previously loaded NXDL file @@ -975,32 +1143,38 @@ def get_node_at_nxdl_path(nxdl_path: str = None, (class_path, nxdlpath, elist) = get_inherited_nodes(nxdl_path, nx_name, elem) except ValueError as value_error: if exc: - raise NxdlAttributeError(f"Attributes were not found for {nxdl_path}. " - "Please check this entry in the template dictionary.") \ - from value_error + raise NxdlAttributeError( + f"Attributes were not found for {nxdl_path}. " + "Please check this entry in the template dictionary." + ) from value_error return None if class_path and nxdlpath and elist: elem = elist[0] else: elem = None if exc: - raise NxdlAttributeError(f"Attributes were not found for {nxdl_path}. " - "Please check this entry in the template dictionary.") + raise NxdlAttributeError( + f"Attributes were not found for {nxdl_path}. " + "Please check this entry in the template dictionary." + ) return elem def process_node(hdf_node, hdf_path, parser, logger, doc=True): """Processes an hdf5 node. -- it logs the node found and also checks for its attributes -- retrieves the corresponding nxdl documentation -TODO: -- follow variants -- NOMAD parser: store in NOMAD """ - hdf_info = {'hdf_path': hdf_path, 'hdf_node': hdf_node} + - it logs the node found and also checks for its attributes + - retrieves the corresponding nxdl documentation + TODO: + - follow variants + - NOMAD parser: store in NOMAD""" + hdf_info = {"hdf_path": hdf_path, "hdf_node": hdf_node} if isinstance(hdf_node, h5py.Dataset): - logger.debug(f'===== FIELD (/{hdf_path}): {hdf_node}') - val = str(hdf_node[()]).split('\n') if len(hdf_node.shape) <= 1 else str( - hdf_node[0]).split('\n') + logger.debug(f"===== FIELD (/{hdf_path}): {hdf_node}") + val = ( + str(hdf_node[()]).split("\n") + if len(hdf_node.shape) <= 1 + else str(hdf_node[0]).split("\n") + ) logger.debug(f'value: {val[0]} {"..." if len(val) > 1 else ""}') else: logger.debug( @@ -1010,46 +1184,54 @@ def process_node(hdf_node, hdf_path, parser, logger, doc=True): ) (req_str, nxdef, nxdl_path) = get_nxdl_doc(hdf_info, logger, doc) if parser is not None and isinstance(hdf_node, h5py.Dataset): - parser({"hdf_info": hdf_info, + parser( + { + "hdf_info": hdf_info, "nxdef": nxdef, "nxdl_path": nxdl_path, "val": val, - "logger": logger}) + "logger": logger, + } + ) for key, value in hdf_node.attrs.items(): - logger.debug(f'===== ATTRS (/{hdf_path}@{key})') - val = str(value).split('\n') + logger.debug(f"===== ATTRS (/{hdf_path}@{key})") + val = str(value).split("\n") logger.debug(f'value: {val[0]} {"..." if len(val) > 1 else ""}') - (req_str, nxdef, nxdl_path) = \ - get_nxdl_doc(hdf_info, logger, doc, attr=key) + (req_str, nxdef, nxdl_path) = get_nxdl_doc(hdf_info, logger, doc, attr=key) if ( parser is not None and req_str is not None - and 'NOT IN SCHEMA' not in req_str - and 'None' not in req_str + and "NOT IN SCHEMA" not in req_str + and "None" not in req_str ): - parser({"hdf_info": hdf_info, + parser( + { + "hdf_info": hdf_info, "nxdef": nxdef, "nxdl_path": nxdl_path, "val": val, - "logger": logger}, attr=key) + "logger": logger, + }, + attr=key, + ) def logger_auxiliary_signal(logger, nxdata): """Handle the presence of auxiliary signal""" - aux = nxdata.attrs.get('auxiliary_signals') + aux = nxdata.attrs.get("auxiliary_signals") if aux is not None: if isinstance(aux, str): aux = [aux] for asig in aux: - logger.debug(f'Further auxiliary signal has been identified: {asig}') + logger.debug(f"Further auxiliary signal has been identified: {asig}") return logger def print_default_plotable_header(logger): """Print a three-lines header""" - logger.debug('========================') - logger.debug('=== Default Plotable ===') - logger.debug('========================') + logger.debug("========================") + logger.debug("=== Default Plotable ===") + logger.debug("========================") def get_default_plotable(root, logger): @@ -1067,10 +1249,10 @@ def get_default_plotable(root, logger): if not nxentry: nxentry = entry_helper(root) if not nxentry: - logger.debug('No NXentry has been found') + logger.debug("No NXentry has been found") return - logger.debug('') - logger.debug('NXentry has been identified: ' + nxentry.name) + logger.debug("") + logger.debug("NXentry has been identified: " + nxentry.name) # nxdata nxdata = None nxgroup = nxentry @@ -1086,10 +1268,10 @@ def get_default_plotable(root, logger): else: nxdata = nxgroup if not nxdata: - logger.debug('No NXdata group has been found') + logger.debug("No NXdata group has been found") return - logger.debug('') - logger.debug('NXdata group has been identified: ' + nxdata.name) + logger.debug("") + logger.debug("NXdata group has been identified: " + nxdata.name) process_node(nxdata, nxdata.name, None, logger, False) # signal signal = None @@ -1101,10 +1283,10 @@ def get_default_plotable(root, logger): if not signal: signal = signal_helper(nxdata) if not signal: - logger.debug('No Signal has been found') + logger.debug("No Signal has been found") return - logger.debug('') - logger.debug('Signal has been identified: ' + signal.name) + logger.debug("") + logger.debug("Signal has been identified: " + signal.name) process_node(signal, signal.name, None, logger, False) logger = logger_auxiliary_signal(logger, nxdata) # check auxiliary_signals dim = len(signal.shape) @@ -1116,8 +1298,11 @@ def entry_helper(root): """Check entry related data""" nxentries = [] for key in root.keys(): - if isinstance(root[key], h5py.Group) and root[key].attrs.get('NX_class') and \ - root[key].attrs['NX_class'] == "NXentry": + if ( + isinstance(root[key], h5py.Group) + and root[key].attrs.get("NX_class") + and root[key].attrs["NX_class"] == "NXentry" + ): nxentries.append(root[key]) if len(nxentries) >= 1: return nxentries[0] @@ -1126,11 +1311,14 @@ def entry_helper(root): def nxdata_helper(nxentry): """Check if nxentry hdf5 object has a NX_class and, if it contains NXdata, -return its value""" + return its value""" lnxdata = [] for key in nxentry.keys(): - if isinstance(nxentry[key], h5py.Group) and nxentry[key].attrs.get('NX_class') and \ - nxentry[key].attrs['NX_class'] == "NXdata": + if ( + isinstance(nxentry[key], h5py.Group) + and nxentry[key].attrs.get("NX_class") + and nxentry[key].attrs["NX_class"] == "NXdata" + ): lnxdata.append(nxentry[key]) if len(lnxdata) >= 1: return lnxdata[0] @@ -1143,12 +1331,17 @@ def signal_helper(nxdata): for key in nxdata.keys(): if isinstance(nxdata[key], h5py.Dataset): signals.append(nxdata[key]) - if len(signals) == 1: # v3: as there was no selection given, only 1 data field shall exists + if ( + len(signals) == 1 + ): # v3: as there was no selection given, only 1 data field shall exists return signals[0] if len(signals) > 1: # v2: select the one with an attribute signal="1" attribute for sig in signals: - if sig.attrs.get("signal") and sig.attrs.get("signal") is str and \ - sig.attrs.get("signal") == "1": + if ( + sig.attrs.get("signal") + and sig.attrs.get("signal") is str + and sig.attrs.get("signal") == "1" + ): return sig return None @@ -1160,7 +1353,7 @@ def find_attrib_axis_actual_dim_num(nxdata, a_item, ax_list): for key in nxdata.keys(): if isinstance(nxdata[key], h5py.Dataset): try: - if nxdata[key].attrs['axis'] == a_item + 1: + if nxdata[key].attrs["axis"] == a_item + 1: lax.append(nxdata[key]) except KeyError: pass @@ -1169,7 +1362,7 @@ def find_attrib_axis_actual_dim_num(nxdata, a_item, ax_list): # if there are more alternatives, prioritise the one with an attribute primary="1" elif len(lax) > 1: for sax in lax: - if sax.attrs.get('primary') and sax.attrs.get('primary') == 1: + if sax.attrs.get("primary") and sax.attrs.get("primary") == 1: ax_list.insert(0, sax) else: ax_list.append(sax) @@ -1180,7 +1373,7 @@ def get_single_or_multiple_axes(nxdata, ax_datasets, a_item, ax_list): try: if isinstance(ax_datasets, str): # single axis is defined # explicite definition of dimension number - ind = nxdata.attrs.get(ax_datasets + '_indices') + ind = nxdata.attrs.get(ax_datasets + "_indices") if ind and ind is int: if ind == a_item: ax_list.append(nxdata[ax_datasets]) @@ -1189,7 +1382,7 @@ def get_single_or_multiple_axes(nxdata, ax_datasets, a_item, ax_list): else: # multiple axes are listed # explicite definition of dimension number for aax in ax_datasets: - ind = nxdata.attrs.get(aax + '_indices') + ind = nxdata.attrs.get(aax + "_indices") if ind and isinstance(ind, int): if ind == a_item: ax_list.append(nxdata[aax]) @@ -1207,22 +1400,25 @@ def axis_helper(dim, nxdata, signal, axes, logger): ax_datasets = nxdata.attrs.get("axes") # primary axes listed in attribute axes ax_list = get_single_or_multiple_axes(nxdata, ax_datasets, a_item, ax_list) for attr in nxdata.attrs.keys(): # check for corresponding AXISNAME_indices - if attr.endswith('_indices') and nxdata.attrs[attr] == a_item and \ - nxdata[attr.split('_indices')[0]] not in ax_list: - ax_list.append(nxdata[attr.split('_indices')[0]]) + if ( + attr.endswith("_indices") + and nxdata.attrs[attr] == a_item + and nxdata[attr.split("_indices")[0]] not in ax_list + ): + ax_list.append(nxdata[attr.split("_indices")[0]]) # v2 # check for ':' separated axes defined in Signal if not ax_list: try: - ax_datasets = signal.attrs.get("axes").split(':') + ax_datasets = signal.attrs.get("axes").split(":") ax_list.append(nxdata[ax_datasets[a_item]]) except (KeyError, AttributeError): pass if not ax_list: # check for axis/primary specifications find_attrib_axis_actual_dim_num(nxdata, a_item, ax_list) axes.append(ax_list) - logger.debug('') + logger.debug("") logger.debug( - f'For Axis #{a_item}, {len(ax_list)} axes have been identified: {str(ax_list)}' + f"For Axis #{a_item}, {len(ax_list)} axes have been identified: {str(ax_list)}" ) @@ -1230,38 +1426,43 @@ def get_all_is_a_rel_from_hdf_node(hdf_node, hdf_path): """Return list of nxdl concept paths for a nxdl element which corresponds to hdf node. """ - hdf_info = {'hdf_path': hdf_path, 'hdf_node': hdf_node} - (_, _, elist) = \ - get_inherited_nodes(None, nx_name=get_nxdl_entry(hdf_info), hdf_node=hdf_node, - hdf_path=hdf_info['hdf_path'] if 'hdf_path' in hdf_info else None, - hdf_root=hdf_info['hdf_root'] if 'hdf_root' in hdf_info else None) + hdf_info = {"hdf_path": hdf_path, "hdf_node": hdf_node} + (_, _, elist) = get_inherited_nodes( + None, + nx_name=get_nxdl_entry(hdf_info), + hdf_node=hdf_node, + hdf_path=hdf_info["hdf_path"] if "hdf_path" in hdf_info else None, + hdf_root=hdf_info["hdf_root"] if "hdf_root" in hdf_info else None, + ) return elist def hdf_node_to_self_concept_path(hdf_info, logger): - """ Get concept or nxdl path from given hdf_node. - """ + """Get concept or nxdl path from given hdf_node.""" # The bellow logger is for deactivatine unnecessary debug message above if logger is None: logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) (_, _, nxdl_path) = get_nxdl_doc(hdf_info, logger, None) - con_path = '' + con_path = "" if nxdl_path: for nd_ in nxdl_path: - con_path = con_path + '/' + get_node_name(nd_) + con_path = con_path + "/" + get_node_name(nd_) return con_path class HandleNexus: """documentation""" - def __init__(self, logger, nexus_file, - d_inq_nd=None, c_inq_nd=None): + + def __init__(self, logger, nexus_file, d_inq_nd=None, c_inq_nd=None): self.logger = logger local_dir = os.path.abspath(os.path.dirname(__file__)) - self.input_file_name = nexus_file if nexus_file is not None else \ - os.path.join(local_dir, '../../tests/data/nexus/201805_WSe2_arpes.nxs') + self.input_file_name = ( + nexus_file + if nexus_file is not None + else os.path.join(local_dir, "../../tests/data/nexus/201805_WSe2_arpes.nxs") + ) self.parser = None self.in_file = None self.d_inq_nd = d_inq_nd @@ -1273,46 +1474,55 @@ def visit_node(self, hdf_name, hdf_node): """Function called by h5py that iterates on each node of hdf5file. It allows h5py visititems function to visit nodes.""" if self.d_inq_nd is None and self.c_inq_nd is None: - process_node(hdf_node, '/' + hdf_name, self.parser, self.logger) - elif (self.d_inq_nd is not None - and hdf_name in (self.d_inq_nd, self.d_inq_nd[1:])): - process_node(hdf_node, '/' + hdf_name, self.parser, self.logger) + process_node(hdf_node, "/" + hdf_name, self.parser, self.logger) + elif self.d_inq_nd is not None and hdf_name in ( + self.d_inq_nd, + self.d_inq_nd[1:], + ): + process_node(hdf_node, "/" + hdf_name, self.parser, self.logger) elif self.c_inq_nd is not None: - attributed_concept = self.c_inq_nd.split('@') + attributed_concept = self.c_inq_nd.split("@") attr = attributed_concept[1] if len(attributed_concept) > 1 else None - elist = get_all_is_a_rel_from_hdf_node(hdf_node, '/' + hdf_name) + elist = get_all_is_a_rel_from_hdf_node(hdf_node, "/" + hdf_name) if elist is None: return fnd_superclass = False fnd_superclass_attr = False for elem in reversed(elist): - tmp_path = elem.get('nxdlbase').split('.nxdl')[0] - con_path = '/NX' + tmp_path.split('NX')[-1] + elem.get('nxdlpath') + tmp_path = elem.get("nxdlbase").split(".nxdl")[0] + con_path = "/NX" + tmp_path.split("NX")[-1] + elem.get("nxdlpath") if fnd_superclass or con_path == attributed_concept[0]: fnd_superclass = True if attr is None: self.hdf_path_list_for_c_inq_nd.append(hdf_name) break for attribute in hdf_node.attrs.keys(): - attr_concept = get_nxdl_child(elem, attribute, nexus_type='attribute', - go_base=False) - if attr_concept is not None and \ - attr_concept.get('nxdlpath').endswith(attr): + attr_concept = get_nxdl_child( + elem, attribute, nexus_type="attribute", go_base=False + ) + if attr_concept is not None and attr_concept.get( + "nxdlpath" + ).endswith(attr): fnd_superclass_attr = True - con_path = '/NX' + tmp_path.split('NX')[-1] \ - + attr_concept.get('nxdlpath') - self.hdf_path_list_for_c_inq_nd.append(hdf_name + "@" + attribute) + con_path = ( + "/NX" + + tmp_path.split("NX")[-1] + + attr_concept.get("nxdlpath") + ) + self.hdf_path_list_for_c_inq_nd.append( + hdf_name + "@" + attribute + ) break if fnd_superclass_attr: break def not_yet_visited(self, root, name): """checking if a new node has already been visited in its path""" - path = name.split('/') + path = name.split("/") for i in range(1, len(path)): - act_path = '/'.join(path[:i]) + act_path = "/".join(path[:i]) # print(act_path+' - '+name) - if root['/' + act_path] == root['/' + name]: + if root["/" + act_path] == root["/" + name]: return False return True @@ -1323,7 +1533,7 @@ def full_visit(self, root, hdf_node, name, func): func(name, hdf_node) if isinstance(hdf_node, h5py.Group): for ch_name, child in hdf_node.items(): - full_name = ch_name if len(name) == 0 else name + '/' + ch_name + full_name = ch_name if len(name) == 0 else name + "/" + ch_name if self.not_yet_visited(root, full_name): self.full_visit(root, child, full_name, func) @@ -1333,9 +1543,10 @@ def process_nexus_master_file(self, parser): self.in_file = h5py.File( self.input_file_name[0] if isinstance(self.input_file_name, list) - else self.input_file_name, 'r' + else self.input_file_name, + "r", ) - self.full_visit(self.in_file, self.in_file, '', self.visit_node) + self.full_visit(self.in_file, self.in_file, "", self.visit_node) if self.d_inq_nd is None and self.c_inq_nd is None: get_default_plotable(self.in_file, self.logger) # To log the provided concept and concepts founded @@ -1347,48 +1558,58 @@ def process_nexus_master_file(self, parser): @click.command() @click.option( - '-f', - '--nexus-file', + "-f", + "--nexus-file", required=False, default=None, - help=('NeXus file with extension .nxs to learn NeXus different concept' - ' documentation and concept.') + help=( + "NeXus file with extension .nxs to learn NeXus different concept" + " documentation and concept." + ), ) @click.option( - '-d', - '--documentation', + "-d", + "--documentation", required=False, default=None, - help=("Definition path in nexus output (.nxs) file. Returns debug" - "log relavent with that definition path. Example: /entry/data/delays") + help=( + "Definition path in nexus output (.nxs) file. Returns debug" + "log relavent with that definition path. Example: /entry/data/delays" + ), ) @click.option( - '-c', - '--concept', + "-c", + "--concept", required=False, default=None, - help=("Concept path from application definition file (.nxdl,xml). Finds out" - "all the available concept definition (IS-A realation) for rendered" - "concept path. Example: /NXarpes/ENTRY/INSTRUMENT/analyser") + help=( + "Concept path from application definition file (.nxdl,xml). Finds out" + "all the available concept definition (IS-A realation) for rendered" + "concept path. Example: /NXarpes/ENTRY/INSTRUMENT/analyser" + ), ) def main(nexus_file, documentation, concept): """The main function to call when used as a script.""" logging_format = "%(levelname)s: %(message)s" stdout_handler = logging.StreamHandler(sys.stdout) stdout_handler.setLevel(logging.DEBUG) - logging.basicConfig(level=logging.INFO, format=logging_format, handlers=[stdout_handler]) + logging.basicConfig( + level=logging.INFO, format=logging_format, handlers=[stdout_handler] + ) logger = logging.getLogger(__name__) logger.addHandler(stdout_handler) logger.setLevel(logging.DEBUG) logger.propagate = False if documentation and concept: - raise ValueError("Only one option either documentation (-d) or is_a relation " - "with a concept (-c) can be requested.") - nexus_helper = HandleNexus(logger, nexus_file, - d_inq_nd=documentation, - c_inq_nd=concept) + raise ValueError( + "Only one option either documentation (-d) or is_a relation " + "with a concept (-c) can be requested." + ) + nexus_helper = HandleNexus( + logger, nexus_file, d_inq_nd=documentation, c_inq_nd=concept + ) nexus_helper.process_nexus_master_file(None) -if __name__ == '__main__': +if __name__ == "__main__": main() # pylint: disable=no-value-for-parameter From 5a1de22aba651428ed2b6623e401fea9186c0e93 Mon Sep 17 00:00:00 2001 From: Sandor Brockhauser Date: Fri, 16 Jun 2023 00:48:53 +0200 Subject: [PATCH 29/93] linting --- dev_tools/docs/nxdl.py | 2 +- dev_tools/utils/nexus.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/dev_tools/docs/nxdl.py b/dev_tools/docs/nxdl.py index 1316e230ac..03c72be21d 100644 --- a/dev_tools/docs/nxdl.py +++ b/dev_tools/docs/nxdl.py @@ -7,12 +7,12 @@ from typing import Optional import lxml -from ..utils import nexus as pynxtools_nxlib from ..globals.directories import get_nxdl_root from ..globals.errors import NXDLParseError from ..globals.nxdl import NXDL_NAMESPACE from ..globals.urls import REPO_URL +from ..utils import nexus as pynxtools_nxlib from ..utils.types import PathLike from .anchor_list import AnchorRegistry diff --git a/dev_tools/utils/nexus.py b/dev_tools/utils/nexus.py index 7b09e30f33..def84b1605 100644 --- a/dev_tools/utils/nexus.py +++ b/dev_tools/utils/nexus.py @@ -2,15 +2,16 @@ """Read files from different format and print it in a standard NeXus format """ +import logging import os +import sys +import textwrap import xml.etree.ElementTree as ET from functools import lru_cache from glob import glob -import sys -import logging -import textwrap -import h5py + import click +import h5py class NxdlAttributeError(Exception): From 286c0c2e6262686ec87311cf299b8b49ae07b035 Mon Sep 17 00:00:00 2001 From: Sandor Brockhauser Date: Fri, 16 Jun 2023 00:54:10 +0200 Subject: [PATCH 30/93] imports --- requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements.txt b/requirements.txt index 6d024bda3a..ac0c657370 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,8 @@ # Prepare for Documentation lxml pyyaml +click>=7.1.2 +h5py>=3.6.0 # Documentation building sphinx>=5 From 6837fb6b1847a90e99979d0190dacb58078585d7 Mon Sep 17 00:00:00 2001 From: domna Date: Fri, 16 Jun 2023 10:01:05 +0200 Subject: [PATCH 31/93] Adds pyproject --- .gitignore | 20 +++++++++++++++++++ MANIFEST.in | 4 ++++ dev_tools/utils/nexus.py | 21 ++++++-------------- pyproject.toml | 43 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 73 insertions(+), 15 deletions(-) create mode 100644 MANIFEST.in create mode 100644 pyproject.toml diff --git a/.gitignore b/.gitignore index ff21c1627c..50408db4bd 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,23 @@ makelog.txt # Unknown /python/ __github_creds__.txt + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000000..20485f6286 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,4 @@ +recursive-include applications/ *.nxdl.xml +recursive-include contributed_definitions/ *.nxdl.xml +recursive-include base_classes/ *.nxdl.xml +include ./ *.xsd \ No newline at end of file diff --git a/dev_tools/utils/nexus.py b/dev_tools/utils/nexus.py index def84b1605..8e8ef771fc 100644 --- a/dev_tools/utils/nexus.py +++ b/dev_tools/utils/nexus.py @@ -1457,13 +1457,11 @@ class HandleNexus: def __init__(self, logger, nexus_file, d_inq_nd=None, c_inq_nd=None): self.logger = logger - local_dir = os.path.abspath(os.path.dirname(__file__)) - self.input_file_name = ( - nexus_file - if nexus_file is not None - else os.path.join(local_dir, "../../tests/data/nexus/201805_WSe2_arpes.nxs") - ) + if nexus_file is None: + raise ValueError("Nexus file not specified. Cannot proceed.") + + self.input_file_name = nexus_file self.parser = None self.in_file = None self.d_inq_nd = d_inq_nd @@ -1558,15 +1556,8 @@ def process_nexus_master_file(self, parser): @click.command() -@click.option( - "-f", - "--nexus-file", - required=False, - default=None, - help=( - "NeXus file with extension .nxs to learn NeXus different concept" - " documentation and concept." - ), +@click.argument( + 'nexus_file', ) @click.option( "-d", diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000..6a90ec5732 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,43 @@ +[build-system] +requires = ["setuptools>=64.0.1", "setuptools-scm[toml]>=6.2"] +build-backend = "setuptools.build_meta" + +[project] +name = "nexusdefinitions" +dynamic = ["version"] +authors = [ + { name = "NIAC" } +] +description = "Nexus definitions" +readme = "README.md" +license = { file = "LGPL.txt" } +requires-python = "" +classifiers = [ + "Operating System :: OS Independent" +] +dependencies = [ + "lxml", + "pyyaml", + "click>=7.1.2", + "h5py>=3.6.0", + "sphinx>=5", + "sphinx-tabs", + "pytest", + "black>=22.3", + "flake8>=4", + "isort>=5.10", + "click>=7.1.2", +] + +[project.urls] +"Homepage" = "https://nexusformat.org" + +[project.scripts] +read_nexus = "dev_tools.utils.nexus:main" + +[tools.setuptools_scm] +version_scheme = "guess-next-dev" +local_scheme = "node-and-date" + +[tool.setuptools] +packages = ["dev_tools"] From 52a21eea39df34f6ba1d3cefd8f1181c48a3fa7e Mon Sep 17 00:00:00 2001 From: Sandor Brockhauser Date: Fri, 16 Jun 2023 10:16:51 +0200 Subject: [PATCH 32/93] linting --- dev_tools/utils/nexus.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev_tools/utils/nexus.py b/dev_tools/utils/nexus.py index 8e8ef771fc..e797092d1f 100644 --- a/dev_tools/utils/nexus.py +++ b/dev_tools/utils/nexus.py @@ -1557,7 +1557,7 @@ def process_nexus_master_file(self, parser): @click.command() @click.argument( - 'nexus_file', + "nexus_file", ) @click.option( "-d", From d3d101f862572fa619ae11f90a703a49e885842d Mon Sep 17 00:00:00 2001 From: Sandor Brockhauser Date: Fri, 16 Jun 2023 10:58:21 +0200 Subject: [PATCH 33/93] adjusted default location of definitions inside the module --- Makefile | 1 - dev_tools/utils/nexus.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 44c076c341..ae556d7339 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,6 @@ PYTHON = python3 SPHINX = sphinx-build BUILD_DIR = "build" -export NEXUS_DEF_PATH = $(shell pwd) .PHONY: help install style autoformat test clean prepare html pdf impatient-guide all local diff --git a/dev_tools/utils/nexus.py b/dev_tools/utils/nexus.py index e797092d1f..4bcb1c9e96 100644 --- a/dev_tools/utils/nexus.py +++ b/dev_tools/utils/nexus.py @@ -45,7 +45,7 @@ def get_nexus_definitions_path(): return os.environ["NEXUS_DEF_PATH"] except KeyError: # or it should be available locally under the dir 'definitions' local_dir = os.path.abspath(os.path.dirname(__file__)) - return os.path.join(local_dir, f"..{os.sep}definitions") + return os.path.join(local_dir, f"..{os.sep}..") def get_hdf_root(hdf_node): From 025c0786858e80d416baf90c36b0d8dd100fb8d8 Mon Sep 17 00:00:00 2001 From: Sandor Brockhauser Date: Fri, 16 Jun 2023 11:59:32 +0200 Subject: [PATCH 34/93] new characters as Code Camp suggested --- dev_tools/docs/nxdl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev_tools/docs/nxdl.py b/dev_tools/docs/nxdl.py index 03c72be21d..c8c6936410 100644 --- a/dev_tools/docs/nxdl.py +++ b/dev_tools/docs/nxdl.py @@ -668,7 +668,7 @@ def get_first_parent_ref(self, path, tag): ) parent_display_name = f"{parent_def_name[1:]}{parent_path}" return ( - f":abbr:`... (override: {parent_display_name})" - + f"`:ref:`🔗 `" + f":abbr:`⤆ (override: {parent_display_name})" + + f"`:ref:`... `" ) return "" From adf098ed87b9e03585a30c7f3094fed989d35fa3 Mon Sep 17 00:00:00 2001 From: Sandor Brockhauser Date: Fri, 16 Jun 2023 12:08:40 +0200 Subject: [PATCH 35/93] make new char available for latex --- manual/source/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/manual/source/conf.py b/manual/source/conf.py index 346e664d95..638a665591 100644 --- a/manual/source/conf.py +++ b/manual/source/conf.py @@ -97,4 +97,5 @@ 'preamble': r''' \usepackage{amsbsy} \DeclareUnicodeCharacter{1F517}{X}''' + \DeclareUnicodeCharacter{2906}{<-}''' } From 222a3c0a09fbabcd722061c763838f7464a14221 Mon Sep 17 00:00:00 2001 From: Sandor Brockhauser Date: Fri, 16 Jun 2023 12:17:08 +0200 Subject: [PATCH 36/93] make new char available for latex --- manual/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manual/source/conf.py b/manual/source/conf.py index 638a665591..6ee889c54a 100644 --- a/manual/source/conf.py +++ b/manual/source/conf.py @@ -96,6 +96,6 @@ 'maxlistdepth':7, # some application definitions are deeply nested 'preamble': r''' \usepackage{amsbsy} - \DeclareUnicodeCharacter{1F517}{X}''' + \DeclareUnicodeCharacter{1F517}{X} \DeclareUnicodeCharacter{2906}{<-}''' } From 7252a4983fdd935c1ef4df4c3c3e038cb71ad4b1 Mon Sep 17 00:00:00 2001 From: Sandor Brockhauser Date: Fri, 16 Jun 2023 12:18:08 +0200 Subject: [PATCH 37/93] make new char available for latex --- manual/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manual/source/conf.py b/manual/source/conf.py index 6ee889c54a..dcd444c8ee 100644 --- a/manual/source/conf.py +++ b/manual/source/conf.py @@ -97,5 +97,5 @@ 'preamble': r''' \usepackage{amsbsy} \DeclareUnicodeCharacter{1F517}{X} - \DeclareUnicodeCharacter{2906}{<-}''' + \DeclareUnicodeCharacter{2906}{<=}''' } From fd4b4a6f07b5dae23469dcd0bc68b3dccfc1ea52 Mon Sep 17 00:00:00 2001 From: Sandor Brockhauser Date: Fri, 16 Jun 2023 17:27:11 +0200 Subject: [PATCH 38/93] collapsing doc_enum-s --- dev_tools/docs/nxdl.py | 55 +++++++++++++++++++++++++++--------------- manual/source/conf.py | 1 + 2 files changed, 37 insertions(+), 19 deletions(-) diff --git a/dev_tools/docs/nxdl.py b/dev_tools/docs/nxdl.py index c8c6936410..815e8b7774 100644 --- a/dev_tools/docs/nxdl.py +++ b/dev_tools/docs/nxdl.py @@ -110,7 +110,7 @@ def _parse_nxdl_file(self, nxdl_file: Path): # print official description of this class self._print("") self._print("**Description**:\n") - self._print_doc(self._INDENTATION_UNIT, ns, root, required=True) + self._print_doc_enum("", ns, root, required=True) # print symbol list node_list = root.xpath("nx:symbols", namespaces=ns) @@ -120,7 +120,7 @@ def _parse_nxdl_file(self, nxdl_file: Path): elif len(node_list) > 1: raise Exception(f"Invalid symbol table in {nxclass_name}") else: - self._print_doc(self._INDENTATION_UNIT, ns, node_list[0]) + self._print_doc_enum("", ns, node_list[0]) for node in node_list[0].xpath("nx:symbol", namespaces=ns): doc = self._get_doc_line(ns, node) self._print(f" **{node.get('name')}**", end="") @@ -499,6 +499,35 @@ def _print_doc(self, indent, ns, node, required=False): self._print(f"{indent}{line}") self._print() + def long_doc(self, ns, node): + length = 0 + line = "documentation" + fnd = False + blocks = self._get_doc_blocks(ns, node) + for block in blocks: + lines = block.splitlines() + length += len(lines) + for single_line in lines: + if len(single_line) > 2 and single_line[0] != "." and not fnd: + fnd = True + line = single_line + return (length, line, blocks) + + def _print_doc_enum(self, indent, ns, node, required=False): + collapse_indent = indent + node_list = node.xpath("nx:enumeration", namespaces=ns) + (doclen, line, blocks) = self.long_doc(ns, node) + if len(node_list) + doclen > 1: + collapse_indent = f"{indent} " + self._print(f"{indent}{self._INDENTATION_UNIT}.. collapse:: {line} ...\n") + self._print_doc( + collapse_indent + self._INDENTATION_UNIT, ns, node, required=required + ) + if len(node_list) == 1: + self._print_enumeration( + collapse_indent + self._INDENTATION_UNIT, ns, node_list[0] + ) + def _print_attribute(self, ns, kind, node, optional, indent, parent_path): name = node.get("name") index_name = name @@ -509,10 +538,7 @@ def _print_attribute(self, ns, kind, node, optional, indent, parent_path): self._print( f"{indent}**@{name}**: {optional}{self._format_type(node)}{self._format_units(node)} {self.get_first_parent_ref(f'{parent_path}/{name}', 'attribute')}\n" ) - self._print_doc(indent + self._INDENTATION_UNIT, ns, node) - node_list = node.xpath("nx:enumeration", namespaces=ns) - if len(node_list) == 1: - self._print_enumeration(indent + self._INDENTATION_UNIT, ns, node_list[0]) + self._print_doc_enum(indent, ns, node) def _print_if_deprecated(self, ns, node, indent): deprecated = node.get("deprecated", None) @@ -555,13 +581,7 @@ def _print_full_tree(self, ns, parent, name, indent, parent_path): ) self._print_if_deprecated(ns, node, indent + self._INDENTATION_UNIT) - self._print_doc(indent + self._INDENTATION_UNIT, ns, node) - - node_list = node.xpath("nx:enumeration", namespaces=ns) - if len(node_list) == 1: - self._print_enumeration( - indent + self._INDENTATION_UNIT, ns, node_list[0] - ) + self._print_doc_enum(indent, ns, node) for subnode in node.xpath("nx:attribute", namespaces=ns): optional = self._get_required_or_optional_text(subnode) @@ -592,7 +612,7 @@ def _print_full_tree(self, ns, parent, name, indent, parent_path): ) self._print_if_deprecated(ns, node, indent + self._INDENTATION_UNIT) - self._print_doc(indent + self._INDENTATION_UNIT, ns, node) + self._print_doc_enum(indent, ns, node) for subnode in node.xpath("nx:attribute", namespaces=ns): optional = self._get_required_or_optional_text(subnode) @@ -623,7 +643,7 @@ def _print_full_tree(self, ns, parent, name, indent, parent_path): f"(suggested target: ``{node.get('target')}``)" "\n" ) - self._print_doc(indent + self._INDENTATION_UNIT, ns, node) + self._print_doc_enum(indent, ns, node) def _print(self, *args, end="\n"): # TODO: change instances of \t to proper indentation @@ -667,8 +687,5 @@ def get_first_parent_ref(self, path, tag): + parent_path[pos_of_right_slash + 1 :] ) parent_display_name = f"{parent_def_name[1:]}{parent_path}" - return ( - f":abbr:`⤆ (override: {parent_display_name})" - + f"`:ref:`... `" - ) + return f":ref:`⤆ `" return "" diff --git a/manual/source/conf.py b/manual/source/conf.py index dcd444c8ee..a1f854be4b 100644 --- a/manual/source/conf.py +++ b/manual/source/conf.py @@ -42,6 +42,7 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ + 'sphinx_toolbox.collapse', 'sphinx.ext.mathjax', 'sphinx.ext.ifconfig', 'sphinx.ext.viewcode', From 90ff26b8aa6e37552b298b8f3a70e887d3f7afa6 Mon Sep 17 00:00:00 2001 From: Sandor Brockhauser Date: Fri, 16 Jun 2023 17:32:11 +0200 Subject: [PATCH 39/93] missing sphinx dependency --- pyproject.toml | 1 - requirements.txt | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 6a90ec5732..d4cf990c86 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,6 @@ dependencies = [ "black>=22.3", "flake8>=4", "isort>=5.10", - "click>=7.1.2", ] [project.urls] diff --git a/requirements.txt b/requirements.txt index ac0c657370..bbfd892f76 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,6 +7,7 @@ h5py>=3.6.0 # Documentation building sphinx>=5 sphinx-tabs +sphinx-toolbox # Testing pytest From 015aa7761b1814e2dadb4f30aebe0ac39ef04b32 Mon Sep 17 00:00:00 2001 From: Sandor Brockhauser Date: Wed, 21 Jun 2023 12:40:56 +0200 Subject: [PATCH 40/93] removing h5py dependency --- dev_tools/docs/nxdl.py | 2 +- dev_tools/utils/{nexus.py => nxdl_utils.py} | 761 +------------------- 2 files changed, 5 insertions(+), 758 deletions(-) rename dev_tools/utils/{nexus.py => nxdl_utils.py} (51%) diff --git a/dev_tools/docs/nxdl.py b/dev_tools/docs/nxdl.py index 815e8b7774..4c36ef2775 100644 --- a/dev_tools/docs/nxdl.py +++ b/dev_tools/docs/nxdl.py @@ -12,7 +12,7 @@ from ..globals.errors import NXDLParseError from ..globals.nxdl import NXDL_NAMESPACE from ..globals.urls import REPO_URL -from ..utils import nexus as pynxtools_nxlib +from ..utils import nxdl_utils as pynxtools_nxlib from ..utils.types import PathLike from .anchor_list import AnchorRegistry diff --git a/dev_tools/utils/nexus.py b/dev_tools/utils/nxdl_utils.py similarity index 51% rename from dev_tools/utils/nexus.py rename to dev_tools/utils/nxdl_utils.py index 4bcb1c9e96..efba439bec 100644 --- a/dev_tools/utils/nexus.py +++ b/dev_tools/utils/nxdl_utils.py @@ -1,18 +1,13 @@ # pylint: disable=too-many-lines -"""Read files from different format and print it in a standard NeXus format +"""Parse NeXus definition files """ -import logging import os -import sys import textwrap import xml.etree.ElementTree as ET from functools import lru_cache from glob import glob -import click -import h5py - class NxdlAttributeError(Exception): """An exception for throwing an error when an Nxdl attribute is not found.""" @@ -89,49 +84,6 @@ def get_hdf_info_parent(hdf_info): return {"hdf_node": node, "hdf_path": get_parent_path(hdf_info["hdf_path"])} -def get_nx_class_path(hdf_info): - """Get the full path of an HDF5 node using nexus classes - in case of a field, end with the field name""" - hdf_node = hdf_info["hdf_node"] - if hdf_node.name == "/": - return "" - if isinstance(hdf_node, h5py.Group): - return ( - get_nx_class_path(get_hdf_info_parent(hdf_info)) - + "/" - + ( - hdf_node.attrs["NX_class"] - if "NX_class" in hdf_node.attrs.keys() - else hdf_node.name.split("/")[-1] - ) - ) - if isinstance(hdf_node, h5py.Dataset): - return ( - get_nx_class_path(get_hdf_info_parent(hdf_info)) - + "/" - + hdf_node.name.split("/")[-1] - ) - return "" - - -def get_nxdl_entry(hdf_info): - """Get the nxdl application definition for an HDF5 node""" - entry = hdf_info - while ( - isinstance(entry["hdf_node"], h5py.Dataset) - or "NX_class" not in entry["hdf_node"].attrs.keys() - or entry["hdf_node"].attrs["NX_class"] != "NXentry" - ): - entry = get_hdf_info_parent(entry) - if entry["hdf_node"].name == "/": - return "NO NXentry found" - try: - nxdef = entry["hdf_node"]["definition"][()] - return nxdef.decode() - except KeyError: # 'NO Definition referenced' - return "NXentry" - - def get_nx_class(nxdl_elem): """Get the nexus class for a NXDL node""" if "category" in nxdl_elem.attrib.keys(): @@ -476,65 +428,6 @@ def get_required_string(nxdl_elem): return "<>" -def chk_nxdataaxis_v2(hdf_node, name, logger): - """Check if dataset is an axis""" - own_signal = hdf_node.attrs.get("signal") # check for being a Signal - if own_signal is str and own_signal == "1": - logger.debug("Dataset referenced (v2) as NXdata SIGNAL") - own_axes = hdf_node.attrs.get("axes") # check for being an axis - if own_axes is str: - axes = own_axes.split(":") - for i in len(axes): - if axes[i] and name == axes[i]: - logger.debug("Dataset referenced (v2) as NXdata AXIS #%d", i) - return None - ownpaxis = hdf_node.attrs.get("primary") - own_axis = hdf_node.attrs.get("axis") - if own_axis is int: - # also convention v1 - if ownpaxis is int and ownpaxis == 1: - logger.debug("Dataset referenced (v2) as NXdata AXIS #%d", own_axis - 1) - else: - logger.debug( - "Dataset referenced (v2) as NXdata (primary/alternative) AXIS #%d", - own_axis - 1, - ) - return None - - -def chk_nxdataaxis(hdf_node, name, logger): - """NEXUS Data Plotting Standard v3: new version from 2014""" - if not isinstance( - hdf_node, h5py.Dataset - ): # check if it is a field in an NXdata node - return None - parent = hdf_node.parent - if not parent or (parent and not parent.attrs.get("NX_class") == "NXdata"): - return None - signal = parent.attrs.get("signal") # chk for Signal - if signal and name == signal: - logger.debug("Dataset referenced as NXdata SIGNAL") - return None - axes = parent.attrs.get("axes") # check for default Axes - if axes is str: - if name == axes: - logger.debug("Dataset referenced as NXdata AXIS") - return None - elif axes is not None: - for i, j in enumerate(axes): - if name == j: - indices = parent.attrs.get(j + "_indices") - if indices is int: - logger.debug(f"Dataset referenced as NXdata AXIS #{indices}") - else: - logger.debug(f"Dataset referenced as NXdata AXIS #{i}") - return None - indices = parent.attrs.get(name + "_indices") # check for alternative Axes - if indices is int: - logger.debug(f"Dataset referenced as NXdata alternative AXIS #{indices}") - return chk_nxdataaxis_v2(hdf_node, name, logger) # check for older conventions - - # below there are some functions used in get_nxdl_doc function: def write_doc_string(logger, doc, attr): """Simple function that prints a line in the logger if doc exists""" @@ -670,150 +563,11 @@ def other_attrs( return logger, elem, nxdl_path, doc, attr -def check_deprecation_enum_axis(variables, doc, elist, attr, hdf_node): - """Check for several attributes. - deprecation - enums - nxdataaxis""" - logger, elem, path = variables - dep_str = elem.attrib.get("deprecated") # check for deprecation - if dep_str: - if doc: - logger.debug("DEPRECATED - " + dep_str) - for base_elem in elist if not attr else [elem]: # check for enums - sdoc = get_nxdl_child(base_elem, "enumeration", go_base=False) - if sdoc is not None: - if doc: - logger.debug("enumeration (" + get_node_concept_path(base_elem) + "):") - for item in sdoc: - if get_local_name_from_xml(item) == "item": - if doc: - logger.debug("-> " + item.attrib["value"]) - chk_nxdataaxis( - hdf_node, path.split("/")[-1], logger - ) # look for NXdata reference (axes/signal) - for base_elem in elist if not attr else [elem]: # check for doc - sdoc = get_nxdl_child(base_elem, "doc", go_base=False) - if doc: - logger.debug("documentation (" + get_node_concept_path(base_elem) + "):") - logger.debug(sdoc.text if sdoc is not None else "") - return logger, elem, path, doc, elist, attr, hdf_node - - def get_node_concept_path(elem): """get the short version of nxdlbase:nxdlpath""" return str(elem.get("nxdlbase").split("/")[-1] + ":" + elem.get("nxdlpath")) -def get_nxdl_attr_doc( # pylint: disable=too-many-arguments,too-many-locals - elem, elist, attr, hdf_node, logger, doc, nxdl_path, req_str, path, hdf_info -): - """Get nxdl documentation for an attribute""" - new_elem = [] - old_elem = elem - for elem_index, act_elem1 in enumerate(elist): - act_elem = act_elem1 - # NX_class is a compulsory attribute for groups in a nexus file - # which should match the type of the corresponding NXDL element - if ( - attr == "NX_class" - and not isinstance(hdf_node, h5py.Dataset) - and elem_index == 0 - ): - elem = None - logger, doc, attr = write_doc_string(logger, doc, attr) - new_elem = elem - break - # units category is a compulsory attribute for any fields - if attr == "units" and isinstance(hdf_node, h5py.Dataset): - req_str = "<>" - logger, act_elem, nxdl_path, doc, attr = try_find_units( - logger, act_elem, nxdl_path, doc, attr - ) - # units for attributes can be given as ATTRIBUTENAME_units - elif attr.endswith("_units"): - logger, act_elem, nxdl_path, doc, attr, req_str = check_attr_name_nxdl( - (logger, act_elem, nxdl_path, doc, attr, req_str) - ) - # default is allowed for groups - elif attr == "default" and not isinstance(hdf_node, h5py.Dataset): - req_str = "<>" - # try to find if default is defined as a child of the NXDL element - act_elem = get_nxdl_child( - act_elem, attr, nexus_type="attribute", go_base=False - ) - logger, act_elem, nxdl_path, doc, attr = try_find_default( - logger, act_elem1, act_elem, nxdl_path, doc, attr - ) - else: # other attributes - act_elem = get_nxdl_child( - act_elem, attr, nexus_type="attribute", go_base=False - ) - if act_elem is not None: - logger, act_elem, nxdl_path, doc, attr = other_attrs( - logger, act_elem1, act_elem, nxdl_path, doc, attr - ) - if act_elem is not None: - new_elem.append(act_elem) - if req_str is None: - req_str = get_required_string(act_elem) # check for being required - if doc: - logger.debug(req_str) - variables = [logger, act_elem, path] - ( - logger, - elem, - path, - doc, - elist, - attr, - hdf_node, - ) = check_deprecation_enum_axis(variables, doc, elist, attr, hdf_node) - elem = old_elem - if req_str is None and doc: - if attr != "NX_class": - logger.debug("@" + attr + " - IS NOT IN SCHEMA") - logger.debug("") - return (req_str, get_nxdl_entry(hdf_info), nxdl_path) - - -def get_nxdl_doc(hdf_info, logger, doc, attr=False): - """Get nxdl documentation for an HDF5 node (or its attribute)""" - hdf_node = hdf_info["hdf_node"] - # new way: retrieve multiple inherited base classes - (class_path, nxdl_path, elist) = get_inherited_nodes( - None, - nx_name=get_nxdl_entry(hdf_info), - hdf_node=hdf_node, - hdf_path=hdf_info["hdf_path"] if "hdf_path" in hdf_info else None, - hdf_root=hdf_info["hdf_root"] if "hdf_root" in hdf_info else None, - ) - elem = elist[0] if class_path and elist else None - if doc: - logger.debug("classpath: " + str(class_path)) - logger.debug( - "NOT IN SCHEMA" - if elem is None - else "classes:\n" + "\n".join(get_node_concept_path(e) for e in elist) - ) - # old solution with a single elem instead of using elist - path = get_nx_class_path(hdf_info) - req_str = None - if elem is None: - if doc: - logger.debug("") - return ("None", None, None) - if attr: - return get_nxdl_attr_doc( - elem, elist, attr, hdf_node, logger, doc, nxdl_path, req_str, path, hdf_info - ) - req_str = get_required_string(elem) # check for being required - if doc: - logger.debug(req_str) - variables = [logger, elem, path] - logger, elem, path, doc, elist, attr, hdf_node = check_deprecation_enum_axis( - variables, doc, elist, attr, hdf_node - ) - return (req_str, get_nxdl_entry(hdf_info), nxdl_path) - - def get_doc(node, ntype, nxhtml, nxpath): """Get documentation""" # URL for html documentation @@ -1044,44 +798,11 @@ def walk_elist(elist, html_name): return elist, html_name -def helper_get_inherited_nodes(hdf_info2, elist, pind, attr): - """find the best fitting name in all children""" - hdf_path, hdf_node, hdf_class_path = hdf_info2 - hdf_name = hdf_path[pind] - hdf_class_name = hdf_class_path[pind] - if pind < len(hdf_path) - (2 if attr else 1): - act_nexus_type = "group" - elif pind == len(hdf_path) - 1 and attr: - act_nexus_type = "attribute" - else: - act_nexus_type = "field" if isinstance(hdf_node, h5py.Dataset) else "group" - # find the best fitting name in all children - bestfit = -1 - html_name = None - for ind in range(len(elist) - 1, -1, -1): - newelem, fit = get_best_child( - elist[ind], hdf_node, hdf_name, hdf_class_name, act_nexus_type - ) - if fit >= bestfit and newelem is not None: - html_name = get_node_name(newelem) - return hdf_path, hdf_node, hdf_class_path, elist, pind, attr, html_name - - -def get_hdf_path(hdf_info): - """Get the hdf_path from an hdf_info""" - if "hdf_path" in hdf_info: - return hdf_info["hdf_path"].split("/")[1:] - return hdf_info["hdf_node"].name.split("/")[1:] - - @lru_cache(maxsize=None) def get_inherited_nodes( nxdl_path: str = None, # pylint: disable=too-many-arguments,too-many-locals nx_name: str = None, elem: ET.Element = None, - hdf_node=None, - hdf_path=None, - hdf_root=None, attr=False, ): """Returns a list of ET.Element for the given path.""" @@ -1091,38 +812,10 @@ def get_inherited_nodes( nxdl_elem_path = [elist[0]] class_path = [] # type: ignore[var-annotated] - if hdf_node is not None: - hdf_info = {"hdf_node": hdf_node} - if hdf_path: - hdf_info["hdf_path"] = hdf_path - if hdf_root: - hdf_root["hdf_root"] = hdf_root - hdf_node = hdf_info["hdf_node"] - hdf_path = get_hdf_path(hdf_info) - hdf_class_path = get_nx_class_path(hdf_info).split("/")[1:] - if attr: - hdf_path.append(attr) - hdf_class_path.append(attr) - path = hdf_path - else: - html_path = nxdl_path.split("/")[1:] - path = html_path + html_path = nxdl_path.split("/")[1:] + path = html_path for pind in range(len(path)): - if hdf_node is not None: - hdf_info2 = [hdf_path, hdf_node, hdf_class_path] - [ - hdf_path, - hdf_node, - hdf_class_path, - elist, - pind, - attr, - html_name, - ] = helper_get_inherited_nodes(hdf_info2, elist, pind, attr) - if html_name is None: # return if NOT IN SCHEMA - return (class_path, nxdl_elem_path, None) - else: - html_name = html_path[pind] + html_name = html_path[pind] elist, html_name = walk_elist(elist, html_name) if elist: class_path.append(get_nx_class(elist[0])) @@ -1159,449 +852,3 @@ def get_node_at_nxdl_path( "Please check this entry in the template dictionary." ) return elem - - -def process_node(hdf_node, hdf_path, parser, logger, doc=True): - """Processes an hdf5 node. - - it logs the node found and also checks for its attributes - - retrieves the corresponding nxdl documentation - TODO: - - follow variants - - NOMAD parser: store in NOMAD""" - hdf_info = {"hdf_path": hdf_path, "hdf_node": hdf_node} - if isinstance(hdf_node, h5py.Dataset): - logger.debug(f"===== FIELD (/{hdf_path}): {hdf_node}") - val = ( - str(hdf_node[()]).split("\n") - if len(hdf_node.shape) <= 1 - else str(hdf_node[0]).split("\n") - ) - logger.debug(f'value: {val[0]} {"..." if len(val) > 1 else ""}') - else: - logger.debug( - f"===== GROUP (/{hdf_path} " - f"[{get_nxdl_entry(hdf_info)}" - f"::{get_nx_class_path(hdf_info)}]): {hdf_node}" - ) - (req_str, nxdef, nxdl_path) = get_nxdl_doc(hdf_info, logger, doc) - if parser is not None and isinstance(hdf_node, h5py.Dataset): - parser( - { - "hdf_info": hdf_info, - "nxdef": nxdef, - "nxdl_path": nxdl_path, - "val": val, - "logger": logger, - } - ) - for key, value in hdf_node.attrs.items(): - logger.debug(f"===== ATTRS (/{hdf_path}@{key})") - val = str(value).split("\n") - logger.debug(f'value: {val[0]} {"..." if len(val) > 1 else ""}') - (req_str, nxdef, nxdl_path) = get_nxdl_doc(hdf_info, logger, doc, attr=key) - if ( - parser is not None - and req_str is not None - and "NOT IN SCHEMA" not in req_str - and "None" not in req_str - ): - parser( - { - "hdf_info": hdf_info, - "nxdef": nxdef, - "nxdl_path": nxdl_path, - "val": val, - "logger": logger, - }, - attr=key, - ) - - -def logger_auxiliary_signal(logger, nxdata): - """Handle the presence of auxiliary signal""" - aux = nxdata.attrs.get("auxiliary_signals") - if aux is not None: - if isinstance(aux, str): - aux = [aux] - for asig in aux: - logger.debug(f"Further auxiliary signal has been identified: {asig}") - return logger - - -def print_default_plotable_header(logger): - """Print a three-lines header""" - logger.debug("========================") - logger.debug("=== Default Plotable ===") - logger.debug("========================") - - -def get_default_plotable(root, logger): - """Get default plotable""" - print_default_plotable_header(logger) - # v3 from 2014 - # nxentry - nxentry = None - default_nxentry_group_name = root.attrs.get("default") - if default_nxentry_group_name: - try: - nxentry = root[default_nxentry_group_name] - except KeyError: - nxentry = None - if not nxentry: - nxentry = entry_helper(root) - if not nxentry: - logger.debug("No NXentry has been found") - return - logger.debug("") - logger.debug("NXentry has been identified: " + nxentry.name) - # nxdata - nxdata = None - nxgroup = nxentry - default_group_name = nxgroup.attrs.get("default") - while default_group_name: - try: - nxgroup = nxgroup[default_group_name] - default_group_name = nxgroup.attrs.get("default") - except KeyError: - pass - if nxgroup == nxentry: - nxdata = nxdata_helper(nxentry) - else: - nxdata = nxgroup - if not nxdata: - logger.debug("No NXdata group has been found") - return - logger.debug("") - logger.debug("NXdata group has been identified: " + nxdata.name) - process_node(nxdata, nxdata.name, None, logger, False) - # signal - signal = None - signal_dataset_name = nxdata.attrs.get("signal") - try: - signal = nxdata[signal_dataset_name] - except (TypeError, KeyError): - signal = None - if not signal: - signal = signal_helper(nxdata) - if not signal: - logger.debug("No Signal has been found") - return - logger.debug("") - logger.debug("Signal has been identified: " + signal.name) - process_node(signal, signal.name, None, logger, False) - logger = logger_auxiliary_signal(logger, nxdata) # check auxiliary_signals - dim = len(signal.shape) - axes = [] # axes - axis_helper(dim, nxdata, signal, axes, logger) - - -def entry_helper(root): - """Check entry related data""" - nxentries = [] - for key in root.keys(): - if ( - isinstance(root[key], h5py.Group) - and root[key].attrs.get("NX_class") - and root[key].attrs["NX_class"] == "NXentry" - ): - nxentries.append(root[key]) - if len(nxentries) >= 1: - return nxentries[0] - return None - - -def nxdata_helper(nxentry): - """Check if nxentry hdf5 object has a NX_class and, if it contains NXdata, - return its value""" - lnxdata = [] - for key in nxentry.keys(): - if ( - isinstance(nxentry[key], h5py.Group) - and nxentry[key].attrs.get("NX_class") - and nxentry[key].attrs["NX_class"] == "NXdata" - ): - lnxdata.append(nxentry[key]) - if len(lnxdata) >= 1: - return lnxdata[0] - return None - - -def signal_helper(nxdata): - """Check signal related data""" - signals = [] - for key in nxdata.keys(): - if isinstance(nxdata[key], h5py.Dataset): - signals.append(nxdata[key]) - if ( - len(signals) == 1 - ): # v3: as there was no selection given, only 1 data field shall exists - return signals[0] - if len(signals) > 1: # v2: select the one with an attribute signal="1" attribute - for sig in signals: - if ( - sig.attrs.get("signal") - and sig.attrs.get("signal") is str - and sig.attrs.get("signal") == "1" - ): - return sig - return None - - -def find_attrib_axis_actual_dim_num(nxdata, a_item, ax_list): - """Finds axis that have defined dimensions""" - # find those with attribute axis= actual dimension number - lax = [] - for key in nxdata.keys(): - if isinstance(nxdata[key], h5py.Dataset): - try: - if nxdata[key].attrs["axis"] == a_item + 1: - lax.append(nxdata[key]) - except KeyError: - pass - if len(lax) == 1: - ax_list.append(lax[0]) - # if there are more alternatives, prioritise the one with an attribute primary="1" - elif len(lax) > 1: - for sax in lax: - if sax.attrs.get("primary") and sax.attrs.get("primary") == 1: - ax_list.insert(0, sax) - else: - ax_list.append(sax) - - -def get_single_or_multiple_axes(nxdata, ax_datasets, a_item, ax_list): - """Gets either single or multiple axes from the NXDL""" - try: - if isinstance(ax_datasets, str): # single axis is defined - # explicite definition of dimension number - ind = nxdata.attrs.get(ax_datasets + "_indices") - if ind and ind is int: - if ind == a_item: - ax_list.append(nxdata[ax_datasets]) - elif a_item == 0: # positional determination of the dimension number - ax_list.append(nxdata[ax_datasets]) - else: # multiple axes are listed - # explicite definition of dimension number - for aax in ax_datasets: - ind = nxdata.attrs.get(aax + "_indices") - if ind and isinstance(ind, int): - if ind == a_item: - ax_list.append(nxdata[aax]) - if not ax_list: # positional determination of the dimension number - ax_list.append(nxdata[ax_datasets[a_item]]) - except KeyError: - pass - return ax_list - - -def axis_helper(dim, nxdata, signal, axes, logger): - """Check axis related data""" - for a_item in range(dim): - ax_list = [] - ax_datasets = nxdata.attrs.get("axes") # primary axes listed in attribute axes - ax_list = get_single_or_multiple_axes(nxdata, ax_datasets, a_item, ax_list) - for attr in nxdata.attrs.keys(): # check for corresponding AXISNAME_indices - if ( - attr.endswith("_indices") - and nxdata.attrs[attr] == a_item - and nxdata[attr.split("_indices")[0]] not in ax_list - ): - ax_list.append(nxdata[attr.split("_indices")[0]]) - # v2 # check for ':' separated axes defined in Signal - if not ax_list: - try: - ax_datasets = signal.attrs.get("axes").split(":") - ax_list.append(nxdata[ax_datasets[a_item]]) - except (KeyError, AttributeError): - pass - if not ax_list: # check for axis/primary specifications - find_attrib_axis_actual_dim_num(nxdata, a_item, ax_list) - axes.append(ax_list) - logger.debug("") - logger.debug( - f"For Axis #{a_item}, {len(ax_list)} axes have been identified: {str(ax_list)}" - ) - - -def get_all_is_a_rel_from_hdf_node(hdf_node, hdf_path): - """Return list of nxdl concept paths for a nxdl element which corresponds to - hdf node. - """ - hdf_info = {"hdf_path": hdf_path, "hdf_node": hdf_node} - (_, _, elist) = get_inherited_nodes( - None, - nx_name=get_nxdl_entry(hdf_info), - hdf_node=hdf_node, - hdf_path=hdf_info["hdf_path"] if "hdf_path" in hdf_info else None, - hdf_root=hdf_info["hdf_root"] if "hdf_root" in hdf_info else None, - ) - return elist - - -def hdf_node_to_self_concept_path(hdf_info, logger): - """Get concept or nxdl path from given hdf_node.""" - # The bellow logger is for deactivatine unnecessary debug message above - if logger is None: - logger = logging.getLogger(__name__) - logger.setLevel(logging.INFO) - (_, _, nxdl_path) = get_nxdl_doc(hdf_info, logger, None) - con_path = "" - if nxdl_path: - for nd_ in nxdl_path: - con_path = con_path + "/" + get_node_name(nd_) - return con_path - - -class HandleNexus: - """documentation""" - - def __init__(self, logger, nexus_file, d_inq_nd=None, c_inq_nd=None): - self.logger = logger - - if nexus_file is None: - raise ValueError("Nexus file not specified. Cannot proceed.") - - self.input_file_name = nexus_file - self.parser = None - self.in_file = None - self.d_inq_nd = d_inq_nd - self.c_inq_nd = c_inq_nd - # Aggregating hdf path corresponds to concept query node - self.hdf_path_list_for_c_inq_nd = [] - - def visit_node(self, hdf_name, hdf_node): - """Function called by h5py that iterates on each node of hdf5file. - It allows h5py visititems function to visit nodes.""" - if self.d_inq_nd is None and self.c_inq_nd is None: - process_node(hdf_node, "/" + hdf_name, self.parser, self.logger) - elif self.d_inq_nd is not None and hdf_name in ( - self.d_inq_nd, - self.d_inq_nd[1:], - ): - process_node(hdf_node, "/" + hdf_name, self.parser, self.logger) - elif self.c_inq_nd is not None: - attributed_concept = self.c_inq_nd.split("@") - attr = attributed_concept[1] if len(attributed_concept) > 1 else None - elist = get_all_is_a_rel_from_hdf_node(hdf_node, "/" + hdf_name) - if elist is None: - return - fnd_superclass = False - fnd_superclass_attr = False - for elem in reversed(elist): - tmp_path = elem.get("nxdlbase").split(".nxdl")[0] - con_path = "/NX" + tmp_path.split("NX")[-1] + elem.get("nxdlpath") - if fnd_superclass or con_path == attributed_concept[0]: - fnd_superclass = True - if attr is None: - self.hdf_path_list_for_c_inq_nd.append(hdf_name) - break - for attribute in hdf_node.attrs.keys(): - attr_concept = get_nxdl_child( - elem, attribute, nexus_type="attribute", go_base=False - ) - if attr_concept is not None and attr_concept.get( - "nxdlpath" - ).endswith(attr): - fnd_superclass_attr = True - con_path = ( - "/NX" - + tmp_path.split("NX")[-1] - + attr_concept.get("nxdlpath") - ) - self.hdf_path_list_for_c_inq_nd.append( - hdf_name + "@" + attribute - ) - break - if fnd_superclass_attr: - break - - def not_yet_visited(self, root, name): - """checking if a new node has already been visited in its path""" - path = name.split("/") - for i in range(1, len(path)): - act_path = "/".join(path[:i]) - # print(act_path+' - '+name) - if root["/" + act_path] == root["/" + name]: - return False - return True - - def full_visit(self, root, hdf_node, name, func): - """visiting recursivly all children, but avoiding endless cycles""" - # print(name) - if len(name) > 0: - func(name, hdf_node) - if isinstance(hdf_node, h5py.Group): - for ch_name, child in hdf_node.items(): - full_name = ch_name if len(name) == 0 else name + "/" + ch_name - if self.not_yet_visited(root, full_name): - self.full_visit(root, child, full_name, func) - - def process_nexus_master_file(self, parser): - """Process a nexus master file by processing all its nodes and their attributes""" - self.parser = parser - self.in_file = h5py.File( - self.input_file_name[0] - if isinstance(self.input_file_name, list) - else self.input_file_name, - "r", - ) - self.full_visit(self.in_file, self.in_file, "", self.visit_node) - if self.d_inq_nd is None and self.c_inq_nd is None: - get_default_plotable(self.in_file, self.logger) - # To log the provided concept and concepts founded - if self.c_inq_nd is not None: - for hdf_path in self.hdf_path_list_for_c_inq_nd: - self.logger.info(hdf_path) - self.in_file.close() - - -@click.command() -@click.argument( - "nexus_file", -) -@click.option( - "-d", - "--documentation", - required=False, - default=None, - help=( - "Definition path in nexus output (.nxs) file. Returns debug" - "log relavent with that definition path. Example: /entry/data/delays" - ), -) -@click.option( - "-c", - "--concept", - required=False, - default=None, - help=( - "Concept path from application definition file (.nxdl,xml). Finds out" - "all the available concept definition (IS-A realation) for rendered" - "concept path. Example: /NXarpes/ENTRY/INSTRUMENT/analyser" - ), -) -def main(nexus_file, documentation, concept): - """The main function to call when used as a script.""" - logging_format = "%(levelname)s: %(message)s" - stdout_handler = logging.StreamHandler(sys.stdout) - stdout_handler.setLevel(logging.DEBUG) - logging.basicConfig( - level=logging.INFO, format=logging_format, handlers=[stdout_handler] - ) - logger = logging.getLogger(__name__) - logger.addHandler(stdout_handler) - logger.setLevel(logging.DEBUG) - logger.propagate = False - if documentation and concept: - raise ValueError( - "Only one option either documentation (-d) or is_a relation " - "with a concept (-c) can be requested." - ) - nexus_helper = HandleNexus( - logger, nexus_file, d_inq_nd=documentation, c_inq_nd=concept - ) - nexus_helper.process_nexus_master_file(None) - - -if __name__ == "__main__": - main() # pylint: disable=no-value-for-parameter From f2cd2ac51c37bfd4144961cbe1e5ab901fe43303 Mon Sep 17 00:00:00 2001 From: Sandor Brockhauser Date: Wed, 21 Jun 2023 12:53:20 +0200 Subject: [PATCH 41/93] remove dependencies also from pypi configuration --- pyproject.toml | 3 --- requirements.txt | 2 -- 2 files changed, 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d4cf990c86..97430f09bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,8 +18,6 @@ classifiers = [ dependencies = [ "lxml", "pyyaml", - "click>=7.1.2", - "h5py>=3.6.0", "sphinx>=5", "sphinx-tabs", "pytest", @@ -32,7 +30,6 @@ dependencies = [ "Homepage" = "https://nexusformat.org" [project.scripts] -read_nexus = "dev_tools.utils.nexus:main" [tools.setuptools_scm] version_scheme = "guess-next-dev" diff --git a/requirements.txt b/requirements.txt index bbfd892f76..54b7bb86f8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,6 @@ # Prepare for Documentation lxml pyyaml -click>=7.1.2 -h5py>=3.6.0 # Documentation building sphinx>=5 From aabf43ba356c2e55aa57169d424b8ef846099266 Mon Sep 17 00:00:00 2001 From: Sandor Brockhauser Date: Wed, 21 Jun 2023 16:20:06 +0200 Subject: [PATCH 42/93] added sphinx-toolbox --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 97430f09bc..979c51eac0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,6 +20,7 @@ dependencies = [ "pyyaml", "sphinx>=5", "sphinx-tabs", + "sphinx-toolbox", "pytest", "black>=22.3", "flake8>=4", From c510daebaaca5477d5f2a37a335cf97b8bf3cb3e Mon Sep 17 00:00:00 2001 From: Sandor Brockhauser Date: Wed, 21 Jun 2023 17:20:14 +0200 Subject: [PATCH 43/93] collapsing is deactivated for this PR --- dev_tools/docs/nxdl.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dev_tools/docs/nxdl.py b/dev_tools/docs/nxdl.py index 4c36ef2775..d35c01fa44 100644 --- a/dev_tools/docs/nxdl.py +++ b/dev_tools/docs/nxdl.py @@ -517,9 +517,9 @@ def _print_doc_enum(self, indent, ns, node, required=False): collapse_indent = indent node_list = node.xpath("nx:enumeration", namespaces=ns) (doclen, line, blocks) = self.long_doc(ns, node) - if len(node_list) + doclen > 1: - collapse_indent = f"{indent} " - self._print(f"{indent}{self._INDENTATION_UNIT}.. collapse:: {line} ...\n") + # if len(node_list) + doclen > 1: + # collapse_indent = f"{indent} " + # self._print(f"{indent}{self._INDENTATION_UNIT}.. collapse:: {line} ...\n") self._print_doc( collapse_indent + self._INDENTATION_UNIT, ns, node, required=required ) From 9240450e1ea88fbe528fd4b8c912f0f441dd7e07 Mon Sep 17 00:00:00 2001 From: Sherjeel Shabih Date: Wed, 5 Jul 2023 15:09:03 +0200 Subject: [PATCH 44/93] Move all of nyaml2nxdl files in a folder --- README.md => dev_tools/nyaml2nxdl/README.md | 0 __init__.py => dev_tools/nyaml2nxdl/__init__.py | 0 comment_collector.py => dev_tools/nyaml2nxdl/comment_collector.py | 0 nyaml2nxdl.py => dev_tools/nyaml2nxdl/nyaml2nxdl.py | 0 .../nyaml2nxdl/nyaml2nxdl_backward_tools.py | 0 .../nyaml2nxdl/nyaml2nxdl_forward_tools.py | 0 nyaml2nxdl_helper.py => dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py | 0 7 files changed, 0 insertions(+), 0 deletions(-) rename README.md => dev_tools/nyaml2nxdl/README.md (100%) rename __init__.py => dev_tools/nyaml2nxdl/__init__.py (100%) rename comment_collector.py => dev_tools/nyaml2nxdl/comment_collector.py (100%) rename nyaml2nxdl.py => dev_tools/nyaml2nxdl/nyaml2nxdl.py (100%) rename nyaml2nxdl_backward_tools.py => dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py (100%) rename nyaml2nxdl_forward_tools.py => dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py (100%) rename nyaml2nxdl_helper.py => dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py (100%) diff --git a/README.md b/dev_tools/nyaml2nxdl/README.md similarity index 100% rename from README.md rename to dev_tools/nyaml2nxdl/README.md diff --git a/__init__.py b/dev_tools/nyaml2nxdl/__init__.py similarity index 100% rename from __init__.py rename to dev_tools/nyaml2nxdl/__init__.py diff --git a/comment_collector.py b/dev_tools/nyaml2nxdl/comment_collector.py similarity index 100% rename from comment_collector.py rename to dev_tools/nyaml2nxdl/comment_collector.py diff --git a/nyaml2nxdl.py b/dev_tools/nyaml2nxdl/nyaml2nxdl.py similarity index 100% rename from nyaml2nxdl.py rename to dev_tools/nyaml2nxdl/nyaml2nxdl.py diff --git a/nyaml2nxdl_backward_tools.py b/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py similarity index 100% rename from nyaml2nxdl_backward_tools.py rename to dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py diff --git a/nyaml2nxdl_forward_tools.py b/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py similarity index 100% rename from nyaml2nxdl_forward_tools.py rename to dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py diff --git a/nyaml2nxdl_helper.py b/dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py similarity index 100% rename from nyaml2nxdl_helper.py rename to dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py From 666203a6a2ae3b41821820181d8140f97b749526 Mon Sep 17 00:00:00 2001 From: Sherjeel Shabih Date: Wed, 5 Jul 2023 15:31:07 +0200 Subject: [PATCH 45/93] Moving test to correct folder for nexus definitions repo --- README.md | 5 ----- test_nyaml2nxdl.py => dev_tools/tests/test_nyaml2nxdl.py | 0 2 files changed, 5 deletions(-) delete mode 100644 README.md rename test_nyaml2nxdl.py => dev_tools/tests/test_nyaml2nxdl.py (100%) diff --git a/README.md b/README.md deleted file mode 100644 index 7a71982697..0000000000 --- a/README.md +++ /dev/null @@ -1,5 +0,0 @@ -This is the place for storing code for tests of the yaml2nxdl and nxdl2yaml NeXus schema translation routines. - -## Contact person in FAIRmat for these tests -1. Rubel Mozumder -2. Andrea Albino \ No newline at end of file diff --git a/test_nyaml2nxdl.py b/dev_tools/tests/test_nyaml2nxdl.py similarity index 100% rename from test_nyaml2nxdl.py rename to dev_tools/tests/test_nyaml2nxdl.py From ff05cb5b6147e2eb8e0c44892b93abb46107d82b Mon Sep 17 00:00:00 2001 From: Sandor Brockhauser Date: Mon, 19 Jun 2023 15:01:41 +0200 Subject: [PATCH 46/93] linting --- dev_tools/nyaml2nxdl/comment_collector.py | 199 ++-- dev_tools/nyaml2nxdl/nyaml2nxdl.py | 206 +++-- .../nyaml2nxdl/nyaml2nxdl_backward_tools.py | 645 +++++++------ .../nyaml2nxdl/nyaml2nxdl_forward_tools.py | 854 ++++++++++-------- dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py | 85 +- 5 files changed, 1100 insertions(+), 889 deletions(-) diff --git a/dev_tools/nyaml2nxdl/comment_collector.py b/dev_tools/nyaml2nxdl/comment_collector.py index 5f0c5e3bce..dcb21021be 100644 --- a/dev_tools/nyaml2nxdl/comment_collector.py +++ b/dev_tools/nyaml2nxdl/comment_collector.py @@ -31,10 +31,16 @@ """ -from typing import List, Type, Any, Tuple, Union, Dict +from typing import Any +from typing import Dict +from typing import List +from typing import Tuple +from typing import Type +from typing import Union + from pynxtools.nyaml2nxdl.nyaml2nxdl_helper import LineLoader -__all__ = ['Comment', 'CommentCollector', 'XMLComment', 'YAMLComment'] +__all__ = ["Comment", "CommentCollector", "XMLComment", "YAMLComment"] # pylint: disable=inconsistent-return-statements @@ -43,8 +49,7 @@ class CommentCollector: _comment_chain. """ - def __init__(self, input_file: str = None, - loaded_obj: Union[object, Dict] = None): + def __init__(self, input_file: str = None, loaded_obj: Union[object, Dict] = None): """ Initialise CommentCollector parameters: @@ -57,19 +62,21 @@ def __init__(self, input_file: str = None, self._comment_hash: Dict[Tuple, Type[Comment]] = {} self.comment: Type[Comment] if self.file and not loaded_obj: - if self.file.split('.')[-1] == 'xml': + if self.file.split(".")[-1] == "xml": self.comment = XMLComment - if self.file.split('.')[-1] == 'yaml': + if self.file.split(".")[-1] == "yaml": self.comment = YAMLComment with open(self.file, "r", encoding="utf-8") as plain_text_yaml: loader = LineLoader(plain_text_yaml) self.comment.__yaml_dict__ = loader.get_single_data() elif self.file and loaded_obj: - if self.file.split('.')[-1] == 'yaml' and isinstance(loaded_obj, dict): + if self.file.split(".")[-1] == "yaml" and isinstance(loaded_obj, dict): self.comment = YAMLComment self.comment.__yaml_dict__ = loaded_obj else: - raise ValueError("Incorrect inputs for CommentCollector e.g. Wrong file extension.") + raise ValueError( + "Incorrect inputs for CommentCollector e.g. Wrong file extension." + ) else: raise ValueError("Incorrect inputs for CommentCollector") @@ -81,18 +88,20 @@ def extract_all_comment_blocks(self): """ id_ = 0 single_comment = self.comment(comment_id=id_) - with open(self.file, mode='r', encoding='UTF-8') as enc_f: + with open(self.file, mode="r", encoding="UTF-8") as enc_f: lines = enc_f.readlines() # Make an empty line for last comment if no empty lines in original file - if lines[-1] != '': - lines.append('') + if lines[-1] != "": + lines.append("") for line_num, line in enumerate(lines): if single_comment.is_storing_single_comment(): # If the last comment comes without post nxdl fields, groups and attributes - if '++ SHA HASH ++' in line: + if "++ SHA HASH ++" in line: # Handle with stored nxdl.xml file that is not part of yaml - line = '' - single_comment.process_each_line(line + 'post_comment', (line_num + 1)) + line = "" + single_comment.process_each_line( + line + "post_comment", (line_num + 1) + ) self._comment_chain.append(single_comment) break if line_num < (len(lines) - 1): @@ -100,7 +109,9 @@ def extract_all_comment_blocks(self): single_comment.process_each_line(line, (line_num + 1)) else: # For processing last line of file - single_comment.process_each_line(line + 'post_comment', (line_num + 1)) + single_comment.process_each_line( + line + "post_comment", (line_num + 1) + ) self._comment_chain.append(single_comment) else: self._comment_chain.append(single_comment) @@ -109,13 +120,13 @@ def extract_all_comment_blocks(self): def get_comment(self): """ - Return comment from comment_chain that must come earlier in order. + Return comment from comment_chain that must come earlier in order. """ return self._comment_chain[self._comment_tracker] def get_coment_by_line_info(self, comment_locs: Tuple[str, Union[int, str]]): """ - Get comment using line information. + Get comment using line information. """ if comment_locs in self._comment_hash: return self._comment_hash[comment_locs] @@ -129,8 +140,7 @@ def get_coment_by_line_info(self, comment_locs: Tuple[str, Union[int, str]]): return cmnt def remove_comment(self, ind): - """Remove a comment from comment list. - """ + """Remove a comment from comment list.""" if ind < len(self._comment_chain): del self._comment_chain[ind] else: @@ -150,8 +160,10 @@ def __contains__(self, comment_locs: tuple): (__line__doc and 35) """ if not isinstance(comment_locs, tuple): - raise TypeError("Comment_locs should be 'tuple' containing line annotation " - "(e.g.__line__doc) and line_loc (e.g. 35).") + raise TypeError( + "Comment_locs should be 'tuple' containing line annotation " + "(e.g.__line__doc) and line_loc (e.g. 35)." + ) line_annot, line_loc = comment_locs for cmnt in self._comment_chain: if line_annot in cmnt: @@ -162,11 +174,12 @@ def __contains__(self, comment_locs: tuple): return False def __getitem__(self, ind): - """Get comment from self.obj._comment_chain by index. - """ + """Get comment from self.obj._comment_chain by index.""" if isinstance(ind, int): if ind >= len(self._comment_chain): - raise IndexError(f'Oops! Comment index {ind} in {__class__} is out of range!') + raise IndexError( + f"Oops! Comment index {ind} in {__class__} is out of range!" + ) return self._comment_chain[ind] if isinstance(ind, slice): @@ -175,8 +188,7 @@ def __getitem__(self, ind): return self._comment_chain[start_n:end_n] def __iter__(self): - """get comment ieratively - """ + """get comment ieratively""" return iter(self._comment_chain) @@ -186,21 +198,19 @@ class Comment: This class is building yaml comment and the intended line for what comment is written. """ - def __init__(self, - comment_id: int = -1, - last_comment: 'Comment' = None) -> None: + def __init__(self, comment_id: int = -1, last_comment: "Comment" = None) -> None: """Comment object can be considered as a block element that includes - document element (an entity for what the comment is written). + document element (an entity for what the comment is written). """ self._elemt: Any = None self._elemt_text: str = None self._is_elemt_found: bool = None self._is_elemt_stored: bool = None - self._comnt: str = '' + self._comnt: str = "" # If Multiple comments for one element or entity self._comnt_list: List[str] = [] - self.last_comment: 'Comment' = last_comment if last_comment else None + self.last_comment: "Comment" = last_comment if last_comment else None if comment_id >= 0 and last_comment: self.cid = comment_id self.last_comment = last_comment @@ -214,8 +224,9 @@ def __init__(self, raise ValueError("Neither last comment nor comment id dound") self._comnt_start_found: bool = False self._comnt_end_found: bool = False - self.is_storing_single_comment = lambda: not (self._comnt_end_found - and self._is_elemt_stored) + self.is_storing_single_comment = lambda: not ( + self._comnt_end_found and self._is_elemt_stored + ) def get_comment_text(self) -> Union[List, str]: """ @@ -239,7 +250,7 @@ class XMLComment(Comment): XMLComment to store xml comment element. """ - def __init__(self, comment_id: int = -1, last_comment: 'Comment' = None) -> None: + def __init__(self, comment_id: int = -1, last_comment: "Comment" = None) -> None: super().__init__(comment_id, last_comment) def process_each_line(self, text, line_num): @@ -253,76 +264,77 @@ def process_each_line(self, text, line_num): # for multiple comment if exist if self._comnt: self._comnt_list.append(self._comnt) - self._comnt = '' + self._comnt = "" if self._comnt_end_found: self.store_element(text) def append_comment(self, text: str) -> None: # Comment in single line - if '' == text[-4:]: + self._comnt = self._comnt + text.replace("" == text[-4:]: self._comnt_end_found = True self._comnt_start_found = False - self._comnt = self._comnt.replace('-->', '') + self._comnt = self._comnt.replace("-->", "") - elif '-->' == text[0:4] and self._comnt_start_found: + elif "-->" == text[0:4] and self._comnt_start_found: self._comnt_end_found = True self._comnt_start_found = False - self._comnt = self._comnt + '\n' + text.replace('-->', '') + self._comnt = self._comnt + "\n" + text.replace("-->", "") elif self._comnt_start_found: - self._comnt = self._comnt + '\n' + text + self._comnt = self._comnt + "\n" + text # pylint: disable=arguments-differ, arguments-renamed def store_element(self, text) -> None: def collect_xml_attributes(text_part): for part in text_part: part = part.strip() - if part and '">' == ''.join(part[-2:]): + if part and '">' == "".join(part[-2:]): self._is_elemt_stored = True self._is_elemt_found = False - part = ''.join(part[0:-2]) - elif part and '"/>' == ''.join(part[-3:]): + part = "".join(part[0:-2]) + elif part and '"/>' == "".join(part[-3:]): self._is_elemt_stored = True self._is_elemt_found = False - part = ''.join(part[0:-3]) - elif part and '/>' == ''.join(part[-2:]): + part = "".join(part[0:-3]) + elif part and "/>" == "".join(part[-2:]): self._is_elemt_stored = True self._is_elemt_found = False - part = ''.join(part[0:-2]) - elif part and '>' == part[-1]: + part = "".join(part[0:-2]) + elif part and ">" == part[-1]: self._is_elemt_stored = True self._is_elemt_found = False - part = ''.join(part[0:-1]) + part = "".join(part[0:-1]) elif part and '"' == part[-1]: - part = ''.join(part[0:-1]) + part = "".join(part[0:-1]) if '="' in part: lf_prt, rt_prt = part.split('="') else: continue - if ':' in lf_prt: + if ":" in lf_prt: continue self._elemt[lf_prt] = str(rt_prt) + if not self._elemt: self._elemt = {} # First check for comment part has been collected prefectly - if ' Union[List, str]: """ - This method returns list of commnent text. As some xml element might have - multiple separated comment intended for a single element. + This method returns list of commnent text. As some xml element might have + multiple separated comment intended for a single element. """ return self._comnt_list @@ -348,17 +360,19 @@ class YAMLComment(Comment): 1. Do not delete any element form yaml dictionary (for loaded_obj. check: Comment_collector class. because this loaded file has been exploited in nyaml2nxdl forward tools.) """ + # Class level variable. The main reason behind that to follow structure of # abstract class 'Comment' __yaml_dict__: dict = {} __yaml_line_info: dict = {} - __comment_escape_char = {'--': '-\\-'} + __comment_escape_char = {"--": "-\\-"} - def __init__(self, comment_id: int = -1, last_comment: 'Comment' = None) -> None: - """Initialization of YAMLComment follow Comment class. - """ + def __init__(self, comment_id: int = -1, last_comment: "Comment" = None) -> None: + """Initialization of YAMLComment follow Comment class.""" super().__init__(comment_id, last_comment) - self.collect_yaml_line_info(YAMLComment.__yaml_dict__, YAMLComment.__yaml_line_info) + self.collect_yaml_line_info( + YAMLComment.__yaml_dict__, YAMLComment.__yaml_line_info + ) def process_each_line(self, text, line_num): """Take care of each line of text. Through which function the text @@ -369,21 +383,21 @@ def process_each_line(self, text, line_num): if self._comnt_end_found and not self._is_elemt_found: if self._comnt: self._comnt_list.append(self._comnt) - self._comnt = '' + self._comnt = "" if self._comnt_end_found: - line_key = '' - if ':' in text: - ind = text.index(':') - line_key = '__line__' + ''.join(text[0:ind]) + line_key = "" + if ":" in text: + ind = text.index(":") + line_key = "__line__" + "".join(text[0:ind]) for l_num, l_key in self.__yaml_line_info.items(): if line_num == int(l_num) and line_key == l_key: self.store_element(line_key, line_num) break # Comment comes very end of the file - if text == 'post_comment' and line_key == '': - line_key = '__line__post_comment' + if text == "post_comment" and line_key == "": + line_key = "__line__post_comment" self.store_element(line_key, line_num) def has_post_comment(self): @@ -393,7 +407,7 @@ def has_post_comment(self): nxdl element(class, group, filed and attribute.) """ for key, _ in self._elemt.items(): - if '__line__post_comment' == key: + if "__line__post_comment" == key: return True return False @@ -411,17 +425,17 @@ def append_comment(self, text: str) -> None: # For empty line inside doc or yaml file. elif not text: return - elif '# ' == ''.join(text[0:2]): + elif "# " == "".join(text[0:2]): self._comnt_start_found = True self._comnt_end_found = False - self._comnt = '' if not self._comnt else self._comnt + '\n' - self._comnt = self._comnt + ''.join(text[2:]) - elif '#' == text[0]: + self._comnt = "" if not self._comnt else self._comnt + "\n" + self._comnt = self._comnt + "".join(text[2:]) + elif "#" == text[0]: self._comnt_start_found = True self._comnt_end_found = False - self._comnt = '' if not self._comnt else self._comnt + '\n' - self._comnt = self._comnt + ''.join(text[1:]) - elif 'post_comment' == text: + self._comnt = "" if not self._comnt else self._comnt + "\n" + self._comnt = self._comnt + "".join(text[1:]) + elif "post_comment" == text: self._comnt_end_found = True self._comnt_start_found = False # for any line after 'comment block' found @@ -432,8 +446,8 @@ def append_comment(self, text: str) -> None: # pylint: disable=arguments-differ def store_element(self, line_key, line_number): """ - Store comment content and information of commen location (for what comment is - created.). + Store comment content and information of commen location (for what comment is + created.). """ self._elemt = {} self._elemt[line_key] = int(line_number) @@ -454,14 +468,13 @@ def get_line_number(self, line_key): def get_line_info(self): """ - Return line annotation and line number from a comment. + Return line annotation and line number from a comment. """ for line_anno, line_loc in self._elemt.items(): return line_anno, line_loc def replace_scape_char(self, text): - """Replace escape char according to __comment_escape_char dict - """ + """Replace escape char according to __comment_escape_char dict""" for ecp_char, ecp_alt in YAMLComment.__comment_escape_char.items(): if ecp_char in text: text = text.replace(ecp_char, ecp_alt) @@ -472,8 +485,7 @@ def get_element_location(self): Retrun yaml line '__line__KEY' info and and line numner """ if len(self._elemt) > 1: - raise ValueError(f"Comment element should be one but got " - f"{self._elemt}") + raise ValueError(f"Comment element should be one but got " f"{self._elemt}") for key, val in self._elemt.items(): yield key, val @@ -483,7 +495,7 @@ def collect_yaml_line_info(self, yaml_dict, line_info_dict): a yaml file dictonary in another dictionary. """ for line_key, line_n in yaml_dict.items(): - if '__line__' in line_key: + if "__line__" in line_key: line_info_dict[line_n] = line_key for _, val in yaml_dict.items(): @@ -495,13 +507,12 @@ def __contains__(self, line_key): return line_key in self._elemt def __eq__(self, comment_obj): - """Check the self has same value as right comment. - """ + """Check the self has same value as right comment.""" if len(self._comnt_list) != len(comment_obj._comnt_list): return False for left_cmnt, right_cmnt in zip(self._comnt_list, comment_obj._comnt_list): - left_cmnt = left_cmnt.split('\n') - right_cmnt = right_cmnt.split('\n') + left_cmnt = left_cmnt.split("\n") + right_cmnt = right_cmnt.split("\n") for left_line, right_line in zip(left_cmnt, right_cmnt): if left_line.strip() != right_line.strip(): return False diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl.py b/dev_tools/nyaml2nxdl/nyaml2nxdl.py index 160b3f830d..815b015e62 100755 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl.py @@ -26,13 +26,13 @@ import xml.etree.ElementTree as ET import click -from pynxtools.nyaml2nxdl.nyaml2nxdl_helper import (get_sha256_hash, - extend_yamlfile_with_comment, - separate_hash_yaml_and_nxdl) -from pynxtools.nyaml2nxdl.nyaml2nxdl_forward_tools import nyaml2nxdl, pretty_print_xml -from pynxtools.nyaml2nxdl.nyaml2nxdl_backward_tools import (Nxdl2yaml, - compare_niac_and_my) - +from pynxtools.nyaml2nxdl.nyaml2nxdl_backward_tools import Nxdl2yaml +from pynxtools.nyaml2nxdl.nyaml2nxdl_backward_tools import compare_niac_and_my +from pynxtools.nyaml2nxdl.nyaml2nxdl_forward_tools import nyaml2nxdl +from pynxtools.nyaml2nxdl.nyaml2nxdl_forward_tools import pretty_print_xml +from pynxtools.nyaml2nxdl.nyaml2nxdl_helper import extend_yamlfile_with_comment +from pynxtools.nyaml2nxdl.nyaml2nxdl_helper import get_sha256_hash +from pynxtools.nyaml2nxdl.nyaml2nxdl_helper import separate_hash_yaml_and_nxdl DEPTH_SIZE = 4 * " " @@ -42,13 +42,13 @@ def generate_nxdl_or_retrieve_nxdl(yaml_file, out_xml_file, verbose): """ - Generate yaml, nxdl and hash. - if the extracted hash is exactly the same as producd from generated yaml then - retrieve the nxdl part from provided yaml. - Else, generate nxdl from separated yaml with the help of nyaml2nxdl function + Generate yaml, nxdl and hash. + if the extracted hash is exactly the same as producd from generated yaml then + retrieve the nxdl part from provided yaml. + Else, generate nxdl from separated yaml with the help of nyaml2nxdl function """ pa_path, rel_file = os.path.split(yaml_file) - sep_yaml = os.path.join(pa_path, f'temp_{rel_file}') + sep_yaml = os.path.join(pa_path, f"temp_{rel_file}") hash_found = separate_hash_yaml_and_nxdl(yaml_file, sep_yaml, out_xml_file) if hash_found: @@ -67,67 +67,84 @@ def append_yml(input_file, append, verbose): and print both an XML and YML file of the extended base class. """ - nexus_def_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), '../../definitions') - assert [s for s in os.listdir(os.path.join(nexus_def_path, 'base_classes') - ) if append.strip() == s.replace('.nxdl.xml', '')], \ - 'Your base class extension does not match any existing NeXus base classes' - tree = ET.parse(os.path.join(nexus_def_path + '/base_classes', append + '.nxdl.xml')) + nexus_def_path = os.path.join( + os.path.abspath(os.path.dirname(__file__)), "../../definitions" + ) + assert [ + s + for s in os.listdir(os.path.join(nexus_def_path, "base_classes")) + if append.strip() == s.replace(".nxdl.xml", "") + ], "Your base class extension does not match any existing NeXus base classes" + tree = ET.parse( + os.path.join(nexus_def_path + "/base_classes", append + ".nxdl.xml") + ) root = tree.getroot() # warning: tmp files are printed on disk and removed at the ends!! - pretty_print_xml(root, 'tmp.nxdl.xml') - input_tmp_xml = 'tmp.nxdl.xml' - out_tmp_yml = 'tmp_parsed.yaml' + pretty_print_xml(root, "tmp.nxdl.xml") + input_tmp_xml = "tmp.nxdl.xml" + out_tmp_yml = "tmp_parsed.yaml" converter = Nxdl2yaml([], []) converter.print_yml(input_tmp_xml, out_tmp_yml, verbose) - nyaml2nxdl(input_file=out_tmp_yml, - out_file='tmp_parsed.nxdl.xml', - verbose=verbose) - tree = ET.parse('tmp_parsed.nxdl.xml') + nyaml2nxdl(input_file=out_tmp_yml, out_file="tmp_parsed.nxdl.xml", verbose=verbose) + tree = ET.parse("tmp_parsed.nxdl.xml") tree2 = ET.parse(input_file) root_no_duplicates = ET.Element( - 'definition', {'xmlns': 'http://definition.nexusformat.org/nxdl/3.1', - 'xmlns:xsi': 'http://www.w3.org/2001/XMLSchema-instance', - 'xsi:schemaLocation': 'http://www.w3.org/2001/XMLSchema-instance' - } + "definition", + { + "xmlns": "http://definition.nexusformat.org/nxdl/3.1", + "xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance", + "xsi:schemaLocation": "http://www.w3.org/2001/XMLSchema-instance", + }, ) for attribute_keys in root.attrib.keys(): - if attribute_keys != '{http://www.w3.org/2001/XMLSchema-instance}schemaLocation': + if ( + attribute_keys + != "{http://www.w3.org/2001/XMLSchema-instance}schemaLocation" + ): attribute_value = root.attrib[attribute_keys] root_no_duplicates.set(attribute_keys, attribute_value) for elems in root.iter(): - if 'doc' in elems.tag: - root_doc = ET.SubElement(root_no_duplicates, 'doc') + if "doc" in elems.tag: + root_doc = ET.SubElement(root_no_duplicates, "doc") root_doc.text = elems.text break - group = '{http://definition.nexusformat.org/nxdl/3.1}group' - root_no_duplicates = compare_niac_and_my(tree, tree2, verbose, - group, - root_no_duplicates) - field = '{http://definition.nexusformat.org/nxdl/3.1}field' - root_no_duplicates = compare_niac_and_my(tree, tree2, verbose, - field, - root_no_duplicates) - attribute = '{http://definition.nexusformat.org/nxdl/3.1}attribute' - root_no_duplicates = compare_niac_and_my(tree, tree2, verbose, - attribute, - root_no_duplicates) - pretty_print_xml(root_no_duplicates, f"{input_file.replace('.nxdl.xml', '')}" - f"_appended.nxdl.xml") - - input_file_xml = input_file.replace('.nxdl.xml', "_appended.nxdl.xml") - out_file_yml = input_file.replace('.nxdl.xml', "_appended_parsed.yaml") + group = "{http://definition.nexusformat.org/nxdl/3.1}group" + root_no_duplicates = compare_niac_and_my( + tree, tree2, verbose, group, root_no_duplicates + ) + field = "{http://definition.nexusformat.org/nxdl/3.1}field" + root_no_duplicates = compare_niac_and_my( + tree, tree2, verbose, field, root_no_duplicates + ) + attribute = "{http://definition.nexusformat.org/nxdl/3.1}attribute" + root_no_duplicates = compare_niac_and_my( + tree, tree2, verbose, attribute, root_no_duplicates + ) + pretty_print_xml( + root_no_duplicates, + f"{input_file.replace('.nxdl.xml', '')}" f"_appended.nxdl.xml", + ) + + input_file_xml = input_file.replace(".nxdl.xml", "_appended.nxdl.xml") + out_file_yml = input_file.replace(".nxdl.xml", "_appended_parsed.yaml") converter = Nxdl2yaml([], []) converter.print_yml(input_file_xml, out_file_yml, verbose) - nyaml2nxdl(input_file=out_file_yml, - out_file=out_file_yml.replace('.yaml', '.nxdl.xml'), - verbose=verbose) - os.rename(f"{input_file.replace('.nxdl.xml', '_appended_parsed.yaml')}", - f"{input_file.replace('.nxdl.xml', '_appended.yaml')}") - os.rename(f"{input_file.replace('.nxdl.xml', '_appended_parsed.nxdl.xml')}", - f"{input_file.replace('.nxdl.xml', '_appended.nxdl.xml')}") - os.remove('tmp.nxdl.xml') - os.remove('tmp_parsed.yaml') - os.remove('tmp_parsed.nxdl.xml') + nyaml2nxdl( + input_file=out_file_yml, + out_file=out_file_yml.replace(".yaml", ".nxdl.xml"), + verbose=verbose, + ) + os.rename( + f"{input_file.replace('.nxdl.xml', '_appended_parsed.yaml')}", + f"{input_file.replace('.nxdl.xml', '_appended.yaml')}", + ) + os.rename( + f"{input_file.replace('.nxdl.xml', '_appended_parsed.nxdl.xml')}", + f"{input_file.replace('.nxdl.xml', '_appended.nxdl.xml')}", + ) + os.remove("tmp.nxdl.xml") + os.remove("tmp_parsed.yaml") + os.remove("tmp_parsed.nxdl.xml") def split_name_and_extension(file_name): @@ -135,93 +152,98 @@ def split_name_and_extension(file_name): Split file name into extension and rest of the file name. return file raw nam and extension """ - parts = file_name.rsplit('.', 3) + parts = file_name.rsplit(".", 3) if len(parts) == 2: raw = parts[0] ext = parts[1] if len(parts) == 3: raw = parts[0] - ext = '.'.join(parts[1:]) + ext = ".".join(parts[1:]) return raw, ext @click.command() @click.option( - '--input-file', + "--input-file", required=True, prompt=True, - help='The path to the XML or YAML input data file to read and create \ -a YAML or XML file from, respectively.' + help="The path to the XML or YAML input data file to read and create \ +a YAML or XML file from, respectively.", ) @click.option( - '--append', - help='Parse xml file and append to base class, given that the xml file has same name \ -of an existing base class' + "--append", + help="Parse xml file and append to base class, given that the xml file has same name \ +of an existing base class", ) @click.option( - '--check-consistency', + "--check-consistency", is_flag=True, default=False, - help=('Check wether yaml or nxdl has followed general rules of scema or not' - 'check whether your comment in the right place or not. The option render an ' - 'output file of the same extension(*_consistency.yaml or *_consistency.nxdl.xml)') + help=( + "Check wether yaml or nxdl has followed general rules of scema or not" + "check whether your comment in the right place or not. The option render an " + "output file of the same extension(*_consistency.yaml or *_consistency.nxdl.xml)" + ), ) @click.option( - '--verbose', + "--verbose", is_flag=True, default=False, - help='Print in standard output keywords and value types to help \ -possible issues in yaml files' + help="Print in standard output keywords and value types to help \ +possible issues in yaml files", ) def launch_tool(input_file, verbose, append, check_consistency): """ - Main function that distiguishes the input file format and launches the tools. + Main function that distiguishes the input file format and launches the tools. """ if os.path.isfile(input_file): raw_name, ext = split_name_and_extension(input_file) else: raise ValueError("Need a valid input file.") - if ext == 'yaml': - xml_out_file = raw_name + '.nxdl.xml' + if ext == "yaml": + xml_out_file = raw_name + ".nxdl.xml" generate_nxdl_or_retrieve_nxdl(input_file, xml_out_file, verbose) if append: - append_yml(raw_name + '.nxdl.xml', - append, - verbose - ) + append_yml(raw_name + ".nxdl.xml", append, verbose) # For consistency running if check_consistency: - yaml_out_file = raw_name + '_consistency.' + ext + yaml_out_file = raw_name + "_consistency." + ext converter = Nxdl2yaml([], []) converter.print_yml(xml_out_file, yaml_out_file, verbose) os.remove(xml_out_file) - elif ext == 'nxdl.xml': + elif ext == "nxdl.xml": if not append: - yaml_out_file = raw_name + '_parsed' + '.yaml' + yaml_out_file = raw_name + "_parsed" + ".yaml" converter = Nxdl2yaml([], []) converter.print_yml(input_file, yaml_out_file, verbose) # Append nxdl.xml file with yaml output file yaml_hash = get_sha256_hash(yaml_out_file) # Lines as divider between yaml and nxdl - top_lines = [('\n# ++++++++++++++++++++++++++++++++++ SHA HASH' - ' ++++++++++++++++++++++++++++++++++\n'), - f'# {yaml_hash}\n'] - - extend_yamlfile_with_comment(yaml_file=yaml_out_file, - file_to_be_appended=input_file, - top_lines_list=top_lines) + top_lines = [ + ( + "\n# ++++++++++++++++++++++++++++++++++ SHA HASH" + " ++++++++++++++++++++++++++++++++++\n" + ), + f"# {yaml_hash}\n", + ] + + extend_yamlfile_with_comment( + yaml_file=yaml_out_file, + file_to_be_appended=input_file, + top_lines_list=top_lines, + ) else: append_yml(input_file, append, verbose) # Taking care of consistency running if check_consistency: - xml_out_file = raw_name + '_consistency.' + ext + xml_out_file = raw_name + "_consistency." + ext generate_nxdl_or_retrieve_nxdl(yaml_out_file, xml_out_file, verbose) os.remove(yaml_out_file) else: raise ValueError("Provide correct file with extension '.yaml or '.nxdl.xml") -if __name__ == '__main__': +if __name__ == "__main__": launch_tool().parse() # pylint: disable=no-value-for-parameter diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py b/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py index 72f5a6c426..faa22cc23e 100755 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py @@ -2,6 +2,8 @@ """This file collects the function used in the reverse tool nxdl2yaml. """ +import os + # -*- coding: utf-8 -*- # # Copyright The NOMAD Authors. @@ -21,18 +23,17 @@ # limitations under the License. # import sys -from typing import List, Dict import xml.etree.ElementTree as ET -import os +from typing import Dict +from typing import List -from pynxtools.nyaml2nxdl.nyaml2nxdl_helper import (get_node_parent_info, - get_yaml_escape_char_dict, - cleaning_empty_lines) from pynxtools.dataconverter.helpers import remove_namespace_from_tag - +from pynxtools.nyaml2nxdl.nyaml2nxdl_helper import cleaning_empty_lines +from pynxtools.nyaml2nxdl.nyaml2nxdl_helper import get_node_parent_info +from pynxtools.nyaml2nxdl.nyaml2nxdl_helper import get_yaml_escape_char_dict DEPTH_SIZE = " " -CMNT_TAG = '!--' +CMNT_TAG = "!--" def separate_pi_comments(input_file): @@ -43,24 +44,24 @@ def separate_pi_comments(input_file): comment = [] xml_lines = [] - with open(input_file, "r", encoding='utf-8') as file: + with open(input_file, "r", encoding="utf-8") as file: lines = file.readlines() has_pi = True for line in lines: - c_start = '' - def_tag = ' 0 and has_pi: - comment.append(line.replace(cmnt_end, '')) - comments_list.append(''.join(comment)) + comment.append(line.replace(cmnt_end, "")) + comments_list.append("".join(comment)) comment = [] elif def_tag in line or not has_pi: has_pi = False @@ -69,25 +70,24 @@ def separate_pi_comments(input_file): comment.append(line) else: xml_lines.append(line) - return comments_list, ''.join(xml_lines) + return comments_list, "".join(xml_lines) # Collected: https://dustinoprea.com/2019/01/22/python-parsing-xml-and-retaining-the-comments/ class _CommentedTreeBuilder(ET.TreeBuilder): - def comment(self, text): """ defining comment builder in TreeBuilder """ - self.start('!--', {}) + self.start("!--", {}) self.data(text) - self.end('--') + self.end("--") def parse(filepath): """ - Construct parse function for modified tree builder for including modified TreeBuilder - and rebuilding XMLParser. + Construct parse function for modified tree builder for including modified TreeBuilder + and rebuilding XMLParser. """ comments, xml_str = separate_pi_comments(filepath) ctb = _CommentedTreeBuilder() @@ -97,7 +97,7 @@ def parse(filepath): def handle_mapping_char(text, depth=-1, skip_n_line_on_top=False): - """Check for ":" char and replace it by "':'". """ + """Check for ":" char and replace it by "':'".""" escape_char = get_yaml_escape_char_dict() for esc_key, val in escape_char.items(): @@ -119,23 +119,23 @@ def add_new_line_with_pipe_on_top(text, depth): char_list_to_add_new_line_on_top_of_text = [":"] for char in char_list_to_add_new_line_on_top_of_text: if char in text: - return '|' + '\n' + depth * DEPTH_SIZE + text + return "|" + "\n" + depth * DEPTH_SIZE + text return text # pylint: disable=too-many-instance-attributes -class Nxdl2yaml(): +class Nxdl2yaml: """ - Parse XML file and print a YML file + Parse XML file and print a YML file """ def __init__( - self, - symbol_list: List[str], - root_level_definition: List[str], - root_level_doc='', - root_level_symbols=''): - + self, + symbol_list: List[str], + root_level_definition: List[str], + root_level_doc="", + root_level_symbols="", + ): # updated part of yaml_dict self.found_definition = False self.root_level_doc = root_level_doc @@ -157,7 +157,7 @@ def __init__( def print_yml(self, input_file, output_yml, verbose): """ - Parse an XML file provided as input and print a YML file + Parse an XML file provided as input and print a YML file """ if os.path.isfile(output_yml): os.remove(output_yml) @@ -165,7 +165,7 @@ def print_yml(self, input_file, output_yml, verbose): depth = 0 self.pi_comments, root = parse(input_file) - xml_tree = {'tree': root, 'node': root} + xml_tree = {"tree": root, "node": root} self.xmlparse(output_yml, xml_tree, depth, verbose) def handle_symbols(self, depth, node): @@ -177,47 +177,55 @@ def handle_symbols(self, depth, node): f"{node.text.strip() if node.text else ''}" ) depth += 1 - last_comment = '' + last_comment = "" sbl_doc_cmnt_list = [] # Comments that come above symbol tag symbol_cmnt_list = [] for child in list(node): tag = remove_namespace_from_tag(child.tag) if tag == CMNT_TAG and self.include_comment: - last_comment = self.comvert_to_ymal_comment(depth * DEPTH_SIZE, child.text) - if tag == 'doc': + last_comment = self.comvert_to_ymal_comment( + depth * DEPTH_SIZE, child.text + ) + if tag == "doc": symbol_cmnt_list.append(last_comment) # The bellow line is for handling lenth of 'symbol_comments' and # 'symbol_doc_comments'. Otherwise print_root_level_info() gets inconsistency # over for the loop while writting comment on file - sbl_doc_cmnt_list.append('') - last_comment = '' - self.symbol_list.append(self.handle_not_root_level_doc(depth, - text=child.text)) - elif tag == 'symbol': + sbl_doc_cmnt_list.append("") + last_comment = "" + self.symbol_list.append( + self.handle_not_root_level_doc(depth, text=child.text) + ) + elif tag == "symbol": # place holder is symbol name symbol_cmnt_list.append(last_comment) - last_comment = '' - if 'doc' in child.attrib: + last_comment = "" + if "doc" in child.attrib: self.symbol_list.append( - self.handle_not_root_level_doc(depth, - tag=child.attrib['name'], - text=child.attrib['doc'])) + self.handle_not_root_level_doc( + depth, tag=child.attrib["name"], text=child.attrib["doc"] + ) + ) else: for symbol_doc in list(child): tag = remove_namespace_from_tag(symbol_doc.tag) if tag == CMNT_TAG and self.include_comment: - last_comment = self.comvert_to_ymal_comment(depth * DEPTH_SIZE, - symbol_doc.text) - if tag == 'doc': + last_comment = self.comvert_to_ymal_comment( + depth * DEPTH_SIZE, symbol_doc.text + ) + if tag == "doc": sbl_doc_cmnt_list.append(last_comment) - last_comment = '' + last_comment = "" self.symbol_list.append( - self.handle_not_root_level_doc(depth, - tag=child.attrib['name'], - text=symbol_doc.text)) - self.store_root_level_comments('symbol_doc_comments', sbl_doc_cmnt_list) - self.store_root_level_comments('symbol_comments', symbol_cmnt_list) + self.handle_not_root_level_doc( + depth, + tag=child.attrib["name"], + text=symbol_doc.text, + ) + ) + self.store_root_level_comments("symbol_doc_comments", sbl_doc_cmnt_list) + self.store_root_level_comments("symbol_comments", symbol_cmnt_list) def store_root_level_comments(self, holder, comment): """Store yaml text or section line and the comments inteded for that lines or section""" @@ -226,13 +234,13 @@ def store_root_level_comments(self, holder, comment): def handle_definition(self, node): """ - Handle definition group and its attributes - NOTE: Here we tried to store the order of the xml element attributes. So that we get - exactly the same file in nxdl from yaml. + Handle definition group and its attributes + NOTE: Here we tried to store the order of the xml element attributes. So that we get + exactly the same file in nxdl from yaml. """ # pylint: disable=consider-using-f-string # self.root_level_definition[0] = '' - keyword = '' + keyword = "" # tmp_word for reseving the location tmp_word = "#xx#" attribs = node.attrib @@ -249,15 +257,14 @@ def handle_definition(self, node): if keyword_order == -1: self.root_level_definition.append(tmp_word) keyword_order = self.root_level_definition.index(tmp_word) - elif 'schemaLocation' not in item \ - and 'extends' != item: + elif "schemaLocation" not in item and "extends" != item: text = f"{item}: {attribs[item]}" self.root_level_definition.append(text) self.root_level_definition[keyword_order] = f"{keyword}:" def handle_root_level_doc(self, node): """ - Handle the documentation field found at root level. + Handle the documentation field found at root level. """ # tag = remove_namespace_from_tag(node.tag) text = node.text @@ -265,7 +272,7 @@ def handle_root_level_doc(self, node): self.root_level_doc = text # pylint: disable=too-many-branches - def handle_not_root_level_doc(self, depth, text, tag='doc', file_out=None): + def handle_not_root_level_doc(self, depth, text, tag="doc", file_out=None): """ Handle docs field along the yaml file. In this function we also tried to keep the track of intended indentation. E.g. the bollow doc block. @@ -280,7 +287,7 @@ def handle_not_root_level_doc(self, depth, text, tag='doc', file_out=None): text = handle_mapping_char(text, -1, True) if "\n" in text: # To remove '\n' character as it will be added before text. - text = cleaning_empty_lines(text.split('\n')) + text = cleaning_empty_lines(text.split("\n")) text_tmp = [] yaml_indent_n = len((depth + 1) * DEPTH_SIZE) # Find indentaion in the first text line with alphabet @@ -288,12 +295,12 @@ def handle_not_root_level_doc(self, depth, text, tag='doc', file_out=None): while tmp_i != -1: first_line_indent_n = 0 # Taking care of empty text whitout any character - if len(text) == 1 and text[0] == '': + if len(text) == 1 and text[0] == "": break for ch_ in text[tmp_i]: - if ch_ == ' ': + if ch_ == " ": first_line_indent_n = first_line_indent_n + 1 - elif ch_ != '': + elif ch_ != "": tmp_i = -2 break tmp_i = tmp_i + 1 @@ -314,23 +321,23 @@ def handle_not_root_level_doc(self, depth, text, tag='doc', file_out=None): line_indent_n = 0 # Collect first empty space without alphabate for ch_ in line: - if ch_ == ' ': + if ch_ == " ": line_indent_n = line_indent_n + 1 else: break line_indent_n = line_indent_n + indent_diff if line_indent_n < yaml_indent_n: # if line still under yaml identation - text_tmp.append(yaml_indent_n * ' ' + line.strip()) + text_tmp.append(yaml_indent_n * " " + line.strip()) else: - text_tmp.append(line_indent_n * ' ' + line.strip()) + text_tmp.append(line_indent_n * " " + line.strip()) - text = '\n' + '\n'.join(text_tmp) + text = "\n" + "\n".join(text_tmp) if "}" in tag: tag = remove_namespace_from_tag(tag) indent = depth * DEPTH_SIZE elif text: - text = '\n' + (depth + 1) * DEPTH_SIZE + text.strip() + text = "\n" + (depth + 1) * DEPTH_SIZE + text.strip() if "}" in tag: tag = remove_namespace_from_tag(tag) indent = depth * DEPTH_SIZE @@ -360,31 +367,33 @@ def print_root_level_doc(self, file_out): """ indent = 0 * DEPTH_SIZE - if ('root_doc' in self.root_level_comment - and self.root_level_comment['root_doc'] != ''): - text = self.root_level_comment['root_doc'] + if ( + "root_doc" in self.root_level_comment + and self.root_level_comment["root_doc"] != "" + ): + text = self.root_level_comment["root_doc"] self.write_out(indent, text, file_out) text = self.root_level_doc self.write_out(indent, text, file_out) - self.root_level_doc = '' + self.root_level_doc = "" def comvert_to_ymal_comment(self, indent, text): """ - Convert into yaml comment by adding exta '#' char in front of comment lines + Convert into yaml comment by adding exta '#' char in front of comment lines """ - lines = text.split('\n') + lines = text.split("\n") mod_lines = [] for line in lines: line = line.strip() - if line and line[0] != '#': - line = indent + '# ' + line + if line and line[0] != "#": + line = indent + "# " + line mod_lines.append(line) elif line: line = indent + line mod_lines.append(line) # The starting '\n' to keep multiple comments separate - return '\n' + '\n'.join(mod_lines) + return "\n" + "\n".join(mod_lines) def print_root_level_info(self, depth, file_out): """ @@ -403,40 +412,58 @@ def print_root_level_info(self, depth, file_out): has_categoty = True if not has_categoty: - raise ValueError("Definition dose not get any category from 'base or application'.") + raise ValueError( + "Definition dose not get any category from 'base or application'." + ) self.print_root_level_doc(file_out) - if 'symbols' in self.root_level_comment and self.root_level_comment['symbols'] != '': + if ( + "symbols" in self.root_level_comment + and self.root_level_comment["symbols"] != "" + ): indent = depth * DEPTH_SIZE - text = self.root_level_comment['symbols'] + text = self.root_level_comment["symbols"] self.write_out(indent, text, file_out) if self.root_level_symbols: - self.write_out(indent=0 * DEPTH_SIZE, text=self.root_level_symbols, file_out=file_out) + self.write_out( + indent=0 * DEPTH_SIZE, text=self.root_level_symbols, file_out=file_out + ) # symbol_list include 'symbols doc', and all 'symbol' for ind, symbol in enumerate(self.symbol_list): # Taking care of comments that come on to of 'symbols doc' and 'symbol' - if 'symbol_comments' in self.root_level_comment and \ - self.root_level_comment['symbol_comments'][ind] != '': + if ( + "symbol_comments" in self.root_level_comment + and self.root_level_comment["symbol_comments"][ind] != "" + ): indent = depth * DEPTH_SIZE - self.write_out(indent, - self.root_level_comment['symbol_comments'][ind], file_out) - if 'symbol_doc_comments' in self.root_level_comment and \ - self.root_level_comment['symbol_doc_comments'][ind] != '': - + self.write_out( + indent, + self.root_level_comment["symbol_comments"][ind], + file_out, + ) + if ( + "symbol_doc_comments" in self.root_level_comment + and self.root_level_comment["symbol_doc_comments"][ind] != "" + ): indent = depth * DEPTH_SIZE - self.write_out(indent, - self.root_level_comment['symbol_doc_comments'][ind], file_out) + self.write_out( + indent, + self.root_level_comment["symbol_doc_comments"][ind], + file_out, + ) self.write_out(indent=(0 * DEPTH_SIZE), text=symbol, file_out=file_out) if len(self.pi_comments) > 1: indent = DEPTH_SIZE * depth # The first comment is top level copy-right doc string for comment in self.pi_comments[1:]: - self.write_out(indent, self.comvert_to_ymal_comment(indent, comment), file_out) + self.write_out( + indent, self.comvert_to_ymal_comment(indent, comment), file_out + ) if self.root_level_definition: # Soring NXname for writting end of the definition attributes - nx_name = '' + nx_name = "" for defs in self.root_level_definition: - if 'NX' in defs and defs[-1] == ':': + if "NX" in defs and defs[-1] == ":": nx_name = defs continue if defs in ("category: application", "category: base"): @@ -447,55 +474,77 @@ def print_root_level_info(self, depth, file_out): def handle_exists(self, exists_dict, key, val): """ - Create exist component as folows: + Create exist component as folows: - {'min' : value for min, - 'max' : value for max, - 'optional' : value for optional} + {'min' : value for min, + 'max' : value for max, + 'optional' : value for optional} - This is created separately so that the keys stays in order. + This is created separately so that the keys stays in order. """ if not val: - val = '' + val = "" else: val = str(val) - if 'minOccurs' == key: - exists_dict['minOccurs'] = ['min', val] - if 'maxOccurs' == key: - exists_dict['maxOccurs'] = ['max', val] - if 'optional' == key: - exists_dict['optional'] = ['optional', val] - if 'recommended' == key: - exists_dict['recommended'] = ['recommended', val] - if 'required' == key: - exists_dict['required'] = ['required', val] + if "minOccurs" == key: + exists_dict["minOccurs"] = ["min", val] + if "maxOccurs" == key: + exists_dict["maxOccurs"] = ["max", val] + if "optional" == key: + exists_dict["optional"] = ["optional", val] + if "recommended" == key: + exists_dict["recommended"] = ["recommended", val] + if "required" == key: + exists_dict["required"] = ["required", val] # pylint: disable=too-many-branches, consider-using-f-string def handle_group_or_field(self, depth, node, file_out): """Handle all the possible attributes that come along a field or group""" - allowed_attr = ['optional', 'recommended', 'name', 'type', 'axes', 'axis', 'data_offset', - 'interpretation', 'long_name', 'maxOccurs', 'minOccurs', 'nameType', - 'optional', 'primary', 'signal', 'stride', 'units', 'required', - 'deprecated', 'exists'] + allowed_attr = [ + "optional", + "recommended", + "name", + "type", + "axes", + "axis", + "data_offset", + "interpretation", + "long_name", + "maxOccurs", + "minOccurs", + "nameType", + "optional", + "primary", + "signal", + "stride", + "units", + "required", + "deprecated", + "exists", + ] name_type = "" node_attr = node.attrib rm_key_list = [] # Maintain order: name and type in form name(type) or (type)name that come first for key, val in node_attr.items(): - if key == 'name': + if key == "name": name_type = name_type + val rm_key_list.append(key) - if key == 'type': + if key == "type": name_type = name_type + "(%s)" % val rm_key_list.append(key) if not name_type: - raise ValueError(f"No 'name' or 'type' hase been found. But, 'group' or 'field' " - f"must have at list a nme.We got attributes: {node_attr}") - file_out.write('{indent}{name_type}:\n'.format( - indent=depth * DEPTH_SIZE, - name_type=name_type)) + raise ValueError( + f"No 'name' or 'type' hase been found. But, 'group' or 'field' " + f"must have at list a nme.We got attributes: {node_attr}" + ) + file_out.write( + "{indent}{name_type}:\n".format( + indent=depth * DEPTH_SIZE, name_type=name_type + ) + ) for key in rm_key_list: del node_attr[key] @@ -505,31 +554,35 @@ def handle_group_or_field(self, depth, node, file_out): exists_dict = {} for key, val in node_attr.items(): # As both 'minOccurs', 'maxOccurs' and optionality move to the 'exists' - if key in ['minOccurs', 'maxOccurs', 'optional', 'recommended', 'required']: - if 'exists' not in tmp_dict: - tmp_dict['exists'] = [] + if key in ["minOccurs", "maxOccurs", "optional", "recommended", "required"]: + if "exists" not in tmp_dict: + tmp_dict["exists"] = [] self.handle_exists(exists_dict, key, val) - elif key == 'units': - tmp_dict['unit'] = str(val) + elif key == "units": + tmp_dict["unit"] = str(val) else: tmp_dict[key] = str(val) if key not in allowed_attr: - raise ValueError(f"An attribute ({key}) in 'field' or 'group' has been found " - f"that is not allowed. The allowed attr is {allowed_attr}.") + raise ValueError( + f"An attribute ({key}) in 'field' or 'group' has been found " + f"that is not allowed. The allowed attr is {allowed_attr}." + ) if exists_dict: for key, val in exists_dict.items(): - if key in ['minOccurs', 'maxOccurs']: - tmp_dict['exists'] = tmp_dict['exists'] + val - elif key in ['optional', 'recommended', 'required']: - tmp_dict['exists'] = key + if key in ["minOccurs", "maxOccurs"]: + tmp_dict["exists"] = tmp_dict["exists"] + val + elif key in ["optional", "recommended", "required"]: + tmp_dict["exists"] = key depth_ = depth + 1 for key, val in tmp_dict.items(): # Increase depth size inside handle_map...() for writting text with one # more indentation. - file_out.write(f'{depth_ * DEPTH_SIZE}{key}: ' - f'{handle_mapping_char(val, depth_ + 1, False)}\n') + file_out.write( + f"{depth_ * DEPTH_SIZE}{key}: " + f"{handle_mapping_char(val, depth_ + 1, False)}\n" + ) # pylint: disable=too-many-branches, too-many-locals def handle_dimension(self, depth, node, file_out): @@ -540,33 +593,35 @@ def handle_dimension(self, depth, node, file_out): and attributes of dim has been handled inside this function here. """ # pylint: disable=consider-using-f-string - possible_dim_attrs = ['ref', 'required', - 'incr', 'refindex'] - possible_dimemsion_attrs = ['rank'] + possible_dim_attrs = ["ref", "required", "incr", "refindex"] + possible_dimemsion_attrs = ["rank"] # taking care of Dimension tag file_out.write( - '{indent}{tag}:\n'.format( - indent=depth * DEPTH_SIZE, - tag=node.tag.split("}", 1)[1])) + "{indent}{tag}:\n".format( + indent=depth * DEPTH_SIZE, tag=node.tag.split("}", 1)[1] + ) + ) # Taking care of dimension attributes for attr, value in node.attrib.items(): if attr in possible_dimemsion_attrs and not isinstance(value, dict): indent = (depth + 1) * DEPTH_SIZE - file_out.write(f'{indent}{attr}: {value}\n') + file_out.write(f"{indent}{attr}: {value}\n") else: - raise ValueError(f"Dimension has got an attribute {attr} that is not valid." - f"Current the allowd atributes are {possible_dimemsion_attrs}." - f" Please have a look") + raise ValueError( + f"Dimension has got an attribute {attr} that is not valid." + f"Current the allowd atributes are {possible_dimemsion_attrs}." + f" Please have a look" + ) # taking carew of dimension doc for child in list(node): tag = remove_namespace_from_tag(child.tag) - if tag == 'doc': + if tag == "doc": text = self.handle_not_root_level_doc(depth + 1, child.text) file_out.write(text) node.remove(child) - dim_index_value = '' + dim_index_value = "" dim_other_parts = {} dim_cmnt_node = [] # taking care of dim and doc childs of dimension @@ -574,11 +629,12 @@ def handle_dimension(self, depth, node, file_out): tag = remove_namespace_from_tag(child.tag) child_attrs = child.attrib # taking care of index and value attributes - if tag == ('dim'): + if tag == ("dim"): # taking care of index and value in format [[index, value]] - dim_index_value = dim_index_value + '[{index}, {value}], '.format( - index=child_attrs['index'] if "index" in child_attrs else '', - value=child_attrs['value'] if "value" in child_attrs else '') + dim_index_value = dim_index_value + "[{index}, {value}], ".format( + index=child_attrs["index"] if "index" in child_attrs else "", + value=child_attrs["value"] if "value" in child_attrs else "", + ) if "index" in child_attrs: del child_attrs["index"] if "value" in child_attrs: @@ -587,7 +643,7 @@ def handle_dimension(self, depth, node, file_out): # Taking care of doc comes as child of dim for cchild in list(child): ttag = cchild.tag.split("}", 1)[1] - if ttag == ('doc'): + if ttag == ("doc"): if ttag not in dim_other_parts: dim_other_parts[ttag] = [] text = cchild.text @@ -612,25 +668,30 @@ def handle_dimension(self, depth, node, file_out): self.handel_comment(depth + 1, ch_nd, file_out) # index and value attributes of dim elements file_out.write( - '{indent}dim: [{value}]\n'.format( - indent=(depth + 1) * DEPTH_SIZE, - value=dim_index_value[:-2] or '')) + "{indent}dim: [{value}]\n".format( + indent=(depth + 1) * DEPTH_SIZE, value=dim_index_value[:-2] or "" + ) + ) # Write the attributes, except index and value, and doc of dim as child of dim_parameter. # But tthe doc or attributes for each dim come inside list according to the order of dim. if dim_other_parts: file_out.write( - '{indent}dim_parameters:\n'.format( - indent=(depth + 1) * DEPTH_SIZE)) + "{indent}dim_parameters:\n".format(indent=(depth + 1) * DEPTH_SIZE) + ) # depth = depth + 2 dim_paramerter has child such as doc of dim indent = (depth + 2) * DEPTH_SIZE for key, value in dim_other_parts.items(): - if key == 'doc': - value = self.handle_not_root_level_doc(depth + 2, str(value), key, file_out) + if key == "doc": + value = self.handle_not_root_level_doc( + depth + 2, str(value), key, file_out + ) else: # Increase depth size inside handle_map...() for writting text with one # more indentation. - file_out.write(f"{indent}{key}: " - f"{handle_mapping_char(value, depth + 3, False)}\n") + file_out.write( + f"{indent}{key}: " + f"{handle_mapping_char(value, depth + 3, False)}\n" + ) def handle_enumeration(self, depth, node, file_out): """ @@ -642,7 +703,7 @@ def handle_enumeration(self, depth, node, file_out): If no doc are inherited in the enumeration items, a list of the items is given for the enumeration list. - """ + """ # pylint: disable=consider-using-f-string check_doc = [] @@ -652,37 +713,46 @@ def handle_enumeration(self, depth, node, file_out): # pylint: disable=too-many-nested-blocks if check_doc: file_out.write( - '{indent}{tag}: \n'.format( - indent=depth * DEPTH_SIZE, - tag=node.tag.split("}", 1)[1])) + "{indent}{tag}: \n".format( + indent=depth * DEPTH_SIZE, tag=node.tag.split("}", 1)[1] + ) + ) for child in list(node): tag = remove_namespace_from_tag(child.tag) itm_depth = depth + 1 - if tag == ('item'): + if tag == ("item"): file_out.write( - '{indent}{value}: \n'.format( - indent=(itm_depth) * DEPTH_SIZE, - value=child.attrib['value'])) + "{indent}{value}: \n".format( + indent=(itm_depth) * DEPTH_SIZE, value=child.attrib["value"] + ) + ) if list(child): for item_doc in list(child): - if remove_namespace_from_tag(item_doc.tag) == 'doc': + if remove_namespace_from_tag(item_doc.tag) == "doc": item_doc_depth = itm_depth + 1 - self.handle_not_root_level_doc(item_doc_depth, item_doc.text, - item_doc.tag, file_out) - if (remove_namespace_from_tag(item_doc.tag) == CMNT_TAG - and self.include_comment): + self.handle_not_root_level_doc( + item_doc_depth, + item_doc.text, + item_doc.tag, + file_out, + ) + if ( + remove_namespace_from_tag(item_doc.tag) == CMNT_TAG + and self.include_comment + ): self.handel_comment(itm_depth + 1, item_doc, file_out) if tag == CMNT_TAG and self.include_comment: self.handel_comment(itm_depth + 1, child, file_out) else: - enum_list = '' + enum_list = "" remove_nodes = [] for item_child in list(node): tag = remove_namespace_from_tag(item_child.tag) - if tag == ('item'): - enum_list = enum_list + '{value}, '.format( - value=item_child.attrib['value']) + if tag == ("item"): + enum_list = enum_list + "{value}, ".format( + value=item_child.attrib["value"] + ) if tag == CMNT_TAG and self.include_comment: self.handel_comment(depth, item_child, file_out) remove_nodes.append(item_child) @@ -690,134 +760,162 @@ def handle_enumeration(self, depth, node, file_out): node.remove(ch_node) file_out.write( - '{indent}{tag}: [{enum_list}]\n'.format( + "{indent}{tag}: [{enum_list}]\n".format( indent=depth * DEPTH_SIZE, tag=remove_namespace_from_tag(node.tag), - enum_list=enum_list[:-2] or '')) + enum_list=enum_list[:-2] or "", + ) + ) def handle_attributes(self, depth, node, file_out): """Handle the attributes parsed from the xml file""" - allowed_attr = ['name', 'type', 'units', 'nameType', 'recommended', 'optional', - 'minOccurs', 'maxOccurs', 'deprecated'] + allowed_attr = [ + "name", + "type", + "units", + "nameType", + "recommended", + "optional", + "minOccurs", + "maxOccurs", + "deprecated", + ] name = "" node_attr = node.attrib - if 'name' in node_attr: + if "name" in node_attr: pass else: raise ValueError("Attribute must have an name key.") rm_key_list = [] # Maintain order: name and type in form name(type) or (type)name that come first for key, val in node_attr.items(): - if key == 'name': + if key == "name": name = val rm_key_list.append(key) for key in rm_key_list: del node_attr[key] - file_out.write('{indent}{escapesymbol}{name}:\n'.format( - indent=depth * DEPTH_SIZE, - escapesymbol=r'\@', - name=name)) + file_out.write( + "{indent}{escapesymbol}{name}:\n".format( + indent=depth * DEPTH_SIZE, escapesymbol=r"\@", name=name + ) + ) tmp_dict = {} exists_dict = {} for key, val in node_attr.items(): # As both 'minOccurs', 'maxOccurs' and optionality move to the 'exists' - if key in ['minOccurs', 'maxOccurs', 'optional', 'recommended', 'required']: - if 'exists' not in tmp_dict: - tmp_dict['exists'] = [] + if key in ["minOccurs", "maxOccurs", "optional", "recommended", "required"]: + if "exists" not in tmp_dict: + tmp_dict["exists"] = [] self.handle_exists(exists_dict, key, val) - elif key == 'units': - tmp_dict['unit'] = val + elif key == "units": + tmp_dict["unit"] = val else: tmp_dict[key] = val if key not in allowed_attr: - raise ValueError(f"An attribute ({key}) has been found that is not allowed." - f"The allowed attr is {allowed_attr}.") + raise ValueError( + f"An attribute ({key}) has been found that is not allowed." + f"The allowed attr is {allowed_attr}." + ) has_min_max = False has_opt_reco_requ = False if exists_dict: for key, val in exists_dict.items(): - if key in ['minOccurs', 'maxOccurs']: - tmp_dict['exists'] = tmp_dict['exists'] + val + if key in ["minOccurs", "maxOccurs"]: + tmp_dict["exists"] = tmp_dict["exists"] + val has_min_max = True - elif key in ['optional', 'recommended', 'required']: - tmp_dict['exists'] = key + elif key in ["optional", "recommended", "required"]: + tmp_dict["exists"] = key has_opt_reco_requ = True if has_min_max and has_opt_reco_requ: - raise ValueError("Optionality 'exists' can take only either from ['minOccurs'," - " 'maxOccurs'] or from ['optional', 'recommended', 'required']" - ". But not from both of the groups together. Please check in" - " attributes") + raise ValueError( + "Optionality 'exists' can take only either from ['minOccurs'," + " 'maxOccurs'] or from ['optional', 'recommended', 'required']" + ". But not from both of the groups together. Please check in" + " attributes" + ) depth_ = depth + 1 for key, val in tmp_dict.items(): # Increase depth size inside handle_map...() for writting text with one # more indentation. - file_out.write(f'{depth_ * DEPTH_SIZE}{key}: ' - f'{handle_mapping_char(val, depth_ + 1, False)}\n') + file_out.write( + f"{depth_ * DEPTH_SIZE}{key}: " + f"{handle_mapping_char(val, depth_ + 1, False)}\n" + ) def handel_link(self, depth, node, file_out): """ - Handle link elements of nxdl + Handle link elements of nxdl """ - possible_link_attrs = ['name', 'target', 'napimount'] + possible_link_attrs = ["name", "target", "napimount"] node_attr = node.attrib # Handle special cases - if 'name' in node_attr: - file_out.write('{indent}{name}(link):\n'.format( - indent=depth * DEPTH_SIZE, - name=node_attr['name'] or '')) - del node_attr['name'] + if "name" in node_attr: + file_out.write( + "{indent}{name}(link):\n".format( + indent=depth * DEPTH_SIZE, name=node_attr["name"] or "" + ) + ) + del node_attr["name"] depth_ = depth + 1 # Handle general cases for attr_key, val in node_attr.items(): if attr_key in possible_link_attrs: - file_out.write('{indent}{attr}: {value}\n'.format( - indent=depth_ * DEPTH_SIZE, - attr=attr_key, - value=val)) + file_out.write( + "{indent}{attr}: {value}\n".format( + indent=depth_ * DEPTH_SIZE, attr=attr_key, value=val + ) + ) else: - raise ValueError(f"An anexpected attribute '{attr_key}' of link has found." - f"At this moment the alloed keys are {possible_link_attrs}") + raise ValueError( + f"An anexpected attribute '{attr_key}' of link has found." + f"At this moment the alloed keys are {possible_link_attrs}" + ) def handel_choice(self, depth, node, file_out): """ - Handle choice element which is a parent node of group. + Handle choice element which is a parent node of group. """ possible_attr = [] node_attr = node.attrib # Handle special casees - if 'name' in node_attr: - file_out.write('{indent}{attr}(choice): \n'.format( - indent=depth * DEPTH_SIZE, - attr=node_attr['name'])) - del node_attr['name'] + if "name" in node_attr: + file_out.write( + "{indent}{attr}(choice): \n".format( + indent=depth * DEPTH_SIZE, attr=node_attr["name"] + ) + ) + del node_attr["name"] depth_ = depth + 1 # Taking care of general attrinutes. Though, still no attrinutes have found, # but could be used for future for attr in node_attr.items(): if attr in possible_attr: - file_out.write('{indent}{attr}: {value}\n'.format( - indent=depth_ * DEPTH_SIZE, - attr=attr, - value=node_attr[attr])) + file_out.write( + "{indent}{attr}: {value}\n".format( + indent=depth_ * DEPTH_SIZE, attr=attr, value=node_attr[attr] + ) + ) else: - raise ValueError(f"An unexpected attribute '{attr}' of 'choice' has been found." - f"At this moment attributes for choice {possible_attr}") + raise ValueError( + f"An unexpected attribute '{attr}' of 'choice' has been found." + f"At this moment attributes for choice {possible_attr}" + ) def handel_comment(self, depth, node, file_out): """ - Collect comment element and pass to write_out function + Collect comment element and pass to write_out function """ indent = depth * DEPTH_SIZE if self.is_last_element_comment: @@ -834,10 +932,10 @@ def recursion_in_xml_tree(self, depth, xml_tree, output_yml, verbose): behaviour is not triggered as we already handled the symbols' childs. """ - tree = xml_tree['tree'] - node = xml_tree['node'] + tree = xml_tree["tree"] + node = xml_tree["node"] for child in list(node): - xml_tree_children = {'tree': tree, 'node': child} + xml_tree_children = {"tree": tree, "node": child} self.xmlparse(output_yml, xml_tree_children, depth, verbose) # pylint: disable=too-many-branches, too-many-statements @@ -846,63 +944,65 @@ def xmlparse(self, output_yml, xml_tree, depth, verbose): Main of the nxdl2yaml converter. It parses XML tree, then prints recursively each level of the tree """ - tree = xml_tree['tree'] - node = xml_tree['node'] + tree = xml_tree["tree"] + node = xml_tree["node"] if verbose: - sys.stdout.write(f'Node tag: {remove_namespace_from_tag(node.tag)}\n') - sys.stdout.write(f'Attributes: {node.attrib}\n') + sys.stdout.write(f"Node tag: {remove_namespace_from_tag(node.tag)}\n") + sys.stdout.write(f"Attributes: {node.attrib}\n") with open(output_yml, "a", encoding="utf-8") as file_out: tag = remove_namespace_from_tag(node.tag) - if tag == 'definition': + if tag == "definition": self.found_definition = True self.handle_definition(node) # Taking care of root level doc and symbols remove_cmnt_n = None - last_comment = '' + last_comment = "" for child in list(node): tag_tmp = remove_namespace_from_tag(child.tag) if tag_tmp == CMNT_TAG and self.include_comment: - last_comment = self.comvert_to_ymal_comment(depth * DEPTH_SIZE, child.text) + last_comment = self.comvert_to_ymal_comment( + depth * DEPTH_SIZE, child.text + ) remove_cmnt_n = child - if tag_tmp == 'doc': - self.store_root_level_comments('root_doc', last_comment) - last_comment = '' + if tag_tmp == "doc": + self.store_root_level_comments("root_doc", last_comment) + last_comment = "" self.handle_root_level_doc(child) node.remove(child) if remove_cmnt_n is not None: node.remove(remove_cmnt_n) remove_cmnt_n = None - if tag_tmp == 'symbols': - self.store_root_level_comments('symbols', last_comment) - last_comment = '' + if tag_tmp == "symbols": + self.store_root_level_comments("symbols", last_comment) + last_comment = "" self.handle_symbols(depth, child) node.remove(child) if remove_cmnt_n is not None: node.remove(remove_cmnt_n) remove_cmnt_n = None - if tag == ('doc') and depth != 1: + if tag == ("doc") and depth != 1: parent = get_node_parent_info(tree, node)[0] doc_parent = remove_namespace_from_tag(parent.tag) - if doc_parent != 'item': - self.handle_not_root_level_doc(depth, text=node.text, - tag=node.tag, - file_out=file_out) + if doc_parent != "item": + self.handle_not_root_level_doc( + depth, text=node.text, tag=node.tag, file_out=file_out + ) if self.found_definition is True and self.root_level_doc: self.print_root_level_info(depth, file_out) # End of print root-level definitions in file - if tag in ('field', 'group') and depth != 0: + if tag in ("field", "group") and depth != 0: self.handle_group_or_field(depth, node, file_out) - if tag == ('enumeration'): + if tag == ("enumeration"): self.handle_enumeration(depth, node, file_out) - if tag == ('attribute'): + if tag == ("attribute"): self.handle_attributes(depth, node, file_out) - if tag == ('dimensions'): + if tag == ("dimensions"): self.handle_dimension(depth, node, file_out) - if tag == ('link'): + if tag == ("link"): self.handel_link(depth, node, file_out) - if tag == ('choice'): + if tag == ("choice"): self.handel_choice(depth, node, file_out) if tag == CMNT_TAG and self.include_comment: self.handel_comment(depth, node, file_out) @@ -913,24 +1013,23 @@ def xmlparse(self, output_yml, xml_tree, depth, verbose): def compare_niac_and_my(tree, tree2, verbose, node, root_no_duplicates): """This function creates two trees with Niac XML file and My XML file. -The main aim is to compare the two trees and create a new one that is the -union of the two initial trees. - -""" + The main aim is to compare the two trees and create a new one that is the + union of the two initial trees. + """ root = tree.getroot() root2 = tree2.getroot() attrs_list_niac = [] for nodo in root.iter(node): attrs_list_niac.append(nodo.attrib) if verbose: - sys.stdout.write('Attributes found in Niac file: \n') - sys.stdout.write(str(attrs_list_niac) + '\n') - sys.stdout.write(' \n') - sys.stdout.write('Started merging of Niac and My file... \n') + sys.stdout.write("Attributes found in Niac file: \n") + sys.stdout.write(str(attrs_list_niac) + "\n") + sys.stdout.write(" \n") + sys.stdout.write("Started merging of Niac and My file... \n") for elem in root.iter(node): if verbose: - sys.stdout.write('- Niac element inserted: \n') - sys.stdout.write(str(elem.attrib) + '\n') + sys.stdout.write("- Niac element inserted: \n") + sys.stdout.write(str(elem.attrib) + "\n") index = get_node_parent_info(tree, elem)[1] root_no_duplicates.insert(index, elem) @@ -938,10 +1037,10 @@ def compare_niac_and_my(tree, tree2, verbose, node, root_no_duplicates): index = get_node_parent_info(tree2, elem2)[1] if elem2.attrib not in attrs_list_niac: if verbose: - sys.stdout.write('- My element inserted: \n') - sys.stdout.write(str(elem2.attrib) + '\n') + sys.stdout.write("- My element inserted: \n") + sys.stdout.write(str(elem2.attrib) + "\n") root_no_duplicates.insert(index, elem2) if verbose: - sys.stdout.write(' \n') + sys.stdout.write(" \n") return root_no_duplicates diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py b/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py index db4d4c4644..ca0435e374 100644 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py @@ -21,65 +21,67 @@ # limitations under the License. # +import os import sys +import textwrap import xml.etree.ElementTree as ET from xml.dom import minidom -import os -import textwrap import yaml - +from pynxtools.dataconverter.helpers import remove_namespace_from_tag from pynxtools.nexus import nexus from pynxtools.nyaml2nxdl.comment_collector import CommentCollector -from pynxtools.dataconverter.helpers import remove_namespace_from_tag -from pynxtools.nyaml2nxdl.nyaml2nxdl_helper import (get_yaml_escape_char_reverter_dict, - nx_name_type_resolving, - cleaning_empty_lines, LineLoader) - +from pynxtools.nyaml2nxdl.nyaml2nxdl_helper import LineLoader +from pynxtools.nyaml2nxdl.nyaml2nxdl_helper import cleaning_empty_lines +from pynxtools.nyaml2nxdl.nyaml2nxdl_helper import get_yaml_escape_char_reverter_dict +from pynxtools.nyaml2nxdl.nyaml2nxdl_helper import nx_name_type_resolving # pylint: disable=too-many-lines, global-statement, invalid-name -DOM_COMMENT = ("\n" - "# NeXus - Neutron and X-ray Common Data Format\n" - "# \n" - "# Copyright (C) 2014-2022 NeXus International Advisory Committee (NIAC)\n" - "# \n" - "# This library is free software; you can redistribute it and/or\n" - "# modify it under the terms of the GNU Lesser General Public\n" - "# License as published by the Free Software Foundation; either\n" - "# version 3 of the License, or (at your option) any later version.\n" - "#\n" - "# This library is distributed in the hope that it will be useful,\n" - "# but WITHOUT ANY WARRANTY; without even the implied warranty of\n" - "# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU\n" - "# Lesser General Public License for more details.\n" - "#\n" - "# You should have received a copy of the GNU Lesser General Public\n" - "# License along with this library; if not, write to the Free Software\n" - "# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA\n" - "#\n" - "# For further information, see http://www.nexusformat.org\n") +DOM_COMMENT = ( + "\n" + "# NeXus - Neutron and X-ray Common Data Format\n" + "# \n" + "# Copyright (C) 2014-2022 NeXus International Advisory Committee (NIAC)\n" + "# \n" + "# This library is free software; you can redistribute it and/or\n" + "# modify it under the terms of the GNU Lesser General Public\n" + "# License as published by the Free Software Foundation; either\n" + "# version 3 of the License, or (at your option) any later version.\n" + "#\n" + "# This library is distributed in the hope that it will be useful,\n" + "# but WITHOUT ANY WARRANTY; without even the implied warranty of\n" + "# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU\n" + "# Lesser General Public License for more details.\n" + "#\n" + "# You should have received a copy of the GNU Lesser General Public\n" + "# License along with this library; if not, write to the Free Software\n" + "# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA\n" + "#\n" + "# For further information, see http://www.nexusformat.org\n" +) NX_CLSS = nexus.get_nx_classes() -NX_NEW_DEFINED_CLASSES = ['NX_COMPLEX'] +NX_NEW_DEFINED_CLASSES = ["NX_COMPLEX"] NX_TYPE_KEYS = nexus.get_nx_attribute_type() -NX_ATTR_IDNT = '\\@' -NX_UNIT_IDNT = 'unit' +NX_ATTR_IDNT = "\\@" +NX_UNIT_IDNT = "unit" DEPTH_SIZE = " " NX_UNIT_TYPES = nexus.get_nx_units() COMMENT_BLOCKS: CommentCollector -CATEGORY = '' # Definition would be either 'base' or 'application' +CATEGORY = "" # Definition would be either 'base' or 'application' def check_for_dom_comment_in_yaml(): - """Check the yaml file has dom comment or dom comment needed to be hard coded. - """ - dignature_keyword_list = ['NeXus', - 'GNU Lesser General Public', - 'Free Software Foundation', - 'Copyright (C)', - 'WITHOUT ANY WARRANTY'] + """Check the yaml file has dom comment or dom comment needed to be hard coded.""" + dignature_keyword_list = [ + "NeXus", + "GNU Lesser General Public", + "Free Software Foundation", + "Copyright (C)", + "WITHOUT ANY WARRANTY", + ] # Check for dom comments in first three comments - dom_comment = '' + dom_comment = "" dom_comment_ind = 1 for ind, comnt in enumerate(COMMENT_BLOCKS[0:5]): cmnt_list = comnt.get_comment_text() @@ -91,7 +93,7 @@ def check_for_dom_comment_in_yaml(): dom_comment_ind = ind for keyword in dignature_keyword_list: if keyword not in text: - dom_comment = '' + dom_comment = "" break if dom_comment: break @@ -120,11 +122,13 @@ def yml_reader(inputfile): if dom_cmnt_frm_yaml: DOM_COMMENT = dom_cmnt_frm_yaml - if 'category' not in loaded_yaml.keys(): - raise ValueError("All definitions should be either 'base' or 'application' category. " - "No category has been found.") + if "category" not in loaded_yaml.keys(): + raise ValueError( + "All definitions should be either 'base' or 'application' category. " + "No category has been found." + ) global CATEGORY - CATEGORY = loaded_yaml['category'] + CATEGORY = loaded_yaml["category"] return loaded_yaml @@ -134,25 +138,27 @@ def check_for_default_attribute_and_value(xml_element): """ # base:Default attributes and value for all elements of base class except dimension element - base_attr_to_val = {'optional': 'true'} + base_attr_to_val = {"optional": "true"} # application: Default attributes and value for all elements of application class except # dimension element - application_attr_to_val = {'optional': 'false'} + application_attr_to_val = {"optional": "false"} # Default attributes and value for dimension element - base_dim_attr_to_val = {'required': 'false'} - application_dim_attr_to_val = {'required': 'true'} + base_dim_attr_to_val = {"required": "false"} + application_dim_attr_to_val = {"required": "true"} # Eligible tag for default attr and value - elegible_tag = ['group', 'field', 'attribute'] + elegible_tag = ["group", "field", "attribute"] def set_default_attribute(xml_elem, default_attr_to_val): for deflt_attr, deflt_val in default_attr_to_val.items(): - if deflt_attr not in xml_elem.attrib \ - and 'maxOccurs' not in xml_elem.attrib \ - and 'minOccurs' not in xml_elem.attrib \ - and 'recommended' not in xml_elem.attrib: + if ( + deflt_attr not in xml_elem.attrib + and "maxOccurs" not in xml_elem.attrib + and "minOccurs" not in xml_elem.attrib + and "recommended" not in xml_elem.attrib + ): xml_elem.set(deflt_attr, deflt_val) for child in list(xml_element): @@ -161,14 +167,13 @@ def set_default_attribute(xml_elem, default_attr_to_val): continue tag = remove_namespace_from_tag(child.tag) - if tag == 'dim' and CATEGORY == 'base': + if tag == "dim" and CATEGORY == "base": set_default_attribute(child, base_dim_attr_to_val) - if tag == 'dim' and CATEGORY == 'application': + if tag == "dim" and CATEGORY == "application": set_default_attribute(child, application_dim_attr_to_val) - if tag in elegible_tag and CATEGORY == 'base': + if tag in elegible_tag and CATEGORY == "base": set_default_attribute(child, base_attr_to_val) - if tag in elegible_tag and CATEGORY == 'application': - + if tag in elegible_tag and CATEGORY == "application": set_default_attribute(child, application_attr_to_val) check_for_default_attribute_and_value(child) @@ -177,50 +182,53 @@ def yml_reader_nolinetag(inputfile): """ pyyaml based parsing of yaml file in python dict """ - with open(inputfile, 'r', encoding="utf-8") as stream: + with open(inputfile, "r", encoding="utf-8") as stream: parsed_yaml = yaml.safe_load(stream) return parsed_yaml def check_for_skiped_attributes(component, value, allowed_attr=None, verbose=False): """ - Check for any attributes have been skipped or not. - NOTE: We should keep in mind about 'doc' + Check for any attributes have been skipped or not. + NOTE: We should keep in mind about 'doc' """ - block_tag = ['enumeration'] + block_tag = ["enumeration"] if value: for attr, val in value.items(): - if attr in ['doc']: + if attr in ["doc"]: continue - if '__line__' in attr or attr in block_tag: + if "__line__" in attr or attr in block_tag: continue - line_number = f'__line__{attr}' + line_number = f"__line__{attr}" if verbose: print(f"__line__ : {value[line_number]}") - if not isinstance(val, dict) \ - and '\\@' not in attr\ - and attr not in allowed_attr\ - and 'NX' not in attr and val: - - raise ValueError(f"An attribute '{attr}' in part '{component}' has been found" - f". Please check arround line '{value[line_number]}. At this " - f"moment. The allowed attrbutes are {allowed_attr}") + if ( + not isinstance(val, dict) + and "\\@" not in attr + and attr not in allowed_attr + and "NX" not in attr + and val + ): + raise ValueError( + f"An attribute '{attr}' in part '{component}' has been found" + f". Please check arround line '{value[line_number]}. At this " + f"moment. The allowed attrbutes are {allowed_attr}" + ) def format_nxdl_doc(string): - """NeXus format for doc string - """ + """NeXus format for doc string""" string = check_for_mapping_char_other(string) - formatted_doc = '' + formatted_doc = "" if "\n" not in string: if len(string) > 80: - wrapped = textwrap.TextWrapper(width=80, - break_long_words=False, - replace_whitespace=False) - string = '\n'.join(wrapped.wrap(string)) - formatted_doc = '\n' + f"{string}" + wrapped = textwrap.TextWrapper( + width=80, break_long_words=False, replace_whitespace=False + ) + string = "\n".join(wrapped.wrap(string)) + formatted_doc = "\n" + f"{string}" else: - text_lines = string.split('\n') + text_lines = string.split("\n") text_lines = cleaning_empty_lines(text_lines) formatted_doc += "\n" + "\n".join(text_lines) if not formatted_doc.endswith("\n"): @@ -234,12 +242,12 @@ def check_for_mapping_char_other(text): Then replace it by ':'. """ if not text: - text = '' + text = "" text = str(text) - if text == 'True': - text = 'true' - if text == 'False': - text = 'false' + if text == "True": + text = "true" + if text == "False": + text = "false" # Some escape char is not valid in yaml libray which is written while writting # yaml file. In the time of writting nxdl revert to that escape char. escape_reverter = get_yaml_escape_char_reverter_dict() @@ -249,26 +257,20 @@ def check_for_mapping_char_other(text): return str(text).strip() -def xml_handle_doc(obj, value: str, - line_number=None, line_loc=None): - """This function creates a 'doc' element instance, and appends it to an existing element - - """ +def xml_handle_doc(obj, value: str, line_number=None, line_loc=None): + """This function creates a 'doc' element instance, and appends it to an existing element""" # global comment_bolcks - doc_elemt = ET.SubElement(obj, 'doc') + doc_elemt = ET.SubElement(obj, "doc") text = format_nxdl_doc(check_for_mapping_char_other(value)).strip() # To keep the doc middle of doc tag. doc_elemt.text = f"\n{text}\n" if line_loc is not None and line_number is not None: - xml_handle_comment(obj, line_number, - line_loc, doc_elemt) + xml_handle_comment(obj, line_number, line_loc, doc_elemt) def xml_handle_units(obj, value): - """This function creates a 'units' element instance, and appends it to an existing element - - """ - obj.set('units', str(value)) + """This function creates a 'units' element instance, and appends it to an existing element""" + obj.set("units", str(value)) # pylint: disable=too-many-branches @@ -276,46 +278,52 @@ def xml_handle_exists(dct, obj, keyword, value): """ This function creates an 'exists' element instance, and appends it to an existing element """ - line_number = f'__line__{keyword}' - assert value is not None, f'Line {dct[line_number]}: exists argument must not be None !' + line_number = f"__line__{keyword}" + assert ( + value is not None + ), f"Line {dct[line_number]}: exists argument must not be None !" if isinstance(value, list): - if len(value) == 4 and value[0] == 'min' and value[2] == 'max': - obj.set('minOccurs', str(value[1])) - if str(value[3]) != 'infty': - obj.set('maxOccurs', str(value[3])) + if len(value) == 4 and value[0] == "min" and value[2] == "max": + obj.set("minOccurs", str(value[1])) + if str(value[3]) != "infty": + obj.set("maxOccurs", str(value[3])) else: - obj.set('maxOccurs', 'unbounded') - elif len(value) == 2 and value[0] == 'min': - obj.set('minOccurs', str(value[1])) - elif len(value) == 2 and value[0] == 'max': - obj.set('maxOccurs', str(value[1])) - elif len(value) == 4 and value[0] == 'max' and value[2] == 'min': - obj.set('minOccurs', str(value[3])) - if str(value[1]) != 'infty': - obj.set('maxOccurs', str(value[3])) + obj.set("maxOccurs", "unbounded") + elif len(value) == 2 and value[0] == "min": + obj.set("minOccurs", str(value[1])) + elif len(value) == 2 and value[0] == "max": + obj.set("maxOccurs", str(value[1])) + elif len(value) == 4 and value[0] == "max" and value[2] == "min": + obj.set("minOccurs", str(value[3])) + if str(value[1]) != "infty": + obj.set("maxOccurs", str(value[3])) else: - obj.set('maxOccurs', 'unbounded') - elif len(value) == 4 and (value[0] != 'min' or value[2] != 'max'): - raise ValueError(f'Line {dct[line_number]}: exists keyword' - f'needs to go either with an optional [recommended] list with two ' - f'entries either [min, ] or [max, ], or a list of four ' - f'entries [min, , max, ] !') + obj.set("maxOccurs", "unbounded") + elif len(value) == 4 and (value[0] != "min" or value[2] != "max"): + raise ValueError( + f"Line {dct[line_number]}: exists keyword" + f"needs to go either with an optional [recommended] list with two " + f"entries either [min, ] or [max, ], or a list of four " + f"entries [min, , max, ] !" + ) else: - raise ValueError(f'Line {dct[line_number]}: exists keyword ' - f'needs to go either with optional, recommended, a list with two ' - f'entries either [min, ] or [max, ], or a list of four ' - f'entries [min, , max, ] !') + raise ValueError( + f"Line {dct[line_number]}: exists keyword " + f"needs to go either with optional, recommended, a list with two " + f"entries either [min, ] or [max, ], or a list of four " + f"entries [min, , max, ] !" + ) else: # This clause take optional in all concept except dimension where 'required' key is allowed # not the 'optional' key. - if value == 'optional': - obj.set('optional', 'true') - elif value == 'recommended': - obj.set('recommended', 'true') - elif value == 'required': - obj.set('optional', 'false') + if value == "optional": + obj.set("optional", "true") + elif value == "recommended": + obj.set("recommended", "true") + elif value == "required": + obj.set("optional", "false") else: - obj.set('minOccurs', '0') + obj.set("minOccurs", "0") # pylint: disable=too-many-branches, too-many-locals, too-many-statements @@ -323,52 +331,59 @@ def xml_handle_group(dct, obj, keyword, value, verbose=False): """ The function deals with group instances """ - line_number = f'__line__{keyword}' + line_number = f"__line__{keyword}" line_loc = dct[line_number] xml_handle_comment(obj, line_number, line_loc) - list_of_attr = ['name', 'type', 'nameType', 'deprecated', 'optional', 'recommended', - 'exists', 'unit'] + list_of_attr = [ + "name", + "type", + "nameType", + "deprecated", + "optional", + "recommended", + "exists", + "unit", + ] l_bracket = -1 r_bracket = -1 - if keyword.count('(') == 1: - l_bracket = keyword.index('(') - if keyword.count(')') == 1: - r_bracket = keyword.index(')') + if keyword.count("(") == 1: + l_bracket = keyword.index("(") + if keyword.count(")") == 1: + r_bracket = keyword.index(")") keyword_name, keyword_type = nx_name_type_resolving(keyword) if not keyword_name and not keyword_type: raise ValueError("A group must have both value and name. Check for group.") - grp = ET.SubElement(obj, 'group') + grp = ET.SubElement(obj, "group") if l_bracket == 0 and r_bracket > 0: - grp.set('type', keyword_type) + grp.set("type", keyword_type) if keyword_name: - grp.set('name', keyword_name) + grp.set("name", keyword_name) elif l_bracket > 0: - grp.set('name', keyword_name) + grp.set("name", keyword_name) if keyword_type: - grp.set('type', keyword_type) + grp.set("type", keyword_type) else: - grp.set('name', keyword_name) + grp.set("name", keyword_name) if value: rm_key_list = [] for attr, vval in value.items(): - if '__line__' in attr: + if "__line__" in attr: continue line_number = f"__line__{attr}" line_loc = value[line_number] - if attr == 'doc': + if attr == "doc": xml_handle_doc(grp, vval, line_number, line_loc) rm_key_list.append(attr) rm_key_list.append(line_number) - elif attr == 'exists' and vval: + elif attr == "exists" and vval: xml_handle_exists(value, grp, attr, vval) rm_key_list.append(attr) rm_key_list.append(line_number) - xml_handle_comment(obj, - line_number, line_loc, grp) - elif attr == 'unit': + xml_handle_comment(obj, line_number, line_loc, grp) + elif attr == "unit": xml_handle_units(grp, vval) xml_handle_comment(obj, line_number, line_loc, grp) elif attr in list_of_attr and not isinstance(vval, dict) and vval: @@ -381,7 +396,7 @@ def xml_handle_group(dct, obj, keyword, value, verbose=False): for key in rm_key_list: del value[key] # Check for skipped attrinutes - check_for_skiped_attributes('group', value, list_of_attr, verbose) + check_for_skiped_attributes("group", value, list_of_attr, verbose) if isinstance(value, dict) and value != {}: recursive_build(grp, value, verbose) @@ -403,34 +418,37 @@ def xml_handle_dimensions(dct, obj, keyword, value: dict): incr:[...]' """ - possible_dimension_attrs = ['rank'] # nxdl attributes - line_number = f'__line__{keyword}' + possible_dimension_attrs = ["rank"] # nxdl attributes + line_number = f"__line__{keyword}" line_loc = dct[line_number] - assert 'dim' in value.keys(), (f"Line {line_loc}: No dim as child of dimension has " - f"been found.") + assert "dim" in value.keys(), ( + f"Line {line_loc}: No dim as child of dimension has " f"been found." + ) xml_handle_comment(obj, line_number, line_loc) - dims = ET.SubElement(obj, 'dimensions') + dims = ET.SubElement(obj, "dimensions") # Consider all the childs under dimension is dim element and # its attributes rm_key_list = [] - rank = '' + rank = "" for key, val in value.items(): - if '__line__' in key: + if "__line__" in key: continue line_number = f"__line__{key}" line_loc = value[line_number] - if key == 'rank': - rank = val or '' + if key == "rank": + rank = val or "" if isinstance(rank, int) and rank < 0: - raise ValueError(f"Dimension must have some info about rank which is not " - f"available. Please check arround Line: {dct[line_number]}") + raise ValueError( + f"Dimension must have some info about rank which is not " + f"available. Please check arround Line: {dct[line_number]}" + ) dims.set(key, str(val)) rm_key_list.append(key) rm_key_list.append(line_number) xml_handle_comment(obj, line_number, line_loc, dims) # Check dimension doc and handle it - elif key == 'doc' and isinstance(val, str): + elif key == "doc" and isinstance(val, str): xml_handle_doc(dims, val, line_number, line_loc) rm_key_list.append(key) rm_key_list.append(line_number) @@ -450,18 +468,20 @@ def xml_handle_dimensions(dct, obj, keyword, value: dict): # pylint: disable=too-many-locals, too-many-arguments -def xml_handle_dim_from_dimension_dict(dct, dims_obj, keyword, value, rank, verbose=False): +def xml_handle_dim_from_dimension_dict( + dct, dims_obj, keyword, value, rank, verbose=False +): """ - Handling dim element. - NOTE: The inputs 'keyword' and 'value' are as input for xml_handle_dimensions - function. please also read note in xml_handle_dimensions. + Handling dim element. + NOTE: The inputs 'keyword' and 'value' are as input for xml_handle_dimensions + function. please also read note in xml_handle_dimensions. """ - possible_dim_attrs = ['ref', 'incr', 'refindex', 'required'] + possible_dim_attrs = ["ref", "incr", "refindex", "required"] # Some attributes might have equivalent name e.g. 'required' is correct one and # 'optional' could be another name. Then change attribute to the correct one. - wrong_to_correct_attr = [('optional', 'required')] + wrong_to_correct_attr = [("optional", "required")] header_line_number = f"__line__{keyword}" dim_list = [] rm_key_list = [] @@ -469,51 +489,55 @@ def xml_handle_dim_from_dimension_dict(dct, dims_obj, keyword, value, rank, verb # under dim_parameters if not value: return - rank = '' + rank = "" # pylint: disable=too-many-nested-blocks for attr, vvalue in value.items(): - if '__line__' in attr: + if "__line__" in attr: continue line_number = f"__line__{attr}" line_loc = value[line_number] # dim comes in precedence - if attr == 'dim': + if attr == "dim": # dim consists of list of [index, value] llist_ind_value = vvalue - assert isinstance(llist_ind_value, list), (f'Line {value[line_number]}: dim' - f'argument not a list !') + assert isinstance(llist_ind_value, list), ( + f"Line {value[line_number]}: dim" f"argument not a list !" + ) xml_handle_comment(dims_obj, line_number, line_loc) if isinstance(rank, int) and rank > 0: assert rank == len(llist_ind_value), ( f"Wrong dimension rank check around Line {dct[header_line_number]}.\n" f"Line {[dct[header_line_number]]} rank value {rank} " f"is not the same as dim array = " - f"{len(llist_ind_value)}.") + f"{len(llist_ind_value)}." + ) # Taking care of ind and value that comes as list of list for dim_ind_val in llist_ind_value: - dim = ET.SubElement(dims_obj, 'dim') + dim = ET.SubElement(dims_obj, "dim") # Taking care of multidimensions or rank if len(dim_ind_val) >= 1 and dim_ind_val[0]: - dim.set('index', str(dim_ind_val[0])) + dim.set("index", str(dim_ind_val[0])) if len(dim_ind_val) == 2 and dim_ind_val[1]: - dim.set('value', str(dim_ind_val[1])) + dim.set("value", str(dim_ind_val[1])) dim_list.append(dim) rm_key_list.append(attr) rm_key_list.append(line_number) - elif attr == 'dim_parameters' and isinstance(vvalue, dict): + elif attr == "dim_parameters" and isinstance(vvalue, dict): xml_handle_comment(dims_obj, line_number, line_loc) for kkkey, vvval in vvalue.items(): - if '__line__' in kkkey: + if "__line__" in kkkey: continue - cmnt_number = f'__line__{kkkey}' + cmnt_number = f"__line__{kkkey}" cmnt_loc = vvalue[cmnt_number] # Check whether any optional attributes added for tuple_wng_crt in wrong_to_correct_attr: if kkkey == tuple_wng_crt[0]: - raise ValueError(f"{cmnt_loc}: Attribute '{kkkey}' is prohibited, use " - f"'{tuple_wng_crt[1]}") - if kkkey == 'doc' and dim_list: + raise ValueError( + f"{cmnt_loc}: Attribute '{kkkey}' is prohibited, use " + f"'{tuple_wng_crt[1]}" + ) + if kkkey == "doc" and dim_list: # doc comes as list of doc for i, dim in enumerate(dim_list): if isinstance(vvval, list) and i < len(vvval): @@ -539,13 +563,15 @@ def xml_handle_dim_from_dimension_dict(dct, dims_obj, keyword, value, rank, verb rm_key_list.append(attr) rm_key_list.append(line_number) else: - raise ValueError(f"Got unexpected block except 'dim' and 'dim_parameters'." - f"Please check arround line {line_number}") + raise ValueError( + f"Got unexpected block except 'dim' and 'dim_parameters'." + f"Please check arround line {line_number}" + ) for key in rm_key_list: del value[key] - check_for_skiped_attributes('dim', value, possible_dim_attrs, verbose) + check_for_skiped_attributes("dim", value, possible_dim_attrs, verbose) def xml_handle_enumeration(dct, obj, keyword, value, verbose): @@ -555,24 +581,27 @@ def xml_handle_enumeration(dct, obj, keyword, value, verbose): 1) the items are in a list 2) the items are dictionaries and may contain a nested doc """ - line_number = f'__line__{keyword}' + line_number = f"__line__{keyword}" line_loc = dct[line_number] xml_handle_comment(obj, line_number, line_loc) - enum = ET.SubElement(obj, 'enumeration') - - assert value is not None, f'Line {line_loc}: enumeration must \ -bear at least an argument !' - assert len( - value) >= 1, f'Line {dct[line_number]}: enumeration must not be an empty list!' + enum = ET.SubElement(obj, "enumeration") + + assert ( + value is not None + ), f"Line {line_loc}: enumeration must \ +bear at least an argument !" + assert ( + len(value) >= 1 + ), f"Line {dct[line_number]}: enumeration must not be an empty list!" if isinstance(value, list): for element in value: - itm = ET.SubElement(enum, 'item') - itm.set('value', str(element)) + itm = ET.SubElement(enum, "item") + itm.set("value", str(element)) if isinstance(value, dict) and value != {}: for element in value.keys(): - if '__line__' not in element: - itm = ET.SubElement(enum, 'item') - itm.set('value', str(element)) + if "__line__" not in element: + itm = ET.SubElement(enum, "item") + itm.set("value", str(element)) if isinstance(value[element], dict): recursive_build(itm, value[element], verbose) @@ -580,25 +609,25 @@ def xml_handle_enumeration(dct, obj, keyword, value, verbose): # pylint: disable=unused-argument def xml_handle_link(dct, obj, keyword, value, verbose): """ - If we have an NXDL link we decode the name attribute from (link)[:-6] + If we have an NXDL link we decode the name attribute from (link)[:-6] """ line_number = f"__line__{keyword}" line_loc = dct[line_number] xml_handle_comment(obj, line_number, line_loc) - possible_attrs = ['name', 'target', 'napimount'] + possible_attrs = ["name", "target", "napimount"] name = keyword[:-6] - link_obj = ET.SubElement(obj, 'link') - link_obj.set('name', str(name)) + link_obj = ET.SubElement(obj, "link") + link_obj.set("name", str(name)) if value: rm_key_list = [] for attr, vval in value.items(): - if '__line__' in attr: + if "__line__" in attr: continue line_number = f"__line__{attr}" line_loc = value[line_number] - if attr == 'doc': + if attr == "doc": xml_handle_doc(link_obj, vval, line_number, line_loc) rm_key_list.append(attr) rm_key_list.append(line_number) @@ -612,7 +641,7 @@ def xml_handle_link(dct, obj, keyword, value, verbose): for key in rm_key_list: del value[key] # Check for skipped attrinutes - check_for_skiped_attributes('link', value, possible_attrs, verbose) + check_for_skiped_attributes("link", value, possible_attrs, verbose) if isinstance(value, dict) and value != {}: recursive_build(link_obj, value, verbose=None) @@ -620,26 +649,26 @@ def xml_handle_link(dct, obj, keyword, value, verbose): def xml_handle_choice(dct, obj, keyword, value, verbose=False): """ - Build choice xml elements. That consists of groups. + Build choice xml elements. That consists of groups. """ - line_number = f'__line__{keyword}' + line_number = f"__line__{keyword}" line_loc = dct[line_number] xml_handle_comment(obj, line_number, line_loc) # Add attributes in possible if new attributs have been added nexus definition. possible_attr = [] - choice_obj = ET.SubElement(obj, 'choice') + choice_obj = ET.SubElement(obj, "choice") # take care of special attributes name = keyword[:-8] - choice_obj.set('name', name) + choice_obj.set("name", name) if value: rm_key_list = [] for attr, vval in value.items(): - if '__line__' in attr: + if "__line__" in attr: continue line_number = f"__line__{attr}" line_loc = value[line_number] - if attr == 'doc': + if attr == "doc": xml_handle_doc(choice_obj, vval, line_number, line_loc) rm_key_list.append(attr) rm_key_list.append(line_number) @@ -653,40 +682,40 @@ def xml_handle_choice(dct, obj, keyword, value, verbose=False): for key in rm_key_list: del value[key] # Check for skipped attrinutes - check_for_skiped_attributes('choice', value, possible_attr, verbose) + check_for_skiped_attributes("choice", value, possible_attr, verbose) if isinstance(value, dict) and value != {}: recursive_build(choice_obj, value, verbose=None) def xml_handle_symbols(dct, obj, keyword, value: dict): - """Handle a set of NXDL symbols as a child to obj - - """ - line_number = f'__line__{keyword}' + """Handle a set of NXDL symbols as a child to obj""" + line_number = f"__line__{keyword}" line_loc = dct[line_number] - assert len(list(value.keys()) - ) >= 1, f'Line {line_loc}: symbols table must not be empty !' + assert ( + len(list(value.keys())) >= 1 + ), f"Line {line_loc}: symbols table must not be empty !" xml_handle_comment(obj, line_number, line_loc) - syms = ET.SubElement(obj, 'symbols') - if 'doc' in value.keys(): - line_number = '__line__doc' + syms = ET.SubElement(obj, "symbols") + if "doc" in value.keys(): + line_number = "__line__doc" line_loc = value[line_number] xml_handle_comment(syms, line_number, line_loc) - doctag = ET.SubElement(syms, 'doc') - doctag.text = '\n' + textwrap.fill(value['doc'], width=70) + '\n' + doctag = ET.SubElement(syms, "doc") + doctag.text = "\n" + textwrap.fill(value["doc"], width=70) + "\n" rm_key_list = [] for kkeyword, vvalue in value.items(): - if '__line__' in kkeyword: + if "__line__" in kkeyword: continue - if kkeyword != 'doc': - line_number = f'__line__{kkeyword}' + if kkeyword != "doc": + line_number = f"__line__{kkeyword}" line_loc = value[line_number] xml_handle_comment(syms, line_number, line_loc) assert vvalue is not None and isinstance( - vvalue, str), f'Line {line_loc}: put a comment in doc string !' - sym = ET.SubElement(syms, 'symbol') - sym.set('name', str(kkeyword)) + vvalue, str + ), f"Line {line_loc}: put a comment in doc string !" + sym = ET.SubElement(syms, "symbol") + sym.set("name", str(kkeyword)) # sym_doc = ET.SubElement(sym, 'doc') xml_handle_doc(sym, vvalue) rm_key_list.append(kkeyword) @@ -704,15 +733,16 @@ def check_keyword_variable(verbose, dct, keyword, value): keyword_name, keyword_type = nx_name_type_resolving(keyword) if verbose: sys.stdout.write( - f'{keyword_name}({keyword_type}): value type is {type(value)}\n') - if keyword_name == '' and keyword_type == '': - line_number = f'__line__{keyword}' - raise ValueError(f'Line {dct[line_number]}: found an improper yaml key !') + f"{keyword_name}({keyword_type}): value type is {type(value)}\n" + ) + if keyword_name == "" and keyword_type == "": + line_number = f"__line__{keyword}" + raise ValueError(f"Line {dct[line_number]}: found an improper yaml key !") def helper_keyword_type(kkeyword_type): """ - This function is returning a value of keyword_type if it belong to NX_TYPE_KEYS + This function is returning a value of keyword_type if it belong to NX_TYPE_KEYS """ if kkeyword_type in NX_TYPE_KEYS: return kkeyword_type @@ -721,10 +751,10 @@ def helper_keyword_type(kkeyword_type): def verbose_flag(verbose, keyword, value): """ - Verbose stdout printing for nested levels of yaml file, if verbose flag is active + Verbose stdout printing for nested levels of yaml file, if verbose flag is active """ if verbose: - sys.stdout.write(f' key:{keyword}; value type is {type(value)}\n') + sys.stdout.write(f" key:{keyword}; value type is {type(value)}\n") def xml_handle_attributes(dct, obj, keyword, value, verbose): @@ -734,43 +764,53 @@ def xml_handle_attributes(dct, obj, keyword, value, verbose): line_loc = dct[line_number] xml_handle_comment(obj, line_number, line_loc) # list of possible attribute of xml attribute elementsa - attr_attr_list = ['name', 'type', 'unit', 'nameType', - 'optional', 'recommended', 'minOccurs', - 'maxOccurs', 'deprecated', 'exists'] + attr_attr_list = [ + "name", + "type", + "unit", + "nameType", + "optional", + "recommended", + "minOccurs", + "maxOccurs", + "deprecated", + "exists", + ] # as an attribute identifier keyword_name, keyword_typ = nx_name_type_resolving(keyword) - line_number = f'__line__{keyword}' + line_number = f"__line__{keyword}" if verbose: print(f"__line__ : {dct[line_number]}") - if keyword_name == '' and keyword_typ == '': - raise ValueError(f'Line {dct[line_number]}: found an improper yaml key !') - elemt_obj = ET.SubElement(obj, 'attribute') - elemt_obj.set('name', keyword_name[2:]) + if keyword_name == "" and keyword_typ == "": + raise ValueError(f"Line {dct[line_number]}: found an improper yaml key !") + elemt_obj = ET.SubElement(obj, "attribute") + elemt_obj.set("name", keyword_name[2:]) if keyword_typ: - elemt_obj.set('type', keyword_typ) + elemt_obj.set("type", keyword_typ) rm_key_list = [] if value and value: # taking care of attributes of attributes for attr, attr_val in value.items(): - if '__line__' in attr: + if "__line__" in attr: continue line_number = f"__line__{attr}" line_loc = value[line_number] - if attr in ['doc', *attr_attr_list] and not isinstance(attr_val, dict): - if attr == 'unit': + if attr in ["doc", *attr_attr_list] and not isinstance(attr_val, dict): + if attr == "unit": elemt_obj.set(f"{attr}s", str(value[attr])) rm_key_list.append(attr) rm_key_list.append(line_number) xml_handle_comment(obj, line_number, line_loc, elemt_obj) - elif attr == 'exists' and attr_val: + elif attr == "exists" and attr_val: xml_handle_exists(value, elemt_obj, attr, attr_val) rm_key_list.append(attr) rm_key_list.append(line_number) xml_handle_comment(obj, line_number, line_loc, elemt_obj) - elif attr == 'doc': - xml_handle_doc(elemt_obj, format_nxdl_doc(attr_val), - line_number, line_loc) + elif attr == "doc": + xml_handle_doc( + elemt_obj, format_nxdl_doc(attr_val), line_number, line_loc + ) rm_key_list.append(attr) rm_key_list.append(line_number) else: @@ -782,7 +822,7 @@ def xml_handle_attributes(dct, obj, keyword, value, verbose): for key in rm_key_list: del value[key] # Check cor skiped attribute - check_for_skiped_attributes('Attribute', value, attr_attr_list, verbose) + check_for_skiped_attributes("Attribute", value, attr_attr_list, verbose) if value: recursive_build(elemt_obj, value, verbose) @@ -794,25 +834,28 @@ def validate_field_attribute_and_value(v_attr, vval, allowed_attribute, value): """ # check for empty val - if (not isinstance(vval, dict) - and not str(vval)): # check for empty value - + if not isinstance(vval, dict) and not str(vval): # check for empty value line_number = f"__line__{v_attr}" - raise ValueError(f"In a field a valid attrbute ('{v_attr}') found that is not stored." - f" Please check arround line {value[line_number]}") + raise ValueError( + f"In a field a valid attrbute ('{v_attr}') found that is not stored." + f" Please check arround line {value[line_number]}" + ) # The bellow elements might come as child element - skipped_child_name = ['doc', 'dimension', 'enumeration', 'choice', 'exists'] + skipped_child_name = ["doc", "dimension", "enumeration", "choice", "exists"] # check for invalid key or attributes - if (v_attr not in [*skipped_child_name, *allowed_attribute] - and '__line__' not in v_attr + if ( + v_attr not in [*skipped_child_name, *allowed_attribute] + and "__line__" not in v_attr and not isinstance(vval, dict) - and '(' not in v_attr # skip only groups and field that has name and type - and '\\@' not in v_attr): # skip nexus attributes - + and "(" not in v_attr # skip only groups and field that has name and type + and "\\@" not in v_attr + ): # skip nexus attributes line_number = f"__line__{v_attr}" - raise ValueError(f"In a field or group a invalid attribute ('{v_attr}') or child has found." - f" Please check arround line {value[line_number]}.") + raise ValueError( + f"In a field or group a invalid attribute ('{v_attr}') or child has found." + f" Please check arround line {value[line_number]}." + ) def xml_handle_fields(obj, keyword, value, line_annot, line_loc, verbose=False): @@ -830,84 +873,101 @@ def xml_handle_fields(obj, keyword, value, line_annot, line_loc, verbose=False): This simple function will define a new node of xml tree """ # List of possible attributes of xml elements - allowed_attr = ['name', 'type', 'nameType', 'unit', 'minOccurs', 'long_name', - 'axis', 'signal', 'deprecated', 'axes', 'exists', - 'data_offset', 'interpretation', 'maxOccurs', - 'primary', 'recommended', 'optional', 'stride'] + allowed_attr = [ + "name", + "type", + "nameType", + "unit", + "minOccurs", + "long_name", + "axis", + "signal", + "deprecated", + "axes", + "exists", + "data_offset", + "interpretation", + "maxOccurs", + "primary", + "recommended", + "optional", + "stride", + ] xml_handle_comment(obj, line_annot, line_loc) l_bracket = -1 r_bracket = -1 - if keyword.count('(') == 1: - l_bracket = keyword.index('(') - if keyword.count(')') == 1: - r_bracket = keyword.index(')') + if keyword.count("(") == 1: + l_bracket = keyword.index("(") + if keyword.count(")") == 1: + r_bracket = keyword.index(")") keyword_name, keyword_type = nx_name_type_resolving(keyword) if not keyword_type and not keyword_name: raise ValueError("Check for name or type in field.") - elemt_obj = ET.SubElement(obj, 'field') + elemt_obj = ET.SubElement(obj, "field") # type come first if l_bracket == 0 and r_bracket > 0: - elemt_obj.set('type', keyword_type) + elemt_obj.set("type", keyword_type) if keyword_name: - elemt_obj.set('name', keyword_name) + elemt_obj.set("name", keyword_name) elif l_bracket > 0: - elemt_obj.set('name', keyword_name) + elemt_obj.set("name", keyword_name) if keyword_type: - elemt_obj.set('type', keyword_type) + elemt_obj.set("type", keyword_type) else: - elemt_obj.set('name', keyword_name) + elemt_obj.set("name", keyword_name) if value: rm_key_list = [] # In each each if clause apply xml_handle_comment(), to collect # comments on that yaml line. for attr, vval in value.items(): - if '__line__' in attr: + if "__line__" in attr: continue line_number = f"__line__{attr}" line_loc = value[line_number] - if attr == 'doc': - xml_handle_doc(elemt_obj, vval, line_number, line_loc,) + if attr == "doc": + xml_handle_doc( + elemt_obj, + vval, + line_number, + line_loc, + ) rm_key_list.append(attr) rm_key_list.append(line_number) - elif attr == 'exists' and vval: + elif attr == "exists" and vval: xml_handle_exists(value, elemt_obj, attr, vval) rm_key_list.append(attr) rm_key_list.append(line_number) - xml_handle_comment(obj, - line_number, - line_loc, elemt_obj) - elif attr == 'unit': + xml_handle_comment(obj, line_number, line_loc, elemt_obj) + elif attr == "unit": xml_handle_units(elemt_obj, vval) - xml_handle_comment(obj, - line_number, - line_loc, elemt_obj) + xml_handle_comment(obj, line_number, line_loc, elemt_obj) elif attr in allowed_attr and not isinstance(vval, dict) and vval: validate_field_attribute_and_value(attr, vval, allowed_attr, value) elemt_obj.set(attr, check_for_mapping_char_other(vval)) rm_key_list.append(attr) rm_key_list.append(line_number) - xml_handle_comment(obj, - line_number, - line_loc, elemt_obj) + xml_handle_comment(obj, line_number, line_loc, elemt_obj) for key in rm_key_list: del value[key] # Check for skipped attrinutes - check_for_skiped_attributes('field', value, allowed_attr, verbose) + check_for_skiped_attributes("field", value, allowed_attr, verbose) if isinstance(value, dict) and value != {}: recursive_build(elemt_obj, value, verbose) -def xml_handle_comment(obj: ET.Element, - line_annotation: str, - line_loc_no: int, - xml_ele: ET.Element = None, - is_def_cmnt: bool = False): +def xml_handle_comment( + obj: ET.Element, + line_annotation: str, + line_loc_no: int, + xml_ele: ET.Element = None, + is_def_cmnt: bool = False, +): """ Add xml comment: check for comments that has the same 'line_annotation' (e.g. __line__data) and the same line_loc_no (e.g. 30). After that, i @@ -936,7 +996,7 @@ def xml_handle_comment(obj: ET.Element, obj.append(si_comnt) else: raise ValueError("Provied correct parameter values.") - return '' + return "" def recursive_build(obj, dct, verbose): @@ -948,7 +1008,7 @@ def recursive_build(obj, dct, verbose): """ for keyword, value in iter(dct.items()): - if '__line__' in keyword: + if "__line__" in keyword: continue line_number = f"__line__{keyword}" line_loc = dct[line_number] @@ -956,44 +1016,46 @@ def recursive_build(obj, dct, verbose): check_keyword_variable(verbose, dct, keyword, value) if verbose: sys.stdout.write( - f'keyword_name:{keyword_name} keyword_type {keyword_type}\n') + f"keyword_name:{keyword_name} keyword_type {keyword_type}\n" + ) - if keyword[-6:] == '(link)': + if keyword[-6:] == "(link)": xml_handle_link(dct, obj, keyword, value, verbose) - elif keyword[-8:] == '(choice)': + elif keyword[-8:] == "(choice)": xml_handle_choice(dct, obj, keyword, value) # The bellow xml_symbol clause is for the symbols that come ubde filed or attributes # Root level symbols has been inside nyaml2nxdl() - elif keyword_type == '' and keyword_name == 'symbols': + elif keyword_type == "" and keyword_name == "symbols": xml_handle_symbols(dct, obj, keyword, value) - elif ((keyword_type in NX_CLSS) or (keyword_type not in - [*NX_TYPE_KEYS, '', *NX_NEW_DEFINED_CLASSES])): + elif (keyword_type in NX_CLSS) or ( + keyword_type not in [*NX_TYPE_KEYS, "", *NX_NEW_DEFINED_CLASSES] + ): # we can be sure we need to instantiate a new group xml_handle_group(dct, obj, keyword, value, verbose) elif keyword_name[0:2] == NX_ATTR_IDNT: # check if obj qualifies xml_handle_attributes(dct, obj, keyword, value, verbose) - elif keyword == 'doc': + elif keyword == "doc": xml_handle_doc(obj, value, line_number, line_loc) elif keyword == NX_UNIT_IDNT: xml_handle_units(obj, value) - elif keyword == 'enumeration': + elif keyword == "enumeration": xml_handle_enumeration(dct, obj, keyword, value, verbose) - elif keyword == 'dimensions': + elif keyword == "dimensions": xml_handle_dimensions(dct, obj, keyword, value) - elif keyword == 'exists': + elif keyword == "exists": xml_handle_exists(dct, obj, keyword, value) # Handles fileds e.g. AXISNAME - elif keyword_name != '' and '__line__' not in keyword_name: - xml_handle_fields(obj, keyword, - value, line_number, - line_loc, verbose) + elif keyword_name != "" and "__line__" not in keyword_name: + xml_handle_fields(obj, keyword, value, line_number, line_loc, verbose) else: - raise ValueError(f"An unfamiliar type of element {keyword} has been found which is " - f"not be able to be resolved. Chekc arround line {dct[line_number]}") + raise ValueError( + f"An unfamiliar type of element {keyword} has been found which is " + f"not be able to be resolved. Chekc arround line {dct[line_number]}" + ) def pretty_print_xml(xml_root, output_xml, def_comments=None): @@ -1001,10 +1063,10 @@ def pretty_print_xml(xml_root, output_xml, def_comments=None): Print better human-readable indented and formatted xml file using built-in libraries and preceding XML processing instruction """ - dom = minidom.parseString(ET.tostring( - xml_root, encoding='utf-8', method='xml')) + dom = minidom.parseString(ET.tostring(xml_root, encoding="utf-8", method="xml")) proc_instractionn = dom.createProcessingInstruction( - 'xml-stylesheet', 'type="text/xsl" href="nxdlformat.xsl"') + "xml-stylesheet", 'type="text/xsl" href="nxdlformat.xsl"' + ) dom_comment = dom.createComment(DOM_COMMENT) root = dom.firstChild dom.insertBefore(proc_instractionn, root) @@ -1015,27 +1077,27 @@ def pretty_print_xml(xml_root, output_xml, def_comments=None): def_comt_ele = dom.createComment(string) dom.insertBefore(def_comt_ele, root) - xml_string = dom.toprettyxml(indent=1 * DEPTH_SIZE, newl='\n', encoding='UTF-8') - with open('tmp.xml', "wb") as file_tmp: + xml_string = dom.toprettyxml(indent=1 * DEPTH_SIZE, newl="\n", encoding="UTF-8") + with open("tmp.xml", "wb") as file_tmp: file_tmp.write(xml_string) flag = False - with open('tmp.xml', "r", encoding="utf-8") as file_out: + with open("tmp.xml", "r", encoding="utf-8") as file_out: with open(output_xml, "w", encoding="utf-8") as file_out_mod: for i in file_out.readlines(): - if '' not in i and '' not in i and flag is False: + if "" not in i and "" not in i and flag is False: file_out_mod.write(i) - elif '' in i and '' in i: + elif "" in i and "" in i: file_out_mod.write(i) - elif '' in i and '' not in i: + elif "" in i and "" not in i: flag = True white_spaces = len(i) - len(i.lstrip()) file_out_mod.write(i) - elif '' not in i and '' not in i and flag is True: - file_out_mod.write((white_spaces + 5) * ' ' + i) - elif '' not in i and '' in i and flag is True: - file_out_mod.write(white_spaces * ' ' + i) + elif "" not in i and "" not in i and flag is True: + file_out_mod.write((white_spaces + 5) * " " + i) + elif "" not in i and "" in i and flag is True: + file_out_mod.write(white_spaces * " " + i) flag = False - os.remove('tmp.xml') + os.remove("tmp.xml") # pylint: disable=too-many-statements @@ -1046,102 +1108,120 @@ def nyaml2nxdl(input_file: str, out_file, verbose: bool): fields or (their) attributes as childs of the groups """ - def_attributes = ['deprecated', 'ignoreExtraGroups', 'category', 'type', - 'ignoreExtraFields', 'ignoreExtraAttributes', 'restricts'] + def_attributes = [ + "deprecated", + "ignoreExtraGroups", + "category", + "type", + "ignoreExtraFields", + "ignoreExtraAttributes", + "restricts", + ] yml_appdef = yml_reader(input_file) def_cmnt_text = [] if verbose: - sys.stdout.write(f'input-file: {input_file}\n') - sys.stdout.write('application/base contains the following root-level entries:\n') + sys.stdout.write(f"input-file: {input_file}\n") + sys.stdout.write( + "application/base contains the following root-level entries:\n" + ) sys.stdout.write(str(yml_appdef.keys())) - xml_root = ET.Element('definition', {}) - assert 'category' in yml_appdef.keys( - ), 'Required root-level keyword category is missing!' - assert yml_appdef['category'] in ['application', 'base'], 'Only \ -application and base are valid categories!' - assert 'doc' in yml_appdef.keys(), 'Required root-level keyword doc is missing!' - - name_extends = '' + xml_root = ET.Element("definition", {}) + assert ( + "category" in yml_appdef.keys() + ), "Required root-level keyword category is missing!" + assert yml_appdef["category"] in [ + "application", + "base", + ], "Only \ +application and base are valid categories!" + assert "doc" in yml_appdef.keys(), "Required root-level keyword doc is missing!" + + name_extends = "" yml_appdef_copy = yml_appdef.copy() for kkey, vvalue in yml_appdef_copy.items(): - if '__line__' in kkey: + if "__line__" in kkey: continue line_number = f"__line__{kkey}" line_loc_no = yml_appdef[line_number] if not isinstance(vvalue, dict) and kkey in def_attributes: - xml_root.set(kkey, str(vvalue) or '') - cmnt_text = xml_handle_comment(xml_root, - line_number, line_loc_no, - is_def_cmnt=True) + xml_root.set(kkey, str(vvalue) or "") + cmnt_text = xml_handle_comment( + xml_root, line_number, line_loc_no, is_def_cmnt=True + ) def_cmnt_text += cmnt_text if cmnt_text else [] del yml_appdef[line_number] del yml_appdef[kkey] # Taking care or name and extends - elif 'NX' in kkey: + elif "NX" in kkey: # Tacking the attribute order but the correct value will be stored later # check for name first or type first if (NXobject)NXname then type first - l_bracket_ind = kkey.rfind('(') - r_bracket_ind = kkey.rfind(')') + l_bracket_ind = kkey.rfind("(") + r_bracket_ind = kkey.rfind(")") if l_bracket_ind == 0: extend = kkey[1:r_bracket_ind] - name = kkey[r_bracket_ind + 1:] - xml_root.set('extends', extend) - xml_root.set('name', name) + name = kkey[r_bracket_ind + 1 :] + xml_root.set("extends", extend) + xml_root.set("name", name) elif l_bracket_ind > 0: name = kkey[0:l_bracket_ind] - extend = kkey[l_bracket_ind + 1: r_bracket_ind] - xml_root.set('name', name) - xml_root.set('extends', extend) + extend = kkey[l_bracket_ind + 1 : r_bracket_ind] + xml_root.set("name", name) + xml_root.set("extends", extend) else: name = kkey - xml_root.set('name', name) - xml_root.set('extends', 'NXobject') - cmnt_text = xml_handle_comment(xml_root, - line_number, line_loc_no, - is_def_cmnt=True) + xml_root.set("name", name) + xml_root.set("extends", "NXobject") + cmnt_text = xml_handle_comment( + xml_root, line_number, line_loc_no, is_def_cmnt=True + ) def_cmnt_text += cmnt_text if cmnt_text else [] name_extends = kkey - if 'type' not in xml_root.attrib: - xml_root.set('type', "group") + if "type" not in xml_root.attrib: + xml_root.set("type", "group") # Taking care of namespaces - namespaces = {'xmlns': 'http://definition.nexusformat.org/nxdl/3.1', - 'xmlns:xsi': 'http://www.w3.org/2001/XMLSchema-instance', - 'xsi:schemaLocation': 'http://definition.nexusformat.org/nxdl/3.1 ../nxdl.xsd'} + namespaces = { + "xmlns": "http://definition.nexusformat.org/nxdl/3.1", + "xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance", + "xsi:schemaLocation": "http://definition.nexusformat.org/nxdl/3.1 ../nxdl.xsd", + } for key, ns_ in namespaces.items(): xml_root.attrib[key] = ns_ # Taking care of Symbols elements - if 'symbols' in yml_appdef.keys(): - xml_handle_symbols(yml_appdef, - xml_root, - 'symbols', - yml_appdef['symbols']) + if "symbols" in yml_appdef.keys(): + xml_handle_symbols(yml_appdef, xml_root, "symbols", yml_appdef["symbols"]) - del yml_appdef['symbols'] + del yml_appdef["symbols"] del yml_appdef["__line__symbols"] - assert isinstance(yml_appdef['doc'], str) and yml_appdef['doc'] != '', 'Doc \ -has to be a non-empty string!' + assert ( + isinstance(yml_appdef["doc"], str) and yml_appdef["doc"] != "" + ), "Doc \ +has to be a non-empty string!" - line_number = '__line__doc' + line_number = "__line__doc" line_loc_no = yml_appdef[line_number] - xml_handle_doc(xml_root, yml_appdef['doc'], line_number, line_loc_no) + xml_handle_doc(xml_root, yml_appdef["doc"], line_number, line_loc_no) - del yml_appdef['doc'] + del yml_appdef["doc"] root_keys = 0 for key in yml_appdef.keys(): - if '__line__' not in key: + if "__line__" not in key: root_keys += 1 extra_key = key - assert root_keys == 1, (f"Accepting at most keywords: category, doc, symbols, and NX... " - f"at root-level! check key at root level {extra_key}") + assert root_keys == 1, ( + f"Accepting at most keywords: category, doc, symbols, and NX... " + f"at root-level! check key at root level {extra_key}" + ) - assert ('NX' in name_extends and len(name_extends) > 2), 'NX \ -keyword has an invalid pattern, or is too short!' + assert ( + "NX" in name_extends and len(name_extends) > 2 + ), "NX \ +keyword has an invalid pattern, or is too short!" # Taking care if definition has empty content if yml_appdef[name_extends]: recursive_build(xml_root, yml_appdef[name_extends], verbose) @@ -1158,4 +1238,4 @@ def nyaml2nxdl(input_file: str, out_file, verbose: bool): check_for_default_attribute_and_value(xml_root) pretty_print_xml(xml_root, out_file, def_cmnt_text) if verbose: - sys.stdout.write('Parsed YAML to NXDL successfully\n') + sys.stdout.write("Parsed YAML to NXDL successfully\n") diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py b/dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py index 9583b375d2..f4787bd300 100644 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py @@ -28,19 +28,17 @@ # So the corresponding value is to skip them and # and also carefull about this order import hashlib + from yaml.composer import Composer from yaml.constructor import Constructor - +from yaml.loader import Loader from yaml.nodes import ScalarNode from yaml.resolver import BaseResolver -from yaml.loader import Loader # NOTE: If any one change one of the bellow dict please change it for both -ESCAPE_CHAR_DICT_IN_YAML = {"\t": " ", - "\':\'": ":"} +ESCAPE_CHAR_DICT_IN_YAML = {"\t": " ", "':'": ":"} -ESCAPE_CHAR_DICT_IN_XML = {" ": "\t", - "\':\'": ":"} +ESCAPE_CHAR_DICT_IN_XML = {" ": "\t", "':'": ":"} class LineLoader(Loader): # pylint: disable=too-many-ancestors @@ -61,11 +59,13 @@ def construct_mapping(self, node, deep=False): for key_node in node_pair_lst: shadow_key_node = ScalarNode( - tag=BaseResolver.DEFAULT_SCALAR_TAG, value='__line__' + key_node[0].value) + tag=BaseResolver.DEFAULT_SCALAR_TAG, + value="__line__" + key_node[0].value, + ) shadow_value_node = ScalarNode( - tag=BaseResolver.DEFAULT_SCALAR_TAG, value=key_node[0].__line__) - node_pair_lst_for_appending.append( - (shadow_key_node, shadow_value_node)) + tag=BaseResolver.DEFAULT_SCALAR_TAG, value=key_node[0].__line__ + ) + node_pair_lst_for_appending.append((shadow_key_node, shadow_value_node)) node.value = node_pair_lst + node_pair_lst_for_appending return Constructor.construct_mapping(self, node, deep=deep) @@ -84,11 +84,11 @@ def get_yaml_escape_char_reverter_dict(): def type_check(nx_type): """ - Check for nexus type if type is NX_CHAR get '' or get as it is. + Check for nexus type if type is NX_CHAR get '' or get as it is. """ - if nx_type in ['NX_CHAR', '']: - nx_type = '' + if nx_type in ["NX_CHAR", ""]: + nx_type = "" else: nx_type = f"({nx_type})" return nx_type @@ -108,10 +108,10 @@ def get_node_parent_info(tree, node): def cleaning_empty_lines(line_list): """ - Cleaning up empty lines on top and bottom. + Cleaning up empty lines on top and bottom. """ if not isinstance(line_list, list): - line_list = line_list.split('\n') if '\n' in line_list else [''] + line_list = line_list.split("\n") if "\n" in line_list else [""] # Clining up top empty lines while True: @@ -119,7 +119,7 @@ def cleaning_empty_lines(line_list): break line_list = line_list[1:] if len(line_list) == 0: - line_list.append('') + line_list.append("") return line_list # Clining bottom empty lines @@ -128,7 +128,7 @@ def cleaning_empty_lines(line_list): break line_list = line_list[0:-1] if len(line_list) == 0: - line_list.append('') + line_list.append("") return line_list return line_list @@ -140,45 +140,44 @@ def nx_name_type_resolving(tmp): and type {nexus_type} from a YML section string. YML section string syntax: optional_string(nexus_type) """ - if tmp.count('(') == 1 and tmp.count(')') == 1: + if tmp.count("(") == 1 and tmp.count(")") == 1: # we can safely assume that every valid YML key resolves # either an nx_ (type, base, candidate) class contains only 1 '(' and ')' - index_start = tmp.index('(') - index_end = tmp.index(')', index_start + 1) - typ = tmp[index_start + 1:index_end] - nam = tmp.replace('(' + typ + ')', '') + index_start = tmp.index("(") + index_end = tmp.index(")", index_start + 1) + typ = tmp[index_start + 1 : index_end] + nam = tmp.replace("(" + typ + ")", "") return nam, typ # or a name for a member - typ = '' + typ = "" nam = tmp return nam, typ def get_sha256_hash(file_name): - """Generate a sha256_hash for a given file. - """ + """Generate a sha256_hash for a given file.""" sha_hash = hashlib.sha256() - with open(file=file_name, mode='rb',) as file_obj: + with open( + file=file_name, + mode="rb", + ) as file_obj: # Update hash for each 4k block of bytes for b_line in iter(lambda: file_obj.read(4096), b""): sha_hash.update(b_line) return sha_hash.hexdigest() -def extend_yamlfile_with_comment(yaml_file, - file_to_be_appended, - top_lines_list=None): - """Extend yaml file by the file_to_be_appended as comment. - """ +def extend_yamlfile_with_comment(yaml_file, file_to_be_appended, top_lines_list=None): + """Extend yaml file by the file_to_be_appended as comment.""" - with open(yaml_file, mode='a+', encoding='utf-8') as f1_obj: + with open(yaml_file, mode="a+", encoding="utf-8") as f1_obj: if top_lines_list: for line in top_lines_list: f1_obj.write(line) - with open(file_to_be_appended, mode='r', encoding='utf-8') as f2_obj: + with open(file_to_be_appended, mode="r", encoding="utf-8") as f2_obj: lines = f2_obj.readlines() for line in lines: f1_obj.write(f"# {line}") @@ -191,30 +190,30 @@ def separate_hash_yaml_and_nxdl(yaml_file, sep_yaml, sep_xml): ++++++++++++++++++++++++++++++++++\n' # ' """ - sha_hash = '' - with open(yaml_file, 'r', encoding='utf-8') as inp_file: + sha_hash = "" + with open(yaml_file, "r", encoding="utf-8") as inp_file: lines = inp_file.readlines() # file to write yaml part - with open(sep_yaml, 'w', encoding='utf-8') as yml_f_ob, \ - open(sep_xml, 'w', encoding='utf-8') as xml_f_ob: - - last_line = '' + with open(sep_yaml, "w", encoding="utf-8") as yml_f_ob, open( + sep_xml, "w", encoding="utf-8" + ) as xml_f_ob: + last_line = "" write_on_yaml = True for ind, line in enumerate(lines): if ind == 0: last_line = line # Write in file when ensured that the nest line is not with '++ SHA HASH ++' - elif '++ SHA HASH ++' not in line and write_on_yaml: + elif "++ SHA HASH ++" not in line and write_on_yaml: yml_f_ob.write(last_line) last_line = line - elif '++ SHA HASH ++' in line: + elif "++ SHA HASH ++" in line: write_on_yaml = False - last_line = '' + last_line = "" elif not write_on_yaml and not last_line: # The first line of xml file has been found. Onward write lines directly # into xml file. if not sha_hash: - sha_hash = line.split('# ', 1)[-1].strip() + sha_hash = line.split("# ", 1)[-1].strip() else: xml_f_ob.write(line[2:]) # If the yaml fiile does not contain any hash for nxdl then we may have last line. From bf1936a6fd9cf96caf132460d6d5bd71440de7e5 Mon Sep 17 00:00:00 2001 From: Sandor Brockhauser Date: Mon, 19 Jun 2023 15:12:21 +0200 Subject: [PATCH 47/93] linting --- dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py b/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py index ca0435e374..56a33a453c 100644 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py @@ -978,8 +978,8 @@ def xml_handle_comment( """ line_info = (line_annotation, int(line_loc_no)) - if line_info in COMMENT_BLOCKS: - cmnt = COMMENT_BLOCKS.get_coment_by_line_info(line_info) + if line_info in COMMENT_BLOCKS: # noqa: F821 + cmnt = COMMENT_BLOCKS.get_coment_by_line_info(line_info) # noqa: F821 cmnt_text = cmnt.get_comment_text() if is_def_cmnt: @@ -1227,8 +1227,8 @@ def nyaml2nxdl(input_file: str, out_file, verbose: bool): recursive_build(xml_root, yml_appdef[name_extends], verbose) # Taking care of comments that comes at the end of file that is might not be intended for # any nxdl elements. - if COMMENT_BLOCKS[-1].has_post_comment: - post_comment = COMMENT_BLOCKS[-1] + if COMMENT_BLOCKS[-1].has_post_comment: # noqa: F821 + post_comment = COMMENT_BLOCKS[-1] # noqa: F821 (lin_annot, line_loc) = post_comment.get_line_info() xml_handle_comment(xml_root, lin_annot, line_loc) From 9c28230b7a8a396e4468cb59e1bb19c937189072 Mon Sep 17 00:00:00 2001 From: Sandor Brockhauser Date: Mon, 19 Jun 2023 15:48:06 +0200 Subject: [PATCH 48/93] imports --- dev_tools/nyaml2nxdl/comment_collector.py | 2 +- .../nyaml2nxdl/nyaml2nxdl_backward_tools.py | 8 ++++---- .../nyaml2nxdl/nyaml2nxdl_forward_tools.py | 20 +++++++++---------- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/dev_tools/nyaml2nxdl/comment_collector.py b/dev_tools/nyaml2nxdl/comment_collector.py index dcb21021be..0041c14ec4 100644 --- a/dev_tools/nyaml2nxdl/comment_collector.py +++ b/dev_tools/nyaml2nxdl/comment_collector.py @@ -38,7 +38,7 @@ from typing import Type from typing import Union -from pynxtools.nyaml2nxdl.nyaml2nxdl_helper import LineLoader +from .nyaml2nxdl_helper import LineLoader __all__ = ["Comment", "CommentCollector", "XMLComment", "YAMLComment"] diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py b/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py index faa22cc23e..c0f672305a 100755 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py @@ -27,10 +27,10 @@ from typing import Dict from typing import List -from pynxtools.dataconverter.helpers import remove_namespace_from_tag -from pynxtools.nyaml2nxdl.nyaml2nxdl_helper import cleaning_empty_lines -from pynxtools.nyaml2nxdl.nyaml2nxdl_helper import get_node_parent_info -from pynxtools.nyaml2nxdl.nyaml2nxdl_helper import get_yaml_escape_char_dict +from .nyaml2nxdl_helper import remove_namespace_from_tag +from .nyaml2nxdl_helper import cleaning_empty_lines +from .nyaml2nxdl_helper import get_node_parent_info +from .nyaml2nxdl_helper import get_yaml_escape_char_dict DEPTH_SIZE = " " CMNT_TAG = "!--" diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py b/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py index 56a33a453c..85b3ece550 100644 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py @@ -28,13 +28,13 @@ from xml.dom import minidom import yaml -from pynxtools.dataconverter.helpers import remove_namespace_from_tag -from pynxtools.nexus import nexus -from pynxtools.nyaml2nxdl.comment_collector import CommentCollector -from pynxtools.nyaml2nxdl.nyaml2nxdl_helper import LineLoader -from pynxtools.nyaml2nxdl.nyaml2nxdl_helper import cleaning_empty_lines -from pynxtools.nyaml2nxdl.nyaml2nxdl_helper import get_yaml_escape_char_reverter_dict -from pynxtools.nyaml2nxdl.nyaml2nxdl_helper import nx_name_type_resolving +from .nyaml2nxdl_helper import remove_namespace_from_tag +from ..utils import nexus as pynxtools_nxlib +from .nyaml2nxdl.comment_collector import CommentCollector +from .nyaml2nxdl_helper import LineLoader +from .nyaml2nxdl_helper import cleaning_empty_lines +from .nyaml2nxdl_helper import get_yaml_escape_char_reverter_dict +from .nyaml2nxdl_helper import nx_name_type_resolving # pylint: disable=too-many-lines, global-statement, invalid-name DOM_COMMENT = ( @@ -59,13 +59,13 @@ "#\n" "# For further information, see http://www.nexusformat.org\n" ) -NX_CLSS = nexus.get_nx_classes() +NX_CLSS = pynxtools_nxlib.get_nx_classes() NX_NEW_DEFINED_CLASSES = ["NX_COMPLEX"] -NX_TYPE_KEYS = nexus.get_nx_attribute_type() +NX_TYPE_KEYS = pynxtools_nxlib.get_nx_attribute_type() NX_ATTR_IDNT = "\\@" NX_UNIT_IDNT = "unit" DEPTH_SIZE = " " -NX_UNIT_TYPES = nexus.get_nx_units() +NX_UNIT_TYPES = pynxtools_nxlib.get_nx_units() COMMENT_BLOCKS: CommentCollector CATEGORY = "" # Definition would be either 'base' or 'application' From ddaaa6956471c44b2ec4c67a8914cc357d7a7bd2 Mon Sep 17 00:00:00 2001 From: Sandor Brockhauser Date: Mon, 19 Jun 2023 15:57:55 +0200 Subject: [PATCH 49/93] fixing imports --- dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py | 2 +- dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py b/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py index c0f672305a..dcf56b998d 100755 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py @@ -27,10 +27,10 @@ from typing import Dict from typing import List -from .nyaml2nxdl_helper import remove_namespace_from_tag from .nyaml2nxdl_helper import cleaning_empty_lines from .nyaml2nxdl_helper import get_node_parent_info from .nyaml2nxdl_helper import get_yaml_escape_char_dict +from .nyaml2nxdl_helper import remove_namespace_from_tag DEPTH_SIZE = " " CMNT_TAG = "!--" diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py b/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py index 85b3ece550..d54aa9f934 100644 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py @@ -28,13 +28,14 @@ from xml.dom import minidom import yaml -from .nyaml2nxdl_helper import remove_namespace_from_tag + from ..utils import nexus as pynxtools_nxlib from .nyaml2nxdl.comment_collector import CommentCollector from .nyaml2nxdl_helper import LineLoader from .nyaml2nxdl_helper import cleaning_empty_lines from .nyaml2nxdl_helper import get_yaml_escape_char_reverter_dict from .nyaml2nxdl_helper import nx_name_type_resolving +from .nyaml2nxdl_helper import remove_namespace_from_tag # pylint: disable=too-many-lines, global-statement, invalid-name DOM_COMMENT = ( From 357a6a896d0309ea906290aea7084cc93d604ed1 Mon Sep 17 00:00:00 2001 From: Sandor Brockhauser Date: Mon, 19 Jun 2023 18:19:29 +0200 Subject: [PATCH 50/93] test case added --- dev_tools/nyaml2nxdl/nyaml2nxdl.py | 24 +- .../nyaml2nxdl/nyaml2nxdl_forward_tools.py | 2 +- dev_tools/tests/test_nyaml2nxdl.py | 383 +----------------- 3 files changed, 34 insertions(+), 375 deletions(-) diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl.py b/dev_tools/nyaml2nxdl/nyaml2nxdl.py index 815b015e62..dccfff6e40 100755 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl.py @@ -26,13 +26,14 @@ import xml.etree.ElementTree as ET import click -from pynxtools.nyaml2nxdl.nyaml2nxdl_backward_tools import Nxdl2yaml -from pynxtools.nyaml2nxdl.nyaml2nxdl_backward_tools import compare_niac_and_my -from pynxtools.nyaml2nxdl.nyaml2nxdl_forward_tools import nyaml2nxdl -from pynxtools.nyaml2nxdl.nyaml2nxdl_forward_tools import pretty_print_xml -from pynxtools.nyaml2nxdl.nyaml2nxdl_helper import extend_yamlfile_with_comment -from pynxtools.nyaml2nxdl.nyaml2nxdl_helper import get_sha256_hash -from pynxtools.nyaml2nxdl.nyaml2nxdl_helper import separate_hash_yaml_and_nxdl + +from .nyaml2nxdl_backward_tools import Nxdl2yaml +from .nyaml2nxdl_backward_tools import compare_niac_and_my +from .nyaml2nxdl_forward_tools import nyaml2nxdl +from .nyaml2nxdl_forward_tools import pretty_print_xml +from .nyaml2nxdl_helper import extend_yamlfile_with_comment +from .nyaml2nxdl_helper import get_sha256_hash +from .nyaml2nxdl_helper import separate_hash_yaml_and_nxdl DEPTH_SIZE = 4 * " " @@ -152,15 +153,18 @@ def split_name_and_extension(file_name): Split file name into extension and rest of the file name. return file raw nam and extension """ - parts = file_name.rsplit(".", 3) + path = file_name.rsplit("/", 1) + (pathn, filen) = ["", path[0]] if len(path) == 1 else [path[0] + "/", path[1]] + parts = filen.rsplit(".", 2) + raw = ext = "" if len(parts) == 2: raw = parts[0] ext = parts[1] - if len(parts) == 3: + elif len(parts) == 3: raw = parts[0] ext = ".".join(parts[1:]) - return raw, ext + return pathn + raw, ext @click.command() diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py b/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py index d54aa9f934..984d7674f0 100644 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py @@ -30,7 +30,7 @@ import yaml from ..utils import nexus as pynxtools_nxlib -from .nyaml2nxdl.comment_collector import CommentCollector +from .comment_collector import CommentCollector from .nyaml2nxdl_helper import LineLoader from .nyaml2nxdl_helper import cleaning_empty_lines from .nyaml2nxdl_helper import get_yaml_escape_char_reverter_dict diff --git a/dev_tools/tests/test_nyaml2nxdl.py b/dev_tools/tests/test_nyaml2nxdl.py index d0c9f875a5..2722cf4755 100755 --- a/dev_tools/tests/test_nyaml2nxdl.py +++ b/dev_tools/tests/test_nyaml2nxdl.py @@ -1,372 +1,27 @@ -#!/usr/bin/env python3 -"""This tool accomplishes some tests for the yaml2nxdl parser - -""" -# -# Copyright The NOMAD Authors. -# -# This file is part of NOMAD. See https://nomad-lab.eu for further info. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - import os -import sys -import filecmp -from datetime import datetime -from pathlib import Path -import xml.etree.ElementTree as ET -import pytest -from click.testing import CliRunner -import pynxtools.nyaml2nxdl.nyaml2nxdl as nyml2nxdl -from pynxtools.nyaml2nxdl import nyaml2nxdl_forward_tools - - -def delete_duplicates(list_of_matching_string): - """ - Delete duplicate from lists - """ - return list(dict.fromkeys(list_of_matching_string)) - - -def check_file_fresh_baked(test_file): - """ - Get sure that the test file is generated by the converter - """ - path = Path(test_file) - timestamp = datetime.fromtimestamp(path.stat().st_mtime).strftime("%d/%m/%Y %H:%M") - now = datetime.now().strftime("%d/%m/%Y %H:%M") - assert timestamp == now, 'xml file not generated' - - -def find_matches(xml_file, desired_matches): - """ - Read xml file and find desired matches. Return a list of two lists in the form: - [[matching_line],[matching_line_index]] - """ - with open(xml_file, 'r') as file: - xml_reference = file.readlines() - lines = [] - lines_index = [] - found_matches = [] - for i, line in enumerate(xml_reference): - for desired_match in desired_matches: - if str(desired_match) in str(line): - lines.append(line) - lines_index.append(i) - found_matches.append(desired_match) - # ascertain that all the desired matches were found in file - found_matches_clean = delete_duplicates(found_matches) - assert len(found_matches_clean) == len(desired_matches), 'some desired_matches were \ -not found in file' - return [lines, lines_index] - - -def compare_matches(ref_xml_file, test_yml_file, test_xml_file, desired_matches): - """ - Check if a new xml file is generated - and if test xml file is equal to reference xml file - """ - # Reference file is read - ref_matches = find_matches(ref_xml_file, desired_matches) - # Test file is generated - runner = CliRunner() - result = runner.invoke(nyml2nxdl.launch_tool, ['--input-file', test_yml_file]) - assert result.exit_code == 0 - check_file_fresh_baked(test_xml_file) - # Test file is read - test_matches = find_matches(test_xml_file, desired_matches) - assert test_matches == ref_matches - - -def test_links(): - """ - Check the correct parsing of links - """ - data_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), - '../data/nyaml2nxdl') - ref_xml_link_file = 'tests/data/nyaml2nxdl/Ref_NXtest_links.nxdl.xml' - test_yml_link_file = 'tests/data/nyaml2nxdl/NXtest_links.yaml' - test_xml_link_file = 'tests/data/nyaml2nxdl/NXtest_links.nxdl.xml' - # ref_xml_link_file = os.path.abspath(data_path + '/Ref_NXtest_links.nxdl.xml') - # test_yml_link_file = os.path.abspath(data_path + '/NXtest_links.yaml') - # test_xml_link_file = os.path.abspath(data_path + '/NXtest_links.nxdl.xml') - desired_matches = [''] - compare_matches( - ref_xml_link_file, - test_yml_link_file, - test_xml_link_file, - desired_matches) - os.remove('tests/data/nyaml2nxdl/NXtest_links.nxdl.xml') - sys.stdout.write('Test on links okay.\n') - - -def test_docs(): - """In this test an xml file in converted to yml and then back to xml. - The xml trees of the two files are then compared. - """ - ref_xml_file = 'tests/data/nyaml2nxdl/Ref_NXellipsometry-docCheck.nxdl.xml' - test_yml_file = 'tests/data/nyaml2nxdl/NXellipsometry-docCheck.yaml' - test_xml_file = 'tests/data/nyaml2nxdl/NXellipsometry-docCheck.nxdl.xml' - desired_matches = [''] - compare_matches( - ref_xml_file, - test_yml_file, - test_xml_file, - desired_matches) - os.remove('tests/data/nyaml2nxdl/NXellipsometry-docCheck.nxdl.xml') - sys.stdout.write('Test on documentation formatting okay.\n') - - -def test_nxdl2yaml_doc_format_and_nxdl_part_as_comment(): - """ - This test for two reason: - 1. In test-1 an nxdl file with all kind of doc formats are translated - to yaml to check if they are correct. - 2. In test-2: Check the nxdl that comes at the end of yaml file as comment. - """ - ref_xml_file = 'tests/data/nyaml2nxdl/Ref_NXentry.nxdl.xml' - ref_yml_file = 'tests/data/nyaml2nxdl/Ref_NXentry.yaml' - test_yml_file = 'tests/data/nyaml2nxdl/Ref_NXentry_parsed.yaml' - result = CliRunner().invoke(nyml2nxdl.launch_tool, ['--input-file', ref_xml_file]) - assert result.exit_code == 0 - check_file_fresh_baked(test_yml_file) - - result = filecmp.cmp(ref_yml_file, test_yml_file, shallow=False) - assert result, 'Ref YML and parsed YML\ -has not the same structure!!' - os.remove(test_yml_file) - sys.stdout.write('Test on xml -> yml doc formatting okay.\n') - - -def test_fileline_error(): - """ - In this test the yaml fileline in the error message is tested. - """ - test_yml_file = 'tests/data/nyaml2nxdl/NXfilelineError1.yaml' - out_nxdl = 'tests/data/nyaml2nxdl/NXfilelineError1.nxdl.xml' - out_yaml = 'tests/data/nyaml2nxdl/temp_NXfilelineError1.yaml' - result = CliRunner().invoke(nyml2nxdl.launch_tool, ['--input-file', test_yml_file]) - assert result.exit_code == 1 - assert '13' in str(result.exception) - os.remove(out_nxdl) - os.remove(out_yaml) - - test_yml_file = 'tests/data/nyaml2nxdl/NXfilelineError2.yaml' - out_nxdl = 'tests/data/nyaml2nxdl/NXfilelineError2.nxdl.xml' - out_yaml = 'tests/data/nyaml2nxdl/temp_NXfilelineError2.yaml' - result = CliRunner().invoke(nyml2nxdl.launch_tool, ['--input-file', test_yml_file]) - assert result.exit_code == 1 - assert '21' in str(result.exception) - os.remove(out_nxdl) - os.remove(out_yaml) - - test_yml_file = 'tests/data/nyaml2nxdl/NXfilelineError3.yaml' - out_nxdl = 'tests/data/nyaml2nxdl/NXfilelineError3.nxdl.xml' - out_yaml = 'tests/data/nyaml2nxdl/temp_NXfilelineError3.yaml' - result = CliRunner().invoke(nyml2nxdl.launch_tool, ['--input-file', test_yml_file]) - assert result.exit_code == 1 - assert '25' in str(result.exception) - os.remove(out_nxdl) - os.remove(out_yaml) - - sys.stdout.write('Test on xml -> yml fileline error handling okay.\n') - - -def test_symbols(): - """ - Check the correct parsing of symbols - """ - ref_xml_symbol_file = 'tests/data/nyaml2nxdl/Ref_NXnested_symbols.nxdl.xml' - test_yml_symbol_file = 'tests/data/nyaml2nxdl/NXnested_symbols.yaml' - test_xml_symbol_file = 'tests/data/nyaml2nxdl/NXnested_symbols.nxdl.xml' - desired_matches = ['', '', '', '', '', - '', ''] - compare_matches( - ref_xml_attribute_file, - test_yml_attribute_file, - test_xml_attribute_file, - desired_matches) - os.remove('tests/data/nyaml2nxdl/NXattributes.nxdl.xml') - sys.stdout.write('Test on attributes okay.\n') +from click.testing import CliRunner -def test_extends(): - """ - Check the correct handling of extends keyword - """ - ref_xml_attribute_file = 'tests/data/nyaml2nxdl/Ref_NXattributes.nxdl.xml' - test_yml_attribute_file = 'tests/data/nyaml2nxdl/NXattributes.yaml' - test_xml_attribute_file = 'tests/data/nyaml2nxdl/NXattributes.nxdl.xml' - runner = CliRunner() - result = runner.invoke(nyml2nxdl.launch_tool, ['--input-file', test_yml_attribute_file]) - assert result.exit_code == 0 - ref_root_node = ET.parse(ref_xml_attribute_file).getroot() - test_root_node = ET.parse(test_xml_attribute_file).getroot() - assert ref_root_node.attrib == test_root_node.attrib - os.remove('tests/data/nyaml2nxdl/NXattributes.nxdl.xml') - sys.stdout.write('Test on extends keyword okay.\n') - - -def test_symbols_and_enum_docs(): - """ - Check the correct handling of empty attributes - or attributes fields, e.g. doc - """ - ref_xml_file = 'tests/data/nyaml2nxdl/Ref_NXmytests.nxdl.xml' - test_yml_file = 'tests/data/nyaml2nxdl/NXmytests.yaml' - test_xml_file = 'tests/data/nyaml2nxdl/NXmytests.nxdl.xml' - desired_matches = ['', '', '', - '', '', '', ' yml -> xml okay.\n') +# import subprocess -def test_yml_parsing(): - """In this test an xml file in converted to yml and then back to xml. - The xml trees of the two files are then compared. - """ - ref_yml_file = 'tests/data/nyaml2nxdl/Ref_NXellipsometry.yaml' - test_xml_file = 'tests/data/nyaml2nxdl/Ref_NXellipsometry.nxdl.xml' - test_yml_file = 'tests/data/nyaml2nxdl/Ref_NXellipsometry_parsed.yaml' - result = CliRunner().invoke(nyml2nxdl.launch_tool, ['--input-file', ref_yml_file]) - assert result.exit_code == 0 - check_file_fresh_baked(test_xml_file) - result = CliRunner().invoke(nyml2nxdl.launch_tool, ['--input-file', test_xml_file]) +def test_conversion(): + root = find_definition_file("NXentry") + # subprocess.run(["python3","-m","dev_tools.nyaml2nxdl.nyaml2nxdl","--input-file",root]) + result = CliRunner().invoke(conv.launch_tool, ["--input-file", root]) assert result.exit_code == 0 - check_file_fresh_baked(test_yml_file) - - test_yml_tree = nyaml2nxdl_forward_tools.yml_reader(test_yml_file) - - ref_yml_tree = nyaml2nxdl_forward_tools.yml_reader(ref_yml_file) - - assert list(test_yml_tree) == list(ref_yml_tree), 'Ref YML and parsed YML \ -has not the same root entries!!' - os.remove('tests/data/nyaml2nxdl/Ref_NXellipsometry_parsed.yaml') - os.remove('tests/data/nyaml2nxdl/Ref_NXellipsometry.nxdl.xml') - sys.stdout.write('Test on yml -> xml -> yml okay.\n') - - -def test_yml_consistency_comment_parsing(): - """Test comments parsing from yaml. Convert 'yaml' input file to '.nxdl.xml' and - '.nxdl.xml' to '.yaml' - """ - from pynxtools.nyaml2nxdl.comment_collector import CommentCollector - from pynxtools.nyaml2nxdl.nyaml2nxdl_helper import LineLoader - - ref_yml_file = 'tests/data/nyaml2nxdl/Ref_NXcomment.yaml' - test_yml_file = 'tests/data/nyaml2nxdl/Ref_NXcomment_consistency.yaml' - - result = CliRunner().invoke(nyml2nxdl.launch_tool, - ['--input-file', ref_yml_file, - '--check-consistency']) - assert result.exit_code == 0, (f'Exception: {result.exception}, \nExecution Info:' - '{result.exc_info}') - with open(ref_yml_file, 'r', encoding='utf-8') as ref_yml: - loader = LineLoader(ref_yml) - ref_loaded_yaml = loader.get_single_data() - ref_comment_blocks = CommentCollector(ref_yml_file, ref_loaded_yaml) - ref_comment_blocks.extract_all_comment_blocks() - - with open(test_yml_file, 'r', encoding='utf-8') as test_yml: - loader = LineLoader(test_yml) - test_loaded_yaml = loader.get_single_data() - test_comment_blocks = CommentCollector(test_yml_file, test_loaded_yaml) - test_comment_blocks.extract_all_comment_blocks() - - for ref_cmnt, test_cmnt in zip(ref_comment_blocks, test_comment_blocks): - assert ref_cmnt == test_cmnt, 'Comment is not consistent.' - - os.remove(test_yml_file) - - -def test_yml2xml_comment_parsing(): - """To test comment that written in xml for element attributes, e.g. - attribute 'rank' for 'dimension' element and attribute 'exists' for - 'NXentry' group element. - """ - input_yml = 'tests/data/nyaml2nxdl/NXcomment_yaml2nxdl.yaml' - ref_xml = 'tests/data/nyaml2nxdl/Ref_NXcomment_yaml2nxdl.nxdl.xml' - test_xml = 'tests/data/nyaml2nxdl/NXcomment_yaml2nxdl.nxdl.xml' - - result = CliRunner().invoke(nyml2nxdl.launch_tool, - ['--input-file', input_yml]) + yaml = root[:-9] + "_parsed.yaml" + # subprocess.run(["python3","-m","dev_tools.nyaml2nxdl.nyaml2nxdl","--input-file",yaml]) + result = CliRunner().invoke(conv.launch_tool, ["--input-file", yaml]) assert result.exit_code == 0 - - ref_root = ET.parse(ref_xml).getroot() - test_root = ET.parse(test_xml).getroot() - - def recursive_compare(ref_root, test_root): - assert ref_root.attrib.items() == test_root.attrib.items(), ("Got different xml element" - "Atribute.") - if ref_root.text and test_root.text: - assert ref_root.text.strip() == test_root.text.strip(), ("Got differen element text.") - if len(ref_root) > 0 and len(test_root) > 0: - for x, y in zip(ref_root, test_root): - recursive_compare(x, y) - - recursive_compare(ref_root, test_root) - - os.remove(test_xml) + new_root = yaml[:-4] + "nxdl.xml" + with open(root, encoding="utf-8", mode="r") as tmp_f: + root_content = tmp_f.readlines() + with open(new_root, encoding="utf-8", mode="r") as tmp_f: + new_root_content = tmp_f.readlines() + assert root_content == new_root_content + os.remove(yaml) + os.remove(new_root) From 948b044485594f27ed0cee4f4584ff56747c4b40 Mon Sep 17 00:00:00 2001 From: Sherjeel Shabih Date: Wed, 12 Jul 2023 10:19:43 +0200 Subject: [PATCH 51/93] Moved over changes from older nyaml PR --- .gitignore | 5 ----- Makefile | 13 +++++++++++++ pyproject.toml | 3 ++- requirements.txt | 1 + 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 7f125837e5..7867d76657 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ # Hidden files .* -!.github # Python byte / compiled / optimized *.py[cod] @@ -10,10 +9,6 @@ __pycache__/ build/ makelog.txt -# Unknown -/python/ -__github_creds__.txt - # Distribution / packaging .Python build/ diff --git a/Makefile b/Makefile index ae556d7339..113e29db8a 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,7 @@ PYTHON = python3 SPHINX = sphinx-build BUILD_DIR = "build" +NXDL_DIRS := contributed_definitions applications base_classes .PHONY: help install style autoformat test clean prepare html pdf impatient-guide all local @@ -49,6 +50,9 @@ test :: clean :: $(RM) -rf $(BUILD_DIR) + for dir in $(NXDL_DIRS); do\ + $(RM) -rf $${dir}/nyaml;\ + done prepare :: $(PYTHON) -m dev_tools manual --prepare --build-root $(BUILD_DIR) @@ -83,6 +87,15 @@ all :: @echo "HTML built: `ls -lAFgh $(BUILD_DIR)/manual/build/html/index.html`" @echo "PDF built: `ls -lAFgh $(BUILD_DIR)/manual/build/latex/nexus.pdf`" +NXDLS := $(foreach dir,$(NXDL_DIRS),$(wildcard $(dir)/*.nxdl.xml)) +nyaml : $(DIRS) $(NXDLS) + for file in $^; do\ + mkdir -p "$${file%/*}/nyaml";\ + nyaml2nxdl --input-file $${file};\ + FNAME=$${file##*/};\ + mv -- "$${file%.nxdl.xml}_parsed.yaml" "$${file%/*}/nyaml/$${FNAME%.nxdl.xml}.yaml";\ + done + # NeXus - Neutron and X-ray Common Data Format # diff --git a/pyproject.toml b/pyproject.toml index 979c51eac0..67878319bb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,9 +18,9 @@ classifiers = [ dependencies = [ "lxml", "pyyaml", + "click>=7.1.2", "sphinx>=5", "sphinx-tabs", - "sphinx-toolbox", "pytest", "black>=22.3", "flake8>=4", @@ -31,6 +31,7 @@ dependencies = [ "Homepage" = "https://nexusformat.org" [project.scripts] +nyaml2nxdl = "dev_tools.nyaml2nxdl.nyaml2nxdl:launch_tool" [tools.setuptools_scm] version_scheme = "guess-next-dev" diff --git a/requirements.txt b/requirements.txt index 54b7bb86f8..91c5ae31a1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ # Prepare for Documentation lxml pyyaml +click # Documentation building sphinx>=5 From bd2d198f0de4d8e760bdfe5dc5f9594102bc4ae2 Mon Sep 17 00:00:00 2001 From: Sandor Brockhauser Date: Wed, 12 Jul 2023 14:17:06 +0200 Subject: [PATCH 52/93] linting --- dev_tools/docs/nxdl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev_tools/docs/nxdl.py b/dev_tools/docs/nxdl.py index 0369a62973..b873e32871 100755 --- a/dev_tools/docs/nxdl.py +++ b/dev_tools/docs/nxdl.py @@ -12,8 +12,8 @@ from ..globals.errors import NXDLParseError from ..globals.nxdl import NXDL_NAMESPACE from ..globals.urls import REPO_URL -from ..utils.github import get_file_contributors_via_api from ..utils import nxdl_utils as pynxtools_nxlib +from ..utils.github import get_file_contributors_via_api from ..utils.types import PathLike from .anchor_list import AnchorRegistry From 1e959c09ab9d2d4dc2a1f0fb146c8efa2b72dcfc Mon Sep 17 00:00:00 2001 From: Sandor Brockhauser Date: Wed, 12 Jul 2023 16:09:42 +0200 Subject: [PATCH 53/93] make nxdl command is added to support the reverse covertsion that was requested during code camp --- Makefile | 9 +++++++++ README.md | 14 +++++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 113e29db8a..eff518f189 100644 --- a/Makefile +++ b/Makefile @@ -96,6 +96,15 @@ nyaml : $(DIRS) $(NXDLS) mv -- "$${file%.nxdl.xml}_parsed.yaml" "$${file%/*}/nyaml/$${FNAME%.nxdl.xml}.yaml";\ done +NYAMLS := $(foreach dir,$(NXDL_DIRS),$(wildcard $(dir)/nyaml/*.yaml)) +nxdl : $(DIRS) $(NYAMLS) + for file in $^; do\ + mkdir -p "$${file%/*}/nyaml";\ + nyaml2nxdl --input-file $${file};\ + FNAME=$${file##*/};\ + mv -- "$${file%.yaml}.nxdl.xml" "$${file%/*}/../$${FNAME%.yaml}.nxdl.xml";\ + done + # NeXus - Neutron and X-ray Common Data Format # diff --git a/README.md b/README.md index 76112c7704..914d51f684 100755 --- a/README.md +++ b/README.md @@ -7,6 +7,10 @@ ## NeXus definition developers +Note if this package was not properly installed using pip, it has to be first done by + + pip install -e . + After making a change to the NeXus class definitions there are two important checks to be made before commiting the change: @@ -25,6 +29,14 @@ Open the HTML manual in a web brower for visual verification firefox build/manual/build/html/index.html +Convenient editting of definitinos is available in nyaml format. For this, definitinos need to be converted first from xml to yaml format by the command + + make nyaml + +After editing the definitions in nyaml format in the nyaml subdirectories, the following command can be used to update the definitins in nxdl.xml format: + + make nxdl + ### HTML pages with contributor information To build the html pages that contains contributor information on the sidebar set a github access token to an environment variable called GH_TOKEN. @@ -57,4 +69,4 @@ package/ | directory for packaging this content utils/ | various tools used in the definitions tree www/ | launch (home) page of NeXus WWW site xslt/ | various XML stylesheet transformations -dev_tools/ | developer tools for testing and building \ No newline at end of file +dev_tools/ | developer tools for testing and building From 0f4d7233762a906c4b927a16f8d05b4f840c20c7 Mon Sep 17 00:00:00 2001 From: Sandor Brockhauser Date: Wed, 12 Jul 2023 16:30:22 +0200 Subject: [PATCH 54/93] fixing typos and dependencies --- README.md | 4 ++-- dev_tools/nyaml2nxdl/README.md | 8 ++++---- dev_tools/nyaml2nxdl/nyaml2nxdl.py | 2 +- pyproject.toml | 1 + 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 914d51f684..4f4020aa6f 100755 --- a/README.md +++ b/README.md @@ -29,11 +29,11 @@ Open the HTML manual in a web brower for visual verification firefox build/manual/build/html/index.html -Convenient editting of definitinos is available in nyaml format. For this, definitinos need to be converted first from xml to yaml format by the command +Convenient editting of definitions is available in nyaml format. For this, definitions need to be converted first from xml to yaml format by the command make nyaml -After editing the definitions in nyaml format in the nyaml subdirectories, the following command can be used to update the definitins in nxdl.xml format: +After editing the definitions in nyaml format in the nyaml subdirectories, the following command can be used to update the definitions in nxdl.xml format: make nxdl diff --git a/dev_tools/nyaml2nxdl/README.md b/dev_tools/nyaml2nxdl/README.md index ff083e1896..ec69275c25 100644 --- a/dev_tools/nyaml2nxdl/README.md +++ b/dev_tools/nyaml2nxdl/README.md @@ -7,7 +7,7 @@ such as base or contributed classes. Users either create NeXus instances by writ The forward (YAML -> NXDL.XML) and backward (NXDL.XML -> YAML) conversions are implemented. **How the tool works**: -- yaml2nxdl.py +- nyaml2nxdl.py 1. Reads the user-specified NeXus instance, either in YML or XML format. 2. If input is in YAML, creates an instantiated NXDL schema XML tree by walking the dictionary nest. If input is in XML, creates a YML file walking the dictionary nest. @@ -16,12 +16,12 @@ The forward (YAML -> NXDL.XML) and backward (NXDL.XML -> YAML) conversions are i the XML or YAML input file is interpreted as an extension of a base class and the entries contained in it are appended below a standard NeXus base class. You need to specify both your input file (with YAML or XML extension) and NeXus class (with no extension). - Both .yml and .nxdl.xml file of the extended class are printed. + Both .yaml and .nxdl.xml file of the extended class are printed. ```console -user@box:~$ python yaml2nxdl.py +user@box:~$ python nyaml2nxdl.py -Usage: python yaml2nxdl.py [OPTIONS] +Usage: python nyaml2nxdl.py [OPTIONS] Options: --input-file TEXT The path to the input data file to read. diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl.py b/dev_tools/nyaml2nxdl/nyaml2nxdl.py index dccfff6e40..29d2db04ff 100755 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Main file of yaml2nxdl tool. +"""Main file of nyaml2nxdl tool. Users create NeXus instances by writing a YAML file which details a hierarchy of data/metadata elements diff --git a/pyproject.toml b/pyproject.toml index 67878319bb..f4377ff3f4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ dependencies = [ "click>=7.1.2", "sphinx>=5", "sphinx-tabs", + "sphinx-toolbox", "pytest", "black>=22.3", "flake8>=4", From 020175fcf4d5d0513c1ff015ae291fff930f57e8 Mon Sep 17 00:00:00 2001 From: Sandor Brockhauser Date: Fri, 14 Jul 2023 13:16:51 +0200 Subject: [PATCH 55/93] add again the lost function remove_namespace_from_tag --- dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py b/dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py index f4787bd300..c55f5da7a8 100644 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py @@ -41,6 +41,12 @@ ESCAPE_CHAR_DICT_IN_XML = {" ": "\t", "':'": ":"} +def remove_namespace_from_tag(tag): + """Helper function to remove the namespace from an XML tag.""" + + return tag.split("}")[-1] + + class LineLoader(Loader): # pylint: disable=too-many-ancestors """ LineLoader parses a yaml into a python dictionary extended with extra items. From e036a0181d8085c279ed02373f7ff557e2869e7a Mon Sep 17 00:00:00 2001 From: Sandor Brockhauser Date: Wed, 21 Jun 2023 13:04:34 +0200 Subject: [PATCH 56/93] fixing imports --- dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py | 2 +- dev_tools/tests/test_nyaml2nxdl.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py b/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py index 984d7674f0..664f687484 100644 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py @@ -29,7 +29,7 @@ import yaml -from ..utils import nexus as pynxtools_nxlib +from ..utils import nxdl_utils as pynxtools_nxlib from .comment_collector import CommentCollector from .nyaml2nxdl_helper import LineLoader from .nyaml2nxdl_helper import cleaning_empty_lines diff --git a/dev_tools/tests/test_nyaml2nxdl.py b/dev_tools/tests/test_nyaml2nxdl.py index 2722cf4755..792d8d4626 100755 --- a/dev_tools/tests/test_nyaml2nxdl.py +++ b/dev_tools/tests/test_nyaml2nxdl.py @@ -3,7 +3,7 @@ from click.testing import CliRunner from ..nyaml2nxdl import nyaml2nxdl as conv -from ..utils.nexus import find_definition_file +from ..utils.nxdl_utils import find_definition_file # import subprocess From 66c20a6568ee6daeb9f4372c87af706f3593e4c5 Mon Sep 17 00:00:00 2001 From: Rubel Date: Mon, 4 Sep 2023 12:37:59 +0200 Subject: [PATCH 57/93] Modification for changes request from 1303 PR. --- dev_tools/docs/nxdl.py | 15 ++-- dev_tools/nyaml2nxdl/README.md | 4 +- dev_tools/nyaml2nxdl/__init__.py | 1 - dev_tools/nyaml2nxdl/comment_collector.py | 100 +++++++++++----------- 4 files changed, 58 insertions(+), 62 deletions(-) diff --git a/dev_tools/docs/nxdl.py b/dev_tools/docs/nxdl.py index b873e32871..26c53cb208 100755 --- a/dev_tools/docs/nxdl.py +++ b/dev_tools/docs/nxdl.py @@ -537,9 +537,6 @@ def _print_doc_enum(self, indent, ns, node, required=False): collapse_indent = indent node_list = node.xpath("nx:enumeration", namespaces=ns) (doclen, line, blocks) = self.long_doc(ns, node) - # if len(node_list) + doclen > 1: - # collapse_indent = f"{indent} " - # self._print(f"{indent}{self._INDENTATION_UNIT}.. collapse:: {line} ...\n") self._print_doc( collapse_indent + self._INDENTATION_UNIT, ns, node, required=required ) @@ -670,8 +667,9 @@ def _print(self, *args, end="\n"): self._rst_lines.append(" ".join(args) + end) def get_first_parent_ref(self, path, tag): - nx_name = path[1 : path.find("/", 1)] - path = path[path.find("/", 1) :] + spliter = path.find("/", 1) + nx_name = path[1 : spliter] + path = path[spliter :] try: parents = pynxtools_nxlib.get_inherited_nodes(path, nx_name)[2] @@ -681,10 +679,9 @@ def get_first_parent_ref(self, path, tag): parent = parents[1] parent_path = parent_display_name = parent.attrib["nxdlpath"] parent_path_segments = parent_path[1:].split("/") - parent_def_name = parent.attrib["nxdlbase"][ - parent.attrib["nxdlbase"] - .rfind("/") : parent.attrib["nxdlbase"] - .rfind(".nxdl") + nxdl_attr = parent.attrib["nxdlbase"] + parent_def_name = nxdl_attr[nxdl_attr.rfind("/") : + nxdl_attr.rfind(".nxdl") ] # Case where the first parent is a base_class diff --git a/dev_tools/nyaml2nxdl/README.md b/dev_tools/nyaml2nxdl/README.md index ec69275c25..a8af60ad3e 100644 --- a/dev_tools/nyaml2nxdl/README.md +++ b/dev_tools/nyaml2nxdl/README.md @@ -30,7 +30,7 @@ Options: --check-consistency Check consistency by generating another version of the input file. E.g. for input file: NXexample.nxdl.xml the output file NXexample_consistency.nxdl.xml. - --verbose Addictional std output info is printed to help debugging. + --verbose Additional std output info is printed to help debugging. --help Show this message and exit. ``` @@ -40,7 +40,7 @@ Options: **Rule set**: From transcoding YAML files we need to follow several rules. * Named NeXus groups, which are instances of NeXus classes especially base or contributed classes. Creating (NXbeam) is a simple example of a request to define a group named according to NeXus default rules. mybeam1(NXbeam) or mybeam2(NXbeam) are examples how to create multiple named instances at the same hierarchy level. * Members of groups so-called fields or attributes. A simple example of a member is voltage. Here the datatype is implied automatically as the default NeXus NX_CHAR type. By contrast, voltage(NX_FLOAT) can be used to instantiate a member of class which should be of NeXus type NX_FLOAT. -* And attributes of either groups or fields. Names of attributes have to be preceeded by \@ to mark them as attributes. +* And attributes of either groups or fields. The mark '\@' have to precede the name of attributes. * Optionality: For all fields, groups and attributes in `application definitions` are `required` by default, except anything (`recommended` or `optional`) mentioned. **Special keywords**: Several keywords can be used as childs of groups, fields, and attributes to specify the members of these. Groups, fields and attributes are nodes of the XML tree. diff --git a/dev_tools/nyaml2nxdl/__init__.py b/dev_tools/nyaml2nxdl/__init__.py index 22eb35f68d..d0f7a80f2f 100644 --- a/dev_tools/nyaml2nxdl/__init__.py +++ b/dev_tools/nyaml2nxdl/__init__.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ # Load paths """ diff --git a/dev_tools/nyaml2nxdl/comment_collector.py b/dev_tools/nyaml2nxdl/comment_collector.py index 0041c14ec4..b4dcfac6f3 100644 --- a/dev_tools/nyaml2nxdl/comment_collector.py +++ b/dev_tools/nyaml2nxdl/comment_collector.py @@ -19,15 +19,12 @@ # """ -Collect comments in a list by CommentCollector class. Comment is a instance of Comment, -where each comment includes comment text and line info or neighbour info where the -comment must be assinged. +Collect comments in a list by CommentCollector class and each comment is a instance of Comment +class. Comment (sometimes refered as 'comment block') instance consists of text and line info or neighbours info where the +comment must be placed. The class Comment is an abstract class for general functions or method to be implemented XMLComment and YAMLComment class. - -NOTE: Here comment block mainly stands for (comment text + line or element for what comment is -intended.) """ @@ -62,13 +59,17 @@ def __init__(self, input_file: str = None, loaded_obj: Union[object, Dict] = Non self._comment_hash: Dict[Tuple, Type[Comment]] = {} self.comment: Type[Comment] if self.file and not loaded_obj: - if self.file.split(".")[-1] == "xml": + if self.file.endswith(".xml"): self.comment = XMLComment - if self.file.split(".")[-1] == "yaml": + elif self.file.split(".")[-1] == "yaml": self.comment = YAMLComment with open(self.file, "r", encoding="utf-8") as plain_text_yaml: loader = LineLoader(plain_text_yaml) self.comment.__yaml_dict__ = loader.get_single_data() + else: + raise ValueError( + "Input file must be a 'yaml' or 'nxdl.xml' type." + ) elif self.file and loaded_obj: if self.file.split(".")[-1] == "yaml" and isinstance(loaded_obj, dict): self.comment = YAMLComment @@ -93,18 +94,18 @@ def extract_all_comment_blocks(self): # Make an empty line for last comment if no empty lines in original file if lines[-1] != "": lines.append("") + end_line_num = len(lines) - 1 for line_num, line in enumerate(lines): if single_comment.is_storing_single_comment(): # If the last comment comes without post nxdl fields, groups and attributes if "++ SHA HASH ++" in line: # Handle with stored nxdl.xml file that is not part of yaml - line = "" single_comment.process_each_line( - line + "post_comment", (line_num + 1) + "post_comment", (line_num + 1) ) self._comment_chain.append(single_comment) break - if line_num < (len(lines) - 1): + if line_num < end_line_num: # Processing file from Line number 1 single_comment.process_each_line(line, (line_num + 1)) else: @@ -118,7 +119,7 @@ def extract_all_comment_blocks(self): single_comment = self.comment(last_comment=single_comment) single_comment.process_each_line(line, (line_num + 1)) - def get_comment(self): + def get_next_comment(self): """ Return comment from comment_chain that must come earlier in order. """ @@ -228,7 +229,7 @@ def __init__(self, comment_id: int = -1, last_comment: "Comment" = None) -> None self._comnt_end_found and self._is_elemt_stored ) - def get_comment_text(self) -> Union[List, str]: + def get_comment_text_list(self) -> Union[List, str]: """ Extract comment text from entrire comment (comment text + elment or line for what comment is intended) @@ -292,39 +293,39 @@ def store_element(self, text) -> None: def collect_xml_attributes(text_part): for part in text_part: part = part.strip() - if part and '">' == "".join(part[-2:]): + if part and part.endswith('">'): self._is_elemt_stored = True self._is_elemt_found = False - part = "".join(part[0:-2]) - elif part and '"/>' == "".join(part[-3:]): + part = part[0:-2] + elif part and part.endswith('"/>'): self._is_elemt_stored = True self._is_elemt_found = False - part = "".join(part[0:-3]) - elif part and "/>" == "".join(part[-2:]): + part = part[0:-3] + elif part and part.endswith("/>"): self._is_elemt_stored = True self._is_elemt_found = False - part = "".join(part[0:-2]) - elif part and ">" == part[-1]: + part = part[0:-2] + elif part and part.endswith(">"): self._is_elemt_stored = True self._is_elemt_found = False - part = "".join(part[0:-1]) - elif part and '"' == part[-1]: - part = "".join(part[0:-1]) + part = part[0:-1] + elif part and part.endswith('"'): + part = part[0:-1] if '="' in part: lf_prt, rt_prt = part.split('="') + if ":" in lf_prt: + continue else: continue - if ":" in lf_prt: - continue self._elemt[lf_prt] = str(rt_prt) if not self._elemt: self._elemt = {} # First check for comment part has been collected prefectly - if " Union[List, str]: + def get_comment_text_list(self) -> Union[List, str]: """ This method returns list of commnent text. As some xml element might have multiple separated comment intended for a single element. @@ -354,10 +355,10 @@ def get_comment_text(self) -> Union[List, str]: class YAMLComment(Comment): """ - This class for stroing comment text as well as location of the comment e.g. line + This class for storing comment text as well as location of the comment e.g. line number of other in the file. NOTE: - 1. Do not delete any element form yaml dictionary (for loaded_obj. check: Comment_collector + 1. Do not delete any element from yaml dictionary (for loaded_obj. check: Comment_collector class. because this loaded file has been exploited in nyaml2nxdl forward tools.) """ @@ -387,9 +388,9 @@ def process_each_line(self, text, line_num): if self._comnt_end_found: line_key = "" - if ":" in text: - ind = text.index(":") - line_key = "__line__" + "".join(text[0:ind]) + ind = text.find(":") + if ind > 0: + line_key = "__line__" + text[0:ind] for l_num, l_key in self.__yaml_line_info.items(): if line_num == int(l_num) and line_key == l_key: @@ -425,16 +426,16 @@ def append_comment(self, text: str) -> None: # For empty line inside doc or yaml file. elif not text: return - elif "# " == "".join(text[0:2]): + elif text.startswith("# "): self._comnt_start_found = True self._comnt_end_found = False self._comnt = "" if not self._comnt else self._comnt + "\n" - self._comnt = self._comnt + "".join(text[2:]) - elif "#" == text[0]: + self._comnt = self._comnt + text[2:] + elif text.startswith("#"): self._comnt_start_found = True self._comnt_end_found = False self._comnt = "" if not self._comnt else self._comnt + "\n" - self._comnt = self._comnt + "".join(text[1:]) + self._comnt = self._comnt + text[1:] elif "post_comment" == text: self._comnt_end_found = True self._comnt_start_found = False @@ -446,7 +447,7 @@ def append_comment(self, text: str) -> None: # pylint: disable=arguments-differ def store_element(self, line_key, line_number): """ - Store comment content and information of commen location (for what comment is + Store comment contents and information of comment location (for what comment is created.). """ self._elemt = {} @@ -454,15 +455,15 @@ def store_element(self, line_key, line_number): self._is_elemt_found = False self._is_elemt_stored = True - def get_comment_text(self): + def get_comment_text_list(self): """ - Return list of comments if there are multiple comment for same yaml line. + Return list of comments. """ return self._comnt_list def get_line_number(self, line_key): """ - Retrun line number for what line the comment is created + Return line number for which line the comment is created """ return self._elemt[line_key] @@ -473,22 +474,19 @@ def get_line_info(self): for line_anno, line_loc in self._elemt.items(): return line_anno, line_loc - def replace_scape_char(self, text): + def replace_escape_char(self, text): """Replace escape char according to __comment_escape_char dict""" for ecp_char, ecp_alt in YAMLComment.__comment_escape_char.items(): - if ecp_char in text: - text = text.replace(ecp_char, ecp_alt) + text = text.replace(ecp_char, ecp_alt) return text def get_element_location(self): """ - Retrun yaml line '__line__KEY' info and and line numner + Return yaml line '__line__KEY' info and and line numner """ - if len(self._elemt) > 1: + if len(self._elemt) == 1: raise ValueError(f"Comment element should be one but got " f"{self._elemt}") - - for key, val in self._elemt.items(): - yield key, val + return next(self._elemt.items()) def collect_yaml_line_info(self, yaml_dict, line_info_dict): """Collect __line__key and corresponding value from @@ -503,11 +501,13 @@ def collect_yaml_line_info(self, yaml_dict, line_info_dict): self.collect_yaml_line_info(val, line_info_dict) def __contains__(self, line_key): - """For Checking whether __line__NAME is in _elemt dict or not.""" + """For checking whether __line__NAME is in _elemt dict or not.""" return line_key in self._elemt def __eq__(self, comment_obj): """Check the self has same value as right comment.""" + if not isinstance(comment_obj, Comment): + raise TypeError(f"Expecting comment_obj as a instance of {Comment}") if len(self._comnt_list) != len(comment_obj._comnt_list): return False for left_cmnt, right_cmnt in zip(self._comnt_list, comment_obj._comnt_list): From 4d3d8f5065a10183fac3ccf5f4ca132693c196d9 Mon Sep 17 00:00:00 2001 From: Rubel Date: Wed, 6 Sep 2023 11:55:08 +0200 Subject: [PATCH 58/93] changes from Peter and Pete. --- dev_tools/nyaml2nxdl/nyaml2nxdl.py | 287 +++++++++++++++-------------- 1 file changed, 146 insertions(+), 141 deletions(-) diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl.py b/dev_tools/nyaml2nxdl/nyaml2nxdl.py index 29d2db04ff..5e139ea270 100755 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl.py @@ -1,8 +1,10 @@ #!/usr/bin/env python3 -"""Main file of nyaml2nxdl tool. -Users create NeXus instances by writing a YAML file -which details a hierarchy of data/metadata elements +""" +Main file of nyaml2nxdl tool. +To write a definition for a instrument, experiment and/or process in nxdl.xml file from a YAML +file which details a hierarchy of data/metadata elements. It also allows both way +conversion beteen YAML and nxdl.xml files that follows rules of NeXus ontology or data format. """ # -*- coding: utf-8 -*- # @@ -23,6 +25,7 @@ # limitations under the License. # import os +from pathlib import Path import xml.etree.ElementTree as ET import click @@ -36,6 +39,7 @@ from .nyaml2nxdl_helper import separate_hash_yaml_and_nxdl DEPTH_SIZE = 4 * " " +_nxdl = ".nxdl.xml" # NOTE: Some handful links for nyaml2nxdl converter: # https://manual.nexusformat.org/nxdl_desc.html?highlight=optional @@ -44,7 +48,8 @@ def generate_nxdl_or_retrieve_nxdl(yaml_file, out_xml_file, verbose): """ Generate yaml, nxdl and hash. - if the extracted hash is exactly the same as producd from generated yaml then + + If the extracted hash is exactly the same as producd from generated yaml then retrieve the nxdl part from provided yaml. Else, generate nxdl from separated yaml with the help of nyaml2nxdl function """ @@ -55,117 +60,115 @@ def generate_nxdl_or_retrieve_nxdl(yaml_file, out_xml_file, verbose): if hash_found: gen_hash = get_sha256_hash(sep_yaml) if hash_found == gen_hash: - os.remove(sep_yaml) + Path(sep_yaml).unlink() + # os.remove(sep_yaml) return nyaml2nxdl(sep_yaml, out_xml_file, verbose) - os.remove(sep_yaml) + Path(sep_yaml).unlink() + # os.remove(sep_yaml) -# pylint: disable=too-many-locals -def append_yml(input_file, append, verbose): - """Append to an existing NeXus base class new elements provided in YML input file \ -and print both an XML and YML file of the extended base class. +# # pylint: disable=too-many-locals +# def append_yml(input_file, append, verbose): +# """ +# Append to an existing NeXus base class new elements provided in YML input file. +# """ +# nexus_def_path = os.path.join( +# os.path.abspath(os.path.dirname(__file__)), "../../definitions" +# ) +# assert [ +# s +# for s in os.listdir(os.path.join(nexus_def_path, "base_classes")) +# if append.strip() == s.replace(_nxdl, "") +# ], "Your base class extension does not match any existing NeXus base classes" +# tree = ET.parse( +# os.path.join(nexus_def_path + "/base_classes", append + _nxdl) +# ) +# root = tree.getroot() +# # warning: tmp files are printed on disk and removed at the ends!! +# pretty_print_xml(root, "tmp.nxdl.xml") +# input_tmp_xml = "tmp.nxdl.xml" +# out_tmp_yml = "tmp_parsed.yaml" +# converter = Nxdl2yaml([], []) +# converter.print_yml(input_tmp_xml, out_tmp_yml, verbose) +# nyaml2nxdl(input_file=out_tmp_yml, out_file="tmp_parsed.nxdl.xml", verbose=verbose) +# tree = ET.parse("tmp_parsed.nxdl.xml") +# tree2 = ET.parse(input_file) +# root_no_duplicates = ET.Element( +# "definition", +# { +# "xmlns": "http://definition.nexusformat.org/nxdl/3.1", +# "xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance", +# "xsi:schemaLocation": "http://www.w3.org/2001/XMLSchema-instance", +# }, +# ) +# for attribute_keys in root.attrib.keys(): +# if ( +# attribute_keys +# != "{http://www.w3.org/2001/XMLSchema-instance}schemaLocation" +# ): +# attribute_value = root.attrib[attribute_keys] +# root_no_duplicates.set(attribute_keys, attribute_value) +# for elems in root.iter(): +# if "doc" in elems.tag: +# root_doc = ET.SubElement(root_no_duplicates, "doc") +# root_doc.text = elems.text +# break +# group = "{http://definition.nexusformat.org/nxdl/3.1}group" +# root_no_duplicates = compare_niac_and_my( +# tree, tree2, verbose, group, root_no_duplicates +# ) +# field = "{http://definition.nexusformat.org/nxdl/3.1}field" +# root_no_duplicates = compare_niac_and_my( +# tree, tree2, verbose, field, root_no_duplicates +# ) +# attribute = "{http://definition.nexusformat.org/nxdl/3.1}attribute" +# root_no_duplicates = compare_niac_and_my( +# tree, tree2, verbose, attribute, root_no_duplicates +# ) +# pretty_print_xml( +# root_no_duplicates, +# f"{input_file.replace('.nxdl.xml', '')}" f"_appended.nxdl.xml", +# ) -""" - nexus_def_path = os.path.join( - os.path.abspath(os.path.dirname(__file__)), "../../definitions" - ) - assert [ - s - for s in os.listdir(os.path.join(nexus_def_path, "base_classes")) - if append.strip() == s.replace(".nxdl.xml", "") - ], "Your base class extension does not match any existing NeXus base classes" - tree = ET.parse( - os.path.join(nexus_def_path + "/base_classes", append + ".nxdl.xml") - ) - root = tree.getroot() - # warning: tmp files are printed on disk and removed at the ends!! - pretty_print_xml(root, "tmp.nxdl.xml") - input_tmp_xml = "tmp.nxdl.xml" - out_tmp_yml = "tmp_parsed.yaml" - converter = Nxdl2yaml([], []) - converter.print_yml(input_tmp_xml, out_tmp_yml, verbose) - nyaml2nxdl(input_file=out_tmp_yml, out_file="tmp_parsed.nxdl.xml", verbose=verbose) - tree = ET.parse("tmp_parsed.nxdl.xml") - tree2 = ET.parse(input_file) - root_no_duplicates = ET.Element( - "definition", - { - "xmlns": "http://definition.nexusformat.org/nxdl/3.1", - "xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance", - "xsi:schemaLocation": "http://www.w3.org/2001/XMLSchema-instance", - }, - ) - for attribute_keys in root.attrib.keys(): - if ( - attribute_keys - != "{http://www.w3.org/2001/XMLSchema-instance}schemaLocation" - ): - attribute_value = root.attrib[attribute_keys] - root_no_duplicates.set(attribute_keys, attribute_value) - for elems in root.iter(): - if "doc" in elems.tag: - root_doc = ET.SubElement(root_no_duplicates, "doc") - root_doc.text = elems.text - break - group = "{http://definition.nexusformat.org/nxdl/3.1}group" - root_no_duplicates = compare_niac_and_my( - tree, tree2, verbose, group, root_no_duplicates - ) - field = "{http://definition.nexusformat.org/nxdl/3.1}field" - root_no_duplicates = compare_niac_and_my( - tree, tree2, verbose, field, root_no_duplicates - ) - attribute = "{http://definition.nexusformat.org/nxdl/3.1}attribute" - root_no_duplicates = compare_niac_and_my( - tree, tree2, verbose, attribute, root_no_duplicates - ) - pretty_print_xml( - root_no_duplicates, - f"{input_file.replace('.nxdl.xml', '')}" f"_appended.nxdl.xml", - ) - - input_file_xml = input_file.replace(".nxdl.xml", "_appended.nxdl.xml") - out_file_yml = input_file.replace(".nxdl.xml", "_appended_parsed.yaml") - converter = Nxdl2yaml([], []) - converter.print_yml(input_file_xml, out_file_yml, verbose) - nyaml2nxdl( - input_file=out_file_yml, - out_file=out_file_yml.replace(".yaml", ".nxdl.xml"), - verbose=verbose, - ) - os.rename( - f"{input_file.replace('.nxdl.xml', '_appended_parsed.yaml')}", - f"{input_file.replace('.nxdl.xml', '_appended.yaml')}", - ) - os.rename( - f"{input_file.replace('.nxdl.xml', '_appended_parsed.nxdl.xml')}", - f"{input_file.replace('.nxdl.xml', '_appended.nxdl.xml')}", - ) - os.remove("tmp.nxdl.xml") - os.remove("tmp_parsed.yaml") - os.remove("tmp_parsed.nxdl.xml") - - -def split_name_and_extension(file_name): +# input_file_xml = input_file.replace(_nxdl, "_appended.nxdl.xml") +# out_file_yml = input_file.replace(_nxdl, "_appended_parsed.yaml") +# converter = Nxdl2yaml([], []) +# converter.print_yml(input_file_xml, out_file_yml, verbose) +# nyaml2nxdl( +# input_file=out_file_yml, +# out_file=out_file_yml.replace(".yaml", _nxdl), +# verbose=verbose, +# ) +# os.rename( +# f"{input_file.replace('.nxdl.xml', '_appended_parsed.yaml')}", +# f"{input_file.replace('.nxdl.xml', '_appended.yaml')}", +# ) +# os.rename( +# f"{input_file.replace('.nxdl.xml', '_appended_parsed.nxdl.xml')}", +# f"{input_file.replace('.nxdl.xml', '_appended.nxdl.xml')}", +# ) +# # delete the files +# Path("tmp.nxdl.xml").unlink() +# # os.remove("tmp.nxdl.xml") +# Path("tmp_parsed.yaml").unlink() +# # os.remove("tmp_parsed.yaml") +# Path("tmp_parsed.nxdl.xml").unlink() +# # os.remove("tmp_parsed.nxdl.xml") + + +def split_name_and_extension(file_path): """ Split file name into extension and rest of the file name. - return file raw nam and extension - """ - path = file_name.rsplit("/", 1) - (pathn, filen) = ["", path[0]] if len(path) == 1 else [path[0] + "/", path[1]] - parts = filen.rsplit(".", 2) - raw = ext = "" - if len(parts) == 2: - raw = parts[0] - ext = parts[1] - elif len(parts) == 3: - raw = parts[0] - ext = ".".join(parts[1:]) - - return pathn + raw, ext + return file raw name and extension + """ + path = Path(file_path) + ext = ''.join(path.suffixes) + full_path_stem = file_path[0 : file_path.index(ext)] + return full_path_stem, ext[1:] + @click.command() @click.option( @@ -175,17 +178,17 @@ def split_name_and_extension(file_name): help="The path to the XML or YAML input data file to read and create \ a YAML or XML file from, respectively.", ) -@click.option( - "--append", - help="Parse xml file and append to base class, given that the xml file has same name \ -of an existing base class", -) +# @click.option( +# "--append", +# help="Parse xml file and append to base class, given that the xml file has same name \ +# of an existing base class", +# ) @click.option( "--check-consistency", is_flag=True, default=False, help=( - "Check wether yaml or nxdl has followed general rules of scema or not" + "Check if yaml or nxdl has followed general rules of whether schema or not" "check whether your comment in the right place or not. The option render an " "output file of the same extension(*_consistency.yaml or *_consistency.nxdl.xml)" ), @@ -197,9 +200,9 @@ def split_name_and_extension(file_name): help="Print in standard output keywords and value types to help \ possible issues in yaml files", ) -def launch_tool(input_file, verbose, append, check_consistency): +def launch_tool(input_file, verbose, check_consistency): """ - Main function that distiguishes the input file format and launches the tools. + Main function that distinguishes the input file format and launches the tools. """ if os.path.isfile(input_file): raw_name, ext = split_name_and_extension(input_file) @@ -207,44 +210,46 @@ def launch_tool(input_file, verbose, append, check_consistency): raise ValueError("Need a valid input file.") if ext == "yaml": - xml_out_file = raw_name + ".nxdl.xml" + xml_out_file = raw_name + _nxdl generate_nxdl_or_retrieve_nxdl(input_file, xml_out_file, verbose) - if append: - append_yml(raw_name + ".nxdl.xml", append, verbose) + # if append: + # append_yml(raw_name + _nxdl, append, verbose) # For consistency running if check_consistency: yaml_out_file = raw_name + "_consistency." + ext converter = Nxdl2yaml([], []) converter.print_yml(xml_out_file, yaml_out_file, verbose) - os.remove(xml_out_file) + Path(xml_out_file).unlink() + # os.remove(xml_out_file) elif ext == "nxdl.xml": - if not append: - yaml_out_file = raw_name + "_parsed" + ".yaml" - converter = Nxdl2yaml([], []) - converter.print_yml(input_file, yaml_out_file, verbose) - # Append nxdl.xml file with yaml output file - yaml_hash = get_sha256_hash(yaml_out_file) - # Lines as divider between yaml and nxdl - top_lines = [ - ( - "\n# ++++++++++++++++++++++++++++++++++ SHA HASH" - " ++++++++++++++++++++++++++++++++++\n" - ), - f"# {yaml_hash}\n", - ] - - extend_yamlfile_with_comment( - yaml_file=yaml_out_file, - file_to_be_appended=input_file, - top_lines_list=top_lines, - ) - else: - append_yml(input_file, append, verbose) + # if not append: + yaml_out_file = raw_name + "_parsed" + ".yaml" + converter = Nxdl2yaml([], []) + converter.print_yml(input_file, yaml_out_file, verbose) + # Append nxdl.xml file with yaml output file + yaml_hash = get_sha256_hash(yaml_out_file) + # Lines as divider between yaml and nxdl + top_lines = [ + ( + "\n# ++++++++++++++++++++++++++++++++++ SHA HASH" + " ++++++++++++++++++++++++++++++++++\n" + ), + f"# {yaml_hash}\n", + ] + + extend_yamlfile_with_comment( + yaml_file=yaml_out_file, + file_to_be_appended=input_file, + top_lines_list=top_lines, + ) + # else: + # append_yml(input_file, append, verbose) # Taking care of consistency running if check_consistency: xml_out_file = raw_name + "_consistency." + ext generate_nxdl_or_retrieve_nxdl(yaml_out_file, xml_out_file, verbose) - os.remove(yaml_out_file) + Path.unlink(yaml_out_file) + # os.remove(yaml_out_file) else: raise ValueError("Provide correct file with extension '.yaml or '.nxdl.xml") From 6df73b86dc0ba7c20577b193f4818361641ce31f Mon Sep 17 00:00:00 2001 From: Rubel Date: Thu, 21 Sep 2023 13:43:09 +0200 Subject: [PATCH 59/93] Change request in nyaml2nxdl_backward_tools.py --- dev_tools/nyaml2nxdl/comment_collector.py | 11 ++- .../nyaml2nxdl/nyaml2nxdl_backward_tools.py | 93 +++++++++++-------- 2 files changed, 58 insertions(+), 46 deletions(-) diff --git a/dev_tools/nyaml2nxdl/comment_collector.py b/dev_tools/nyaml2nxdl/comment_collector.py index b4dcfac6f3..3a339e7be8 100644 --- a/dev_tools/nyaml2nxdl/comment_collector.py +++ b/dev_tools/nyaml2nxdl/comment_collector.py @@ -84,7 +84,7 @@ def __init__(self, input_file: str = None, loaded_obj: Union[object, Dict] = Non def extract_all_comment_blocks(self): """ - Collect all comments. Note that here comment means (comment text + element or line info + Collect all comments. Note here that comment means (comment text + element or line info intended for comment. """ id_ = 0 @@ -403,7 +403,7 @@ def process_each_line(self, text, line_num): def has_post_comment(self): """ - Ensure is this a post coment or not. + Ensure if this is a post coment or not. Post comment means the comment that come at the very end without having any nxdl element(class, group, filed and attribute.) """ @@ -413,12 +413,13 @@ def has_post_comment(self): return False def append_comment(self, text: str) -> None: - """ + """Append comment texts to associated comment. + Collects all the line of the same comment and append them with that single comment. """ # check for escape char - text = self.replace_scape_char(text) + text = self.replace_escape_char(text) # Empty line after last line of comment if not text and self._comnt_start_found: self._comnt_end_found = True @@ -463,7 +464,7 @@ def get_comment_text_list(self): def get_line_number(self, line_key): """ - Return line number for which line the comment is created + Return line no for which line the comment is created. """ return self._elemt[line_key] diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py b/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py index dcf56b998d..8ed88f2d5d 100755 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py @@ -34,11 +34,16 @@ DEPTH_SIZE = " " CMNT_TAG = "!--" +CMNT_TAG_END = "--" +CMNT_START = "" def separate_pi_comments(input_file): """ - Separate PI comments from ProcessesInstruction (pi) + Separate PI comments from ProcessesInstruction (PI) + Separeate the comments that comes immediately after XML process instruction part, + i.e. copyright comment part. """ comments_list = [] comment = [] @@ -48,21 +53,21 @@ def separate_pi_comments(input_file): lines = file.readlines() has_pi = True for line in lines: - c_start = "" + # c_start = "" def_tag = " 0 and has_pi: - comment.append(line.replace(cmnt_end, "")) + elif CMNT_END in line and len(comment) > 0 and has_pi: + comment.append(line.replace(CMNT_END, "")) comments_list.append("".join(comment)) - comment = [] + comment.clear() elif def_tag in line or not has_pi: has_pi = False xml_lines.append(line) @@ -79,9 +84,9 @@ def comment(self, text): """ defining comment builder in TreeBuilder """ - self.start("!--", {}) + self.start(CMNT_TAG, {}) self.data(text) - self.end("--") + self.end(CMNT_TAG_END) def parse(filepath): @@ -189,7 +194,7 @@ def handle_symbols(self, depth, node): ) if tag == "doc": symbol_cmnt_list.append(last_comment) - # The bellow line is for handling lenth of 'symbol_comments' and + # The line bellow is for handling lenth of 'symbol_comments' and # 'symbol_doc_comments'. Otherwise print_root_level_info() gets inconsistency # over for the loop while writting comment on file sbl_doc_cmnt_list.append("") @@ -235,7 +240,7 @@ def store_root_level_comments(self, holder, comment): def handle_definition(self, node): """ Handle definition group and its attributes - NOTE: Here we tried to store the order of the xml element attributes. So that we get + NOTE: Here we try to store the order of the xml element attributes. So that we get exactly the same file in nxdl from yaml. """ # pylint: disable=consider-using-f-string @@ -247,12 +252,12 @@ def handle_definition(self, node): # for tracking the order of name and type keyword_order = -1 for item in attribs: - if "name" in item: + if "name" == item: keyword = keyword + attribs[item] if keyword_order == -1: self.root_level_definition.append(tmp_word) keyword_order = self.root_level_definition.index(tmp_word) - elif "extends" in item: + elif "extends" == item: keyword = f"{keyword}({attribs[item]})" if keyword_order == -1: self.root_level_definition.append(tmp_word) @@ -266,16 +271,16 @@ def handle_root_level_doc(self, node): """ Handle the documentation field found at root level. """ - # tag = remove_namespace_from_tag(node.tag) text = node.text text = self.handle_not_root_level_doc(depth=0, text=text) self.root_level_doc = text # pylint: disable=too-many-branches def handle_not_root_level_doc(self, depth, text, tag="doc", file_out=None): - """ + """Handle doc field of group, field but not root. + Handle docs field along the yaml file. In this function we also tried to keep - the track of intended indentation. E.g. the bollow doc block. + the track of indentation. E.g. the bollow doc block. * Topic name Description of topic """ @@ -286,24 +291,27 @@ def handle_not_root_level_doc(self, depth, text, tag="doc", file_out=None): else: text = handle_mapping_char(text, -1, True) if "\n" in text: - # To remove '\n' character as it will be added before text. + # To remove '\n' with non-space character as it will be added before text. text = cleaning_empty_lines(text.split("\n")) text_tmp = [] yaml_indent_n = len((depth + 1) * DEPTH_SIZE) - # Find indentaion in the first text line with alphabet - tmp_i = 0 - while tmp_i != -1: - first_line_indent_n = 0 - # Taking care of empty text whitout any character - if len(text) == 1 and text[0] == "": - break - for ch_ in text[tmp_i]: - if ch_ == " ": - first_line_indent_n = first_line_indent_n + 1 - elif ch_ != "": - tmp_i = -2 - break - tmp_i = tmp_i + 1 +# TODO: delete this if converter works + # tmp_i = 0 + # while tmp_i != -1: + # first_line_indent_n = 0 + # # Taking care of empty text whitout any character and non-space character + # if len(text) == 1 and text[0] == "": + # break + # for ch_ in text[tmp_i]: + # if ch_ == " ": + # first_line_indent_n = first_line_indent_n + 1 + # elif ch_ != "": + # tmp_i = -2 + # break + # tmp_i = tmp_i + 1 +# TODO: delete up if converter works + # Find indentaion in the first line text with alphabet + first_line_indent_n = len(text[0]) - len(text[0].lstrip()) # Taking care of doc like bellow: # Text liness # text continues @@ -317,14 +325,17 @@ def handle_not_root_level_doc(self, depth, text, tag="doc", file_out=None): indent_diff = yaml_indent_n - first_line_indent_n # CHeck for first line empty if not keep first line empty - for _, line in enumerate(text): + for line in text: line_indent_n = 0 - # Collect first empty space without alphabate - for ch_ in line: - if ch_ == " ": - line_indent_n = line_indent_n + 1 - else: - break + # count first empty spaces without alphabate + line_indent_n = len(line) - len(line.lstrip()) +# TODO remove the code below + # for ch_ in line: + # if ch_ == " ": + # line_indent_n = line_indent_n + 1 + # else: + # break +# TODO remove the code above line_indent_n = line_indent_n + indent_diff if line_indent_n < yaml_indent_n: # if line still under yaml identation From c51291f0ce0bc5717e00dd9ef5361f1e56be4cf5 Mon Sep 17 00:00:00 2001 From: Rubel Date: Fri, 22 Sep 2023 11:50:28 +0200 Subject: [PATCH 60/93] nyaml2nxdl backward tools: fixing requested changes in that file. --- .../nyaml2nxdl/nyaml2nxdl_backward_tools.py | 374 ++++++++---------- 1 file changed, 158 insertions(+), 216 deletions(-) diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py b/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py index 8ed88f2d5d..c65358fca8 100755 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py @@ -37,7 +37,7 @@ CMNT_TAG_END = "--" CMNT_START = "" - +DEFINITION_CATEGORIES = ("category: application", "category: base") def separate_pi_comments(input_file): """ @@ -147,7 +147,6 @@ def __init__( self.root_level_symbols = root_level_symbols self.root_level_definition = root_level_definition self.symbol_list = symbol_list - self.is_last_element_comment = False self.include_comment = True self.pi_comments = None # NOTE: Here is how root_level_comments organised for storing comments @@ -159,6 +158,49 @@ def __init__( # The 'symbol_comments' contains comments for 'symbols doc' and all 'symbol' # 'symbol_comments': [comments]} self.root_level_comment: Dict[str, str] = {} + self.grp_fld_allowed_attr = ( + "optional", + "recommended", + "name", + "type", + "axes", + "axis", + "data_offset", + "interpretation", + "long_name", + "maxOccurs", + "minOccurs", + "nameType", + "optional", + "primary", + "signal", + "stride", + "units", + "required", + "deprecated", + "exists", + ) + self.attr_allowed_attr = ( + "name", + "type", + "units", + "nameType", + "recommended", + "optional", + "minOccurs", + "maxOccurs", + "deprecated", + ) + self.link_allowed_attr = ("name", + "target", + "napimount") + self.optionality_keys = ("minOccurs", + "maxOccurs", + "optional", + "recommended", + "required") + # "Take care of general attributes. Note other choices may be allowed in the future" + self.choice_allowed_attr = () def print_yml(self, input_file, output_yml, verbose): """ @@ -176,7 +218,6 @@ def print_yml(self, input_file, output_yml, verbose): def handle_symbols(self, depth, node): """Handle symbols field and its childs symbol""" - # pylint: disable=consider-using-f-string self.root_level_symbols = ( f"{remove_namespace_from_tag(node.tag)}: " f"{node.text.strip() if node.text else ''}" @@ -243,8 +284,6 @@ def handle_definition(self, node): NOTE: Here we try to store the order of the xml element attributes. So that we get exactly the same file in nxdl from yaml. """ - # pylint: disable=consider-using-f-string - # self.root_level_definition[0] = '' keyword = "" # tmp_word for reseving the location tmp_word = "#xx#" @@ -280,7 +319,7 @@ def handle_not_root_level_doc(self, depth, text, tag="doc", file_out=None): """Handle doc field of group, field but not root. Handle docs field along the yaml file. In this function we also tried to keep - the track of indentation. E.g. the bollow doc block. + the track of indentation. E.g. the bellow doc block. * Topic name Description of topic """ @@ -295,23 +334,16 @@ def handle_not_root_level_doc(self, depth, text, tag="doc", file_out=None): text = cleaning_empty_lines(text.split("\n")) text_tmp = [] yaml_indent_n = len((depth + 1) * DEPTH_SIZE) -# TODO: delete this if converter works - # tmp_i = 0 - # while tmp_i != -1: - # first_line_indent_n = 0 - # # Taking care of empty text whitout any character and non-space character - # if len(text) == 1 and text[0] == "": - # break - # for ch_ in text[tmp_i]: - # if ch_ == " ": - # first_line_indent_n = first_line_indent_n + 1 - # elif ch_ != "": - # tmp_i = -2 - # break - # tmp_i = tmp_i + 1 -# TODO: delete up if converter works + + # Find indentaion in the first line text with alphabet - first_line_indent_n = len(text[0]) - len(text[0].lstrip()) + first_line_indent_n = 0 + for line in text: + # Consider only the lines that has at least one non-space character + # and skip starting lines of a text block are empty + if len(line.lstrip()) != 0: + first_line_indent_n = len(line) - len(line.lstrip()) + break # Taking care of doc like bellow: # Text liness # text continues @@ -329,13 +361,7 @@ def handle_not_root_level_doc(self, depth, text, tag="doc", file_out=None): line_indent_n = 0 # count first empty spaces without alphabate line_indent_n = len(line) - len(line.lstrip()) -# TODO remove the code below - # for ch_ in line: - # if ch_ == " ": - # line_indent_n = line_indent_n + 1 - # else: - # break -# TODO remove the code above + line_indent_n = line_indent_n + indent_diff if line_indent_n < yaml_indent_n: # if line still under yaml identation @@ -378,10 +404,8 @@ def print_root_level_doc(self, file_out): """ indent = 0 * DEPTH_SIZE - if ( - "root_doc" in self.root_level_comment - and self.root_level_comment["root_doc"] != "" - ): + text = self.root_level_comment.get("root_doc") + if text: text = self.root_level_comment["root_doc"] self.write_out(indent, text, file_out) @@ -397,10 +421,9 @@ def comvert_to_ymal_comment(self, indent, text): mod_lines = [] for line in lines: line = line.strip() - if line and line[0] != "#": - line = indent + "# " + line - mod_lines.append(line) - elif line: + if line: + if line[0] != "#": + line = "# " + line line = indent + line mod_lines.append(line) # The starting '\n' to keep multiple comments separate @@ -411,13 +434,12 @@ def print_root_level_info(self, depth, file_out): Print at the root level of YML file \ the information stored as definition attributes in the XML file """ - # pylint: disable=consider-using-f-string if depth < 0: raise ValueError("Somthing wrong with indentaion in root level.") has_categoty = False for def_line in self.root_level_definition: - if def_line in ("category: application", "category: base"): + if def_line in DEFINITION_CATEGORIES: self.write_out(indent=0 * DEPTH_SIZE, text=def_line, file_out=file_out) # file_out.write(f"{def_line}\n") has_categoty = True @@ -465,13 +487,13 @@ def print_root_level_info(self, depth, file_out): self.write_out(indent=(0 * DEPTH_SIZE), text=symbol, file_out=file_out) if len(self.pi_comments) > 1: indent = DEPTH_SIZE * depth - # The first comment is top level copy-right doc string + # The first comment is top level copyright doc string for comment in self.pi_comments[1:]: self.write_out( indent, self.comvert_to_ymal_comment(indent, comment), file_out ) if self.root_level_definition: - # Soring NXname for writting end of the definition attributes + # Soring NXname for writing end of the definition attributes nx_name = "" for defs in self.root_level_definition: if "NX" in defs and defs[-1] == ":": @@ -499,42 +521,19 @@ def handle_exists(self, exists_dict, key, val): val = str(val) if "minOccurs" == key: exists_dict["minOccurs"] = ["min", val] - if "maxOccurs" == key: + elif "maxOccurs" == key: exists_dict["maxOccurs"] = ["max", val] - if "optional" == key: + elif "optional" == key: exists_dict["optional"] = ["optional", val] - if "recommended" == key: + elif "recommended" == key: exists_dict["recommended"] = ["recommended", val] - if "required" == key: + elif "required" == key: exists_dict["required"] = ["required", val] - # pylint: disable=too-many-branches, consider-using-f-string + # pylint: disable=too-many-branches def handle_group_or_field(self, depth, node, file_out): """Handle all the possible attributes that come along a field or group""" - allowed_attr = [ - "optional", - "recommended", - "name", - "type", - "axes", - "axis", - "data_offset", - "interpretation", - "long_name", - "maxOccurs", - "minOccurs", - "nameType", - "optional", - "primary", - "signal", - "stride", - "units", - "required", - "deprecated", - "exists", - ] - name_type = "" node_attr = node.attrib rm_key_list = [] @@ -543,7 +542,7 @@ def handle_group_or_field(self, depth, node, file_out): if key == "name": name_type = name_type + val rm_key_list.append(key) - if key == "type": + elif key == "type": name_type = name_type + "(%s)" % val rm_key_list.append(key) if not name_type: @@ -551,11 +550,8 @@ def handle_group_or_field(self, depth, node, file_out): f"No 'name' or 'type' hase been found. But, 'group' or 'field' " f"must have at list a nme.We got attributes: {node_attr}" ) - file_out.write( - "{indent}{name_type}:\n".format( - indent=depth * DEPTH_SIZE, name_type=name_type - ) - ) + indent = depth * DEPTH_SIZE + file_out.write(f"{indent}{name_type}:\n") for key in rm_key_list: del node_attr[key] @@ -564,8 +560,13 @@ def handle_group_or_field(self, depth, node, file_out): tmp_dict = {} exists_dict = {} for key, val in node_attr.items(): + if key not in self.grp_fld_allowed_attr: + raise ValueError( + f"An attribute ({key}) in 'field' or 'group' has been found " + f"that is not allowed. The allowed attr is {self.grp_fld_allowed_attr}." + ) # As both 'minOccurs', 'maxOccurs' and optionality move to the 'exists' - if key in ["minOccurs", "maxOccurs", "optional", "recommended", "required"]: + if key in self.optionality_keys: if "exists" not in tmp_dict: tmp_dict["exists"] = [] self.handle_exists(exists_dict, key, val) @@ -573,11 +574,6 @@ def handle_group_or_field(self, depth, node, file_out): tmp_dict["unit"] = str(val) else: tmp_dict[key] = str(val) - if key not in allowed_attr: - raise ValueError( - f"An attribute ({key}) in 'field' or 'group' has been found " - f"that is not allowed. The allowed attr is {allowed_attr}." - ) if exists_dict: for key, val in exists_dict.items(): @@ -603,17 +599,13 @@ def handle_dimension(self, depth, node, file_out): recursion_in_xml_tree(...) functions. But Here it is a bit different. The doc dimension and attributes of dim has been handled inside this function here. """ - # pylint: disable=consider-using-f-string possible_dim_attrs = ["ref", "required", "incr", "refindex"] possible_dimemsion_attrs = ["rank"] # taking care of Dimension tag - file_out.write( - "{indent}{tag}:\n".format( - indent=depth * DEPTH_SIZE, tag=node.tag.split("}", 1)[1] - ) - ) - # Taking care of dimension attributes + indent = depth * DEPTH_SIZE + tag = remove_namespace_from_tag(node.tag) + file_out.write(f"{indent}{tag}:\n") for attr, value in node.attrib.items(): if attr in possible_dimemsion_attrs and not isinstance(value, dict): indent = (depth + 1) * DEPTH_SIZE @@ -640,12 +632,11 @@ def handle_dimension(self, depth, node, file_out): tag = remove_namespace_from_tag(child.tag) child_attrs = child.attrib # taking care of index and value attributes - if tag == ("dim"): + if tag == "dim": # taking care of index and value in format [[index, value]] - dim_index_value = dim_index_value + "[{index}, {value}], ".format( - index=child_attrs["index"] if "index" in child_attrs else "", - value=child_attrs["value"] if "value" in child_attrs else "", - ) + index = child_attrs.get("index", "") + value = child_attrs.get("value", "") + dim_index_value = f"{dim_index_value}[{index}, {value}], " if "index" in child_attrs: del child_attrs["index"] if "value" in child_attrs: @@ -676,19 +667,17 @@ def handle_dimension(self, depth, node, file_out): # All 'dim' element comments on top of 'dim' yaml key if dim_cmnt_node: for ch_nd in dim_cmnt_node: - self.handel_comment(depth + 1, ch_nd, file_out) + self.handle_comment(depth + 1, ch_nd, file_out) # index and value attributes of dim elements - file_out.write( - "{indent}dim: [{value}]\n".format( - indent=(depth + 1) * DEPTH_SIZE, value=dim_index_value[:-2] or "" - ) - ) + indent = (depth + 1) * DEPTH_SIZE + value = dim_index_value[:-2] or "" + file_out.write(f"{indent}dim: [{value}]\n") + # Write the attributes, except index and value, and doc of dim as child of dim_parameter. - # But tthe doc or attributes for each dim come inside list according to the order of dim. + # But the doc or attributes for each dim come inside list according to the order of dim. if dim_other_parts: - file_out.write( - "{indent}dim_parameters:\n".format(indent=(depth + 1) * DEPTH_SIZE) - ) + indent = (depth + 1) * DEPTH_SIZE + file_out.write(f"{indent}dim_parameters:\n") # depth = depth + 2 dim_paramerter has child such as doc of dim indent = (depth + 2) * DEPTH_SIZE for key, value in dim_other_parts.items(): @@ -703,7 +692,7 @@ def handle_dimension(self, depth, node, file_out): f"{indent}{key}: " f"{handle_mapping_char(value, depth + 3, False)}\n" ) - + def handle_enumeration(self, depth, node, file_out): """ Handle the enumeration field parsed from the xml file. @@ -715,7 +704,6 @@ def handle_enumeration(self, depth, node, file_out): enumeration list. """ - # pylint: disable=consider-using-f-string check_doc = [] for child in list(node): @@ -723,21 +711,16 @@ def handle_enumeration(self, depth, node, file_out): check_doc.append(list(child)) # pylint: disable=too-many-nested-blocks if check_doc: - file_out.write( - "{indent}{tag}: \n".format( - indent=depth * DEPTH_SIZE, tag=node.tag.split("}", 1)[1] - ) - ) + indent = depth * DEPTH_SIZE + tag = remove_namespace_from_tag(node.tag) + file_out.write(f"{indent}{tag}: \n") for child in list(node): tag = remove_namespace_from_tag(child.tag) itm_depth = depth + 1 - if tag == ("item"): - file_out.write( - "{indent}{value}: \n".format( - indent=(itm_depth) * DEPTH_SIZE, value=child.attrib["value"] - ) - ) - + if tag == "item": + indent = itm_depth * DEPTH_SIZE + value = child.attrib["value"] + file_out.write(f"{indent}{value}: \n") if list(child): for item_doc in list(child): if remove_namespace_from_tag(item_doc.tag) == "doc": @@ -752,74 +735,54 @@ def handle_enumeration(self, depth, node, file_out): remove_namespace_from_tag(item_doc.tag) == CMNT_TAG and self.include_comment ): - self.handel_comment(itm_depth + 1, item_doc, file_out) + self.handle_comment(itm_depth + 1, item_doc, file_out) if tag == CMNT_TAG and self.include_comment: - self.handel_comment(itm_depth + 1, child, file_out) + self.handle_comment(itm_depth + 1, child, file_out) else: enum_list = "" remove_nodes = [] for item_child in list(node): tag = remove_namespace_from_tag(item_child.tag) - if tag == ("item"): - enum_list = enum_list + "{value}, ".format( - value=item_child.attrib["value"] - ) + if tag == "item": + value = item_child.attrib["value"] + enum_list = f"{enum_list}{value}, " if tag == CMNT_TAG and self.include_comment: - self.handel_comment(depth, item_child, file_out) + self.handle_comment(depth, item_child, file_out) remove_nodes.append(item_child) for ch_node in remove_nodes: node.remove(ch_node) - - file_out.write( - "{indent}{tag}: [{enum_list}]\n".format( - indent=depth * DEPTH_SIZE, - tag=remove_namespace_from_tag(node.tag), - enum_list=enum_list[:-2] or "", - ) - ) + + indent = depth * DEPTH_SIZE + tag = remove_namespace_from_tag(node.tag) + enum_list = enum_list[:-2] or "" + file_out.write(f"{indent}{tag}: [{enum_list}]\n") def handle_attributes(self, depth, node, file_out): """Handle the attributes parsed from the xml file""" - allowed_attr = [ - "name", - "type", - "units", - "nameType", - "recommended", - "optional", - "minOccurs", - "maxOccurs", - "deprecated", - ] - name = "" + nm_attr = "name" node_attr = node.attrib - if "name" in node_attr: - pass - else: - raise ValueError("Attribute must have an name key.") - rm_key_list = [] # Maintain order: name and type in form name(type) or (type)name that come first - for key, val in node_attr.items(): - if key == "name": - name = val - rm_key_list.append(key) - - for key in rm_key_list: - del node_attr[key] + name = node_attr.get(nm_attr, "") + if not name: + raise ValueError("Attribute must have an name key.") + del node_attr[nm_attr] - file_out.write( - "{indent}{escapesymbol}{name}:\n".format( - indent=depth * DEPTH_SIZE, escapesymbol=r"\@", name=name - ) - ) + indent = depth * DEPTH_SIZE + escapesymbol = r"\@" + file_out.write(f"{indent}{escapesymbol}{name}:\n") tmp_dict = {} exists_dict = {} for key, val in node_attr.items(): + if key not in self.attr_allowed_attr: + raise ValueError( + f"An attribute ({key}) has been found that is not allowed." + f"The allowed attr is {self.attr_allowed_attr}." + ) # As both 'minOccurs', 'maxOccurs' and optionality move to the 'exists' - if key in ["minOccurs", "maxOccurs", "optional", "recommended", "required"]: + if key in self.optionality_keys: if "exists" not in tmp_dict: tmp_dict["exists"] = [] self.handle_exists(exists_dict, key, val) @@ -827,11 +790,6 @@ def handle_attributes(self, depth, node, file_out): tmp_dict["unit"] = val else: tmp_dict[key] = val - if key not in allowed_attr: - raise ValueError( - f"An attribute ({key}) has been found that is not allowed." - f"The allowed attr is {allowed_attr}." - ) has_min_max = False has_opt_reco_requ = False @@ -853,94 +811,78 @@ def handle_attributes(self, depth, node, file_out): depth_ = depth + 1 for key, val in tmp_dict.items(): - # Increase depth size inside handle_map...() for writting text with one + # Increase depth size inside handle_map...() for writing text with one # more indentation. file_out.write( f"{depth_ * DEPTH_SIZE}{key}: " f"{handle_mapping_char(val, depth_ + 1, False)}\n" ) - def handel_link(self, depth, node, file_out): + def handle_link(self, depth, node, file_out): """ Handle link elements of nxdl """ - possible_link_attrs = ["name", "target", "napimount"] node_attr = node.attrib # Handle special cases if "name" in node_attr: - file_out.write( - "{indent}{name}(link):\n".format( - indent=depth * DEPTH_SIZE, name=node_attr["name"] or "" - ) - ) + indent = depth * DEPTH_SIZE + name = node_attr["name"] or "" + file_out.write(f"{indent}{name}(link):\n") del node_attr["name"] depth_ = depth + 1 # Handle general cases for attr_key, val in node_attr.items(): - if attr_key in possible_link_attrs: - file_out.write( - "{indent}{attr}: {value}\n".format( - indent=depth_ * DEPTH_SIZE, attr=attr_key, value=val - ) - ) + if attr_key in self.link_allowed_attr: + indent = depth_ * DEPTH_SIZE + file_out.write(f"{indent}{attr_key}: {val}\n") else: raise ValueError( - f"An anexpected attribute '{attr_key}' of link has found." - f"At this moment the alloed keys are {possible_link_attrs}" + f"An unexpected attribute '{attr_key}' of link has found." + f"At this moment the alloed keys are {self.link_allowed_attr}" ) - def handel_choice(self, depth, node, file_out): + def handle_choice(self, depth, node, file_out): """ Handle choice element which is a parent node of group. """ - possible_attr = [] - node_attr = node.attrib # Handle special casees if "name" in node_attr: - file_out.write( - "{indent}{attr}(choice): \n".format( - indent=depth * DEPTH_SIZE, attr=node_attr["name"] - ) - ) + indent = depth * DEPTH_SIZE + attr = node_attr["name"] + file_out.write(f"{indent}{attr}(choice): \n") + del node_attr["name"] depth_ = depth + 1 - # Taking care of general attrinutes. Though, still no attrinutes have found, - # but could be used for future + # Take care of attributes of choice element, attrinutes may come in future. for attr in node_attr.items(): - if attr in possible_attr: - file_out.write( - "{indent}{attr}: {value}\n".format( - indent=depth_ * DEPTH_SIZE, attr=attr, value=node_attr[attr] - ) - ) + if attr in self.choice_allowed_attr: + indent = depth_ * DEPTH_SIZE + value = node_attr[attr] + file_out.write(f"{indent}{attr}: {value}\n") else: raise ValueError( f"An unexpected attribute '{attr}' of 'choice' has been found." - f"At this moment attributes for choice {possible_attr}" + f"At this moment attributes for choice {self.choice_allowed_attr}" ) - def handel_comment(self, depth, node, file_out): + def handle_comment(self, depth, node, file_out): """ Collect comment element and pass to write_out function """ indent = depth * DEPTH_SIZE - if self.is_last_element_comment: - text = self.comvert_to_ymal_comment(indent, node.text) - self.write_out(indent, text, file_out) - else: - text = self.comvert_to_ymal_comment(indent, node.text) - self.write_out(indent, text, file_out) - self.is_last_element_comment = True + text = self.comvert_to_ymal_comment(indent, node.text) + self.write_out(indent, text, file_out) + def recursion_in_xml_tree(self, depth, xml_tree, output_yml, verbose): """ Descend lower level in xml tree. If we are in the symbols branch, the recursive - behaviour is not triggered as we already handled the symbols' childs. + behaviour is not triggered as we already handled the symbols' children. """ tree = xml_tree["tree"] @@ -952,14 +894,14 @@ def recursion_in_xml_tree(self, depth, xml_tree, output_yml, verbose): # pylint: disable=too-many-branches, too-many-statements def xmlparse(self, output_yml, xml_tree, depth, verbose): """ - Main of the nxdl2yaml converter. + Main method of the nxdl2yaml converter. It parses XML tree, then prints recursively each level of the tree """ tree = xml_tree["tree"] node = xml_tree["node"] if verbose: - sys.stdout.write(f"Node tag: {remove_namespace_from_tag(node.tag)}\n") - sys.stdout.write(f"Attributes: {node.attrib}\n") + print(f"Node tag: {remove_namespace_from_tag(node.tag)}\n") + print(f"Attributes: {node.attrib}\n") with open(output_yml, "a", encoding="utf-8") as file_out: tag = remove_namespace_from_tag(node.tag) if tag == "definition": @@ -1012,18 +954,18 @@ def xmlparse(self, output_yml, xml_tree, depth, verbose): if tag == ("dimensions"): self.handle_dimension(depth, node, file_out) if tag == ("link"): - self.handel_link(depth, node, file_out) + self.handle_link(depth, node, file_out) if tag == ("choice"): - self.handel_choice(depth, node, file_out) + self.handle_choice(depth, node, file_out) if tag == CMNT_TAG and self.include_comment: - self.handel_comment(depth, node, file_out) + self.handle_comment(depth, node, file_out) depth += 1 # Write nested nodes self.recursion_in_xml_tree(depth, xml_tree, output_yml, verbose) def compare_niac_and_my(tree, tree2, verbose, node, root_no_duplicates): - """This function creates two trees with Niac XML file and My XML file. + """This function creates two trees with NIAC XML file and My XML file. The main aim is to compare the two trees and create a new one that is the union of the two initial trees. """ From 46c95604a899e8dbf5f946e763262d7ced91c53a Mon Sep 17 00:00:00 2001 From: Rubel Date: Fri, 22 Sep 2023 11:55:01 +0200 Subject: [PATCH 61/93] Updating nyaml2nxdl.py with '--do-not-store-nxdl' --- dev_tools/nyaml2nxdl/nyaml2nxdl.py | 28 +++++++++++++------ .../nyaml2nxdl/nyaml2nxdl_forward_tools.py | 4 +-- dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py | 7 +++-- 3 files changed, 25 insertions(+), 14 deletions(-) diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl.py b/dev_tools/nyaml2nxdl/nyaml2nxdl.py index 5e139ea270..96cfbe38aa 100755 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl.py @@ -34,7 +34,7 @@ from .nyaml2nxdl_backward_tools import compare_niac_and_my from .nyaml2nxdl_forward_tools import nyaml2nxdl from .nyaml2nxdl_forward_tools import pretty_print_xml -from .nyaml2nxdl_helper import extend_yamlfile_with_comment +from .nyaml2nxdl_helper import extend_yamlfile_by_nxdl_as_comment from .nyaml2nxdl_helper import get_sha256_hash from .nyaml2nxdl_helper import separate_hash_yaml_and_nxdl @@ -193,6 +193,15 @@ def split_name_and_extension(file_path): "output file of the same extension(*_consistency.yaml or *_consistency.nxdl.xml)" ), ) +@click.option( + "--do-not-store-nxdl", + is_flag=True, + default=False, + help=( + "Whether the input nxdl file will be stored as a comment" + " at the end of output yaml file." + ), +) @click.option( "--verbose", is_flag=True, @@ -200,7 +209,8 @@ def split_name_and_extension(file_path): help="Print in standard output keywords and value types to help \ possible issues in yaml files", ) -def launch_tool(input_file, verbose, check_consistency): +# def launch_tool(input_file, verbose, check_consistency): +def launch_tool(input_file, verbose, do_not_store_nxdl, check_consistency): """ Main function that distinguishes the input file format and launches the tools. """ @@ -226,7 +236,7 @@ def launch_tool(input_file, verbose, check_consistency): yaml_out_file = raw_name + "_parsed" + ".yaml" converter = Nxdl2yaml([], []) converter.print_yml(input_file, yaml_out_file, verbose) - # Append nxdl.xml file with yaml output file + # Store nxdl.xml file in output yaml file under SHA HASH yaml_hash = get_sha256_hash(yaml_out_file) # Lines as divider between yaml and nxdl top_lines = [ @@ -236,12 +246,12 @@ def launch_tool(input_file, verbose, check_consistency): ), f"# {yaml_hash}\n", ] - - extend_yamlfile_with_comment( - yaml_file=yaml_out_file, - file_to_be_appended=input_file, - top_lines_list=top_lines, - ) + if not do_not_store_nxdl: + extend_yamlfile_by_nxdl_as_comment( + yaml_file=yaml_out_file, + file_to_be_appended=input_file, + top_lines_list=top_lines, + ) # else: # append_yml(input_file, append, verbose) # Taking care of consistency running diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py b/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py index 664f687484..20760672c9 100644 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py @@ -85,7 +85,7 @@ def check_for_dom_comment_in_yaml(): dom_comment = "" dom_comment_ind = 1 for ind, comnt in enumerate(COMMENT_BLOCKS[0:5]): - cmnt_list = comnt.get_comment_text() + cmnt_list = comnt.get_comment_text_list() if len(cmnt_list) == 1: text = cmnt_list[0] else: @@ -981,7 +981,7 @@ def xml_handle_comment( line_info = (line_annotation, int(line_loc_no)) if line_info in COMMENT_BLOCKS: # noqa: F821 cmnt = COMMENT_BLOCKS.get_coment_by_line_info(line_info) # noqa: F821 - cmnt_text = cmnt.get_comment_text() + cmnt_text = cmnt.get_comment_text_list() if is_def_cmnt: return cmnt_text diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py b/dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py index c55f5da7a8..dc609770d7 100644 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py @@ -37,8 +37,7 @@ # NOTE: If any one change one of the bellow dict please change it for both ESCAPE_CHAR_DICT_IN_YAML = {"\t": " ", "':'": ":"} - -ESCAPE_CHAR_DICT_IN_XML = {" ": "\t", "':'": ":"} +ESCAPE_CHAR_DICT_IN_XML = {" ": "\t", ":": "':'"} def remove_namespace_from_tag(tag): @@ -175,7 +174,9 @@ def get_sha256_hash(file_name): return sha_hash.hexdigest() -def extend_yamlfile_with_comment(yaml_file, file_to_be_appended, top_lines_list=None): +def extend_yamlfile_by_nxdl_as_comment( + yaml_file, file_to_be_appended, top_lines_list=None +): """Extend yaml file by the file_to_be_appended as comment.""" with open(yaml_file, mode="a+", encoding="utf-8") as f1_obj: From 031c42333716b026ffb386bbe0a1e6a3c830d0cc Mon Sep 17 00:00:00 2001 From: Rubel Date: Fri, 22 Sep 2023 12:17:00 +0200 Subject: [PATCH 62/93] Removed unused and not functioned functions ('append_yaml', 'compaire_niac_and_my'), and CLI options. --- dev_tools/nyaml2nxdl/nyaml2nxdl.py | 105 +----------------- .../nyaml2nxdl/nyaml2nxdl_backward_tools.py | 35 ------ 2 files changed, 3 insertions(+), 137 deletions(-) diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl.py b/dev_tools/nyaml2nxdl/nyaml2nxdl.py index 96cfbe38aa..52a60b79e3 100755 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl.py @@ -26,14 +26,11 @@ # import os from pathlib import Path -import xml.etree.ElementTree as ET import click from .nyaml2nxdl_backward_tools import Nxdl2yaml -from .nyaml2nxdl_backward_tools import compare_niac_and_my from .nyaml2nxdl_forward_tools import nyaml2nxdl -from .nyaml2nxdl_forward_tools import pretty_print_xml from .nyaml2nxdl_helper import extend_yamlfile_by_nxdl_as_comment from .nyaml2nxdl_helper import get_sha256_hash from .nyaml2nxdl_helper import separate_hash_yaml_and_nxdl @@ -69,95 +66,6 @@ def generate_nxdl_or_retrieve_nxdl(yaml_file, out_xml_file, verbose): # os.remove(sep_yaml) -# # pylint: disable=too-many-locals -# def append_yml(input_file, append, verbose): -# """ -# Append to an existing NeXus base class new elements provided in YML input file. -# """ -# nexus_def_path = os.path.join( -# os.path.abspath(os.path.dirname(__file__)), "../../definitions" -# ) -# assert [ -# s -# for s in os.listdir(os.path.join(nexus_def_path, "base_classes")) -# if append.strip() == s.replace(_nxdl, "") -# ], "Your base class extension does not match any existing NeXus base classes" -# tree = ET.parse( -# os.path.join(nexus_def_path + "/base_classes", append + _nxdl) -# ) -# root = tree.getroot() -# # warning: tmp files are printed on disk and removed at the ends!! -# pretty_print_xml(root, "tmp.nxdl.xml") -# input_tmp_xml = "tmp.nxdl.xml" -# out_tmp_yml = "tmp_parsed.yaml" -# converter = Nxdl2yaml([], []) -# converter.print_yml(input_tmp_xml, out_tmp_yml, verbose) -# nyaml2nxdl(input_file=out_tmp_yml, out_file="tmp_parsed.nxdl.xml", verbose=verbose) -# tree = ET.parse("tmp_parsed.nxdl.xml") -# tree2 = ET.parse(input_file) -# root_no_duplicates = ET.Element( -# "definition", -# { -# "xmlns": "http://definition.nexusformat.org/nxdl/3.1", -# "xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance", -# "xsi:schemaLocation": "http://www.w3.org/2001/XMLSchema-instance", -# }, -# ) -# for attribute_keys in root.attrib.keys(): -# if ( -# attribute_keys -# != "{http://www.w3.org/2001/XMLSchema-instance}schemaLocation" -# ): -# attribute_value = root.attrib[attribute_keys] -# root_no_duplicates.set(attribute_keys, attribute_value) -# for elems in root.iter(): -# if "doc" in elems.tag: -# root_doc = ET.SubElement(root_no_duplicates, "doc") -# root_doc.text = elems.text -# break -# group = "{http://definition.nexusformat.org/nxdl/3.1}group" -# root_no_duplicates = compare_niac_and_my( -# tree, tree2, verbose, group, root_no_duplicates -# ) -# field = "{http://definition.nexusformat.org/nxdl/3.1}field" -# root_no_duplicates = compare_niac_and_my( -# tree, tree2, verbose, field, root_no_duplicates -# ) -# attribute = "{http://definition.nexusformat.org/nxdl/3.1}attribute" -# root_no_duplicates = compare_niac_and_my( -# tree, tree2, verbose, attribute, root_no_duplicates -# ) -# pretty_print_xml( -# root_no_duplicates, -# f"{input_file.replace('.nxdl.xml', '')}" f"_appended.nxdl.xml", -# ) - -# input_file_xml = input_file.replace(_nxdl, "_appended.nxdl.xml") -# out_file_yml = input_file.replace(_nxdl, "_appended_parsed.yaml") -# converter = Nxdl2yaml([], []) -# converter.print_yml(input_file_xml, out_file_yml, verbose) -# nyaml2nxdl( -# input_file=out_file_yml, -# out_file=out_file_yml.replace(".yaml", _nxdl), -# verbose=verbose, -# ) -# os.rename( -# f"{input_file.replace('.nxdl.xml', '_appended_parsed.yaml')}", -# f"{input_file.replace('.nxdl.xml', '_appended.yaml')}", -# ) -# os.rename( -# f"{input_file.replace('.nxdl.xml', '_appended_parsed.nxdl.xml')}", -# f"{input_file.replace('.nxdl.xml', '_appended.nxdl.xml')}", -# ) -# # delete the files -# Path("tmp.nxdl.xml").unlink() -# # os.remove("tmp.nxdl.xml") -# Path("tmp_parsed.yaml").unlink() -# # os.remove("tmp_parsed.yaml") -# Path("tmp_parsed.nxdl.xml").unlink() -# # os.remove("tmp_parsed.nxdl.xml") - - def split_name_and_extension(file_path): """ Split file name into extension and rest of the file name. @@ -178,11 +86,7 @@ def split_name_and_extension(file_path): help="The path to the XML or YAML input data file to read and create \ a YAML or XML file from, respectively.", ) -# @click.option( -# "--append", -# help="Parse xml file and append to base class, given that the xml file has same name \ -# of an existing base class", -# ) + @click.option( "--check-consistency", is_flag=True, @@ -222,8 +126,7 @@ def launch_tool(input_file, verbose, do_not_store_nxdl, check_consistency): if ext == "yaml": xml_out_file = raw_name + _nxdl generate_nxdl_or_retrieve_nxdl(input_file, xml_out_file, verbose) - # if append: - # append_yml(raw_name + _nxdl, append, verbose) + # For consistency running if check_consistency: yaml_out_file = raw_name + "_consistency." + ext @@ -252,14 +155,12 @@ def launch_tool(input_file, verbose, do_not_store_nxdl, check_consistency): file_to_be_appended=input_file, top_lines_list=top_lines, ) - # else: - # append_yml(input_file, append, verbose) + # Taking care of consistency running if check_consistency: xml_out_file = raw_name + "_consistency." + ext generate_nxdl_or_retrieve_nxdl(yaml_out_file, xml_out_file, verbose) Path.unlink(yaml_out_file) - # os.remove(yaml_out_file) else: raise ValueError("Provide correct file with extension '.yaml or '.nxdl.xml") diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py b/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py index c65358fca8..e709ffe4a6 100755 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py @@ -962,38 +962,3 @@ def xmlparse(self, output_yml, xml_tree, depth, verbose): depth += 1 # Write nested nodes self.recursion_in_xml_tree(depth, xml_tree, output_yml, verbose) - - -def compare_niac_and_my(tree, tree2, verbose, node, root_no_duplicates): - """This function creates two trees with NIAC XML file and My XML file. - The main aim is to compare the two trees and create a new one that is the - union of the two initial trees. - """ - root = tree.getroot() - root2 = tree2.getroot() - attrs_list_niac = [] - for nodo in root.iter(node): - attrs_list_niac.append(nodo.attrib) - if verbose: - sys.stdout.write("Attributes found in Niac file: \n") - sys.stdout.write(str(attrs_list_niac) + "\n") - sys.stdout.write(" \n") - sys.stdout.write("Started merging of Niac and My file... \n") - for elem in root.iter(node): - if verbose: - sys.stdout.write("- Niac element inserted: \n") - sys.stdout.write(str(elem.attrib) + "\n") - index = get_node_parent_info(tree, elem)[1] - root_no_duplicates.insert(index, elem) - - for elem2 in root2.iter(node): - index = get_node_parent_info(tree2, elem2)[1] - if elem2.attrib not in attrs_list_niac: - if verbose: - sys.stdout.write("- My element inserted: \n") - sys.stdout.write(str(elem2.attrib) + "\n") - root_no_duplicates.insert(index, elem2) - - if verbose: - sys.stdout.write(" \n") - return root_no_duplicates From d702250cf820ba5bea66e42ab6e10b8b0c59f784 Mon Sep 17 00:00:00 2001 From: Rubel Date: Mon, 25 Sep 2023 09:49:04 +0200 Subject: [PATCH 63/93] Updating nyaml_forward_tools.py. --- .../nyaml2nxdl/nyaml2nxdl_forward_tools.py | 229 +++++++++--------- dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py | 4 +- 2 files changed, 123 insertions(+), 110 deletions(-) diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py b/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py index 20760672c9..0ccd0cd6e6 100644 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py @@ -24,8 +24,10 @@ import os import sys import textwrap +import datetime import xml.etree.ElementTree as ET from xml.dom import minidom +import warnings import yaml @@ -39,26 +41,26 @@ # pylint: disable=too-many-lines, global-statement, invalid-name DOM_COMMENT = ( - "\n" - "# NeXus - Neutron and X-ray Common Data Format\n" - "# \n" - "# Copyright (C) 2014-2022 NeXus International Advisory Committee (NIAC)\n" - "# \n" - "# This library is free software; you can redistribute it and/or\n" - "# modify it under the terms of the GNU Lesser General Public\n" - "# License as published by the Free Software Foundation; either\n" - "# version 3 of the License, or (at your option) any later version.\n" - "#\n" - "# This library is distributed in the hope that it will be useful,\n" - "# but WITHOUT ANY WARRANTY; without even the implied warranty of\n" - "# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU\n" - "# Lesser General Public License for more details.\n" - "#\n" - "# You should have received a copy of the GNU Lesser General Public\n" - "# License along with this library; if not, write to the Free Software\n" - "# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA\n" - "#\n" - "# For further information, see http://www.nexusformat.org\n" + f"\n" + f"# NeXus - Neutron and X-ray Common Data Format\n" + f"# \n" + f"# Copyright (C) 2014-{datetime.date.today().year} NeXus International Advisory Committee (NIAC)\n" + f"# \n" + f"# This library is free software; you can redistribute it and/or\n" + f"# modify it under the terms of the GNU Lesser General Public\n" + f"# License as published by the Free Software Foundation; either\n" + f"# version 3 of the License, or (at your option) any later version.\n" + f"#\n" + f"# This library is distributed in the hope that it will be useful,\n" + f"# but WITHOUT ANY WARRANTY; without even the implied warranty of\n" + f"# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU\n" + f"# Lesser General Public License for more details.\n" + f"#\n" + f"# You should have received a copy of the GNU Lesser General Public\n" + f"# License along with this library; if not, write to the Free Software\n" + f"# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA\n" + f"#\n" + f"# For further information, see http://www.nexusformat.org\n" ) NX_CLSS = pynxtools_nxlib.get_nx_classes() NX_NEW_DEFINED_CLASSES = ["NX_COMPLEX"] @@ -67,13 +69,15 @@ NX_UNIT_IDNT = "unit" DEPTH_SIZE = " " NX_UNIT_TYPES = pynxtools_nxlib.get_nx_units() +# Initialised in yml_reader() funtion COMMENT_BLOCKS: CommentCollector CATEGORY = "" # Definition would be either 'base' or 'application' def check_for_dom_comment_in_yaml(): """Check the yaml file has dom comment or dom comment needed to be hard coded.""" - dignature_keyword_list = [ + # some signature keywords to distingush dom comments from other comments. + signature_keyword_list = [ "NeXus", "GNU Lesser General Public", "Free Software Foundation", @@ -84,7 +88,7 @@ def check_for_dom_comment_in_yaml(): # Check for dom comments in first three comments dom_comment = "" dom_comment_ind = 1 - for ind, comnt in enumerate(COMMENT_BLOCKS[0:5]): + for ind, comnt in enumerate(COMMENT_BLOCKS[0:3]): cmnt_list = comnt.get_comment_text_list() if len(cmnt_list) == 1: text = cmnt_list[0] @@ -92,7 +96,7 @@ def check_for_dom_comment_in_yaml(): continue dom_comment = text dom_comment_ind = ind - for keyword in dignature_keyword_list: + for keyword in signature_keyword_list: if keyword not in text: dom_comment = "" break @@ -134,11 +138,11 @@ def yml_reader(inputfile): def check_for_default_attribute_and_value(xml_element): - """NeXus Groups, fields and attributes might have xml default attributes and valuesthat must + """NeXus Groups, fields and attributes might have xml default attributes and values that must come. For example: 'optional' which is 'true' by default for base class and false otherwise. """ - # base:Default attributes and value for all elements of base class except dimension element + # base: Default attributes and value for all elements of base class except dimension element base_attr_to_val = {"optional": "true"} # application: Default attributes and value for all elements of application class except @@ -150,7 +154,7 @@ def check_for_default_attribute_and_value(xml_element): application_dim_attr_to_val = {"required": "true"} # Eligible tag for default attr and value - elegible_tag = ["group", "field", "attribute"] + eligible_tag = ["group", "field", "attribute"] def set_default_attribute(xml_elem, default_attr_to_val): for deflt_attr, deflt_val in default_attr_to_val.items(): @@ -163,7 +167,7 @@ def set_default_attribute(xml_elem, default_attr_to_val): xml_elem.set(deflt_attr, deflt_val) for child in list(xml_element): - # skiping comment 'function' that mainly collect comment from yaml file. + # skipping comment 'function' that mainly collect comment from yaml file. if not isinstance(child.tag, str): continue tag = remove_namespace_from_tag(child.tag) @@ -172,9 +176,9 @@ def set_default_attribute(xml_elem, default_attr_to_val): set_default_attribute(child, base_dim_attr_to_val) if tag == "dim" and CATEGORY == "application": set_default_attribute(child, application_dim_attr_to_val) - if tag in elegible_tag and CATEGORY == "base": + if tag in eligible_tag and CATEGORY == "base": set_default_attribute(child, base_attr_to_val) - if tag in elegible_tag and CATEGORY == "application": + if tag in eligible_tag and CATEGORY == "application": set_default_attribute(child, application_attr_to_val) check_for_default_attribute_and_value(child) @@ -188,7 +192,7 @@ def yml_reader_nolinetag(inputfile): return parsed_yaml -def check_for_skiped_attributes(component, value, allowed_attr=None, verbose=False): +def check_for_skipped_attributes(component, value, allowed_attr=None, verbose=False): """ Check for any attributes have been skipped or not. NOTE: We should keep in mind about 'doc' @@ -196,7 +200,7 @@ def check_for_skiped_attributes(component, value, allowed_attr=None, verbose=Fal block_tag = ["enumeration"] if value: for attr, val in value.items(): - if attr in ["doc"]: + if attr == "doc": continue if "__line__" in attr or attr in block_tag: continue @@ -212,8 +216,8 @@ def check_for_skiped_attributes(component, value, allowed_attr=None, verbose=Fal ): raise ValueError( f"An attribute '{attr}' in part '{component}' has been found" - f". Please check arround line '{value[line_number]}. At this " - f"moment. The allowed attrbutes are {allowed_attr}" + f". Please check around line '{value[line_number]}. At this " + f"time, the allowed attrbutes are {allowed_attr}." ) @@ -221,10 +225,11 @@ def format_nxdl_doc(string): """NeXus format for doc string""" string = check_for_mapping_char_other(string) formatted_doc = "" + string_len = 80 if "\n" not in string: - if len(string) > 80: + if len(string) > string_len: wrapped = textwrap.TextWrapper( - width=80, break_long_words=False, replace_whitespace=False + width=string_len, break_long_words=False, replace_whitespace=False ) string = "\n".join(wrapped.wrap(string)) formatted_doc = "\n" + f"{string}" @@ -255,7 +260,7 @@ def check_for_mapping_char_other(text): for key, val in escape_reverter.items(): if key in text: text = text.replace(key, val) - return str(text).strip() + return text.strip() def xml_handle_doc(obj, value: str, line_number=None, line_loc=None): @@ -284,29 +289,30 @@ def xml_handle_exists(dct, obj, keyword, value): value is not None ), f"Line {dct[line_number]}: exists argument must not be None !" if isinstance(value, list): - if len(value) == 4 and value[0] == "min" and value[2] == "max": - obj.set("minOccurs", str(value[1])) - if str(value[3]) != "infty": - obj.set("maxOccurs", str(value[3])) - else: - obj.set("maxOccurs", "unbounded") + if len(value) == 4: + if value[0] == "min" and value[2] == "max": + obj.set("minOccurs", str(value[1])) + if str(value[3]) != "infty": + obj.set("maxOccurs", str(value[3])) + else: + obj.set("maxOccurs", "unbounded") + elif value[0] == "max" and value[2] == "min": + obj.set("minOccurs", str(value[3])) + if str(value[1]) != "infty": + obj.set("maxOccurs", str(value[3])) + else: + obj.set("maxOccurs", "unbounded") + elif value[0] != "min" or value[2] != "max": + raise ValueError( + f"Line {dct[line_number]}: exists keyword" + f"needs to go either with an optional [recommended] list with two " + f"entries either [min, ] or [max, ], or a list of four " + f"entries [min, , max, ] !" + ) elif len(value) == 2 and value[0] == "min": obj.set("minOccurs", str(value[1])) elif len(value) == 2 and value[0] == "max": obj.set("maxOccurs", str(value[1])) - elif len(value) == 4 and value[0] == "max" and value[2] == "min": - obj.set("minOccurs", str(value[3])) - if str(value[1]) != "infty": - obj.set("maxOccurs", str(value[3])) - else: - obj.set("maxOccurs", "unbounded") - elif len(value) == 4 and (value[0] != "min" or value[2] != "max"): - raise ValueError( - f"Line {dct[line_number]}: exists keyword" - f"needs to go either with an optional [recommended] list with two " - f"entries either [min, ] or [max, ], or a list of four " - f"entries [min, , max, ] !" - ) else: raise ValueError( f"Line {dct[line_number]}: exists keyword " @@ -315,7 +321,7 @@ def xml_handle_exists(dct, obj, keyword, value): f"entries [min, , max, ] !" ) else: - # This clause take optional in all concept except dimension where 'required' key is allowed + # This clause take optional in all cases except dimension where 'required' key is allowed # not the 'optional' key. if value == "optional": obj.set("optional", "true") @@ -353,8 +359,10 @@ def xml_handle_group(dct, obj, keyword, value, verbose=False): r_bracket = keyword.index(")") keyword_name, keyword_type = nx_name_type_resolving(keyword) - if not keyword_name and not keyword_type: - raise ValueError("A group must have both value and name. Check for group.") + if not keyword_type: + # if not keyword_name and not keyword_type: + raise ValueError(f"A group must have a type class from base, application" + f" or contributed class. Check around line: {line_number}.") grp = ET.SubElement(obj, "group") if l_bracket == 0 and r_bracket > 0: @@ -397,8 +405,8 @@ def xml_handle_group(dct, obj, keyword, value, verbose=False): for key in rm_key_list: del value[key] # Check for skipped attrinutes - check_for_skiped_attributes("group", value, list_of_attr, verbose) - if isinstance(value, dict) and value != {}: + check_for_skipped_attributes("group", value, list_of_attr, verbose) + if isinstance(value, dict) and len(value) != 0: recursive_build(grp, value, verbose) @@ -423,11 +431,11 @@ def xml_handle_dimensions(dct, obj, keyword, value: dict): line_number = f"__line__{keyword}" line_loc = dct[line_number] assert "dim" in value.keys(), ( - f"Line {line_loc}: No dim as child of dimension has " f"been found." + f"Line {line_loc}: No dim as child of dimension has been found." ) xml_handle_comment(obj, line_number, line_loc) dims = ET.SubElement(obj, "dimensions") - # Consider all the childs under dimension is dim element and + # Consider all the children under dimension is dim element and # its attributes rm_key_list = [] @@ -442,7 +450,7 @@ def xml_handle_dimensions(dct, obj, keyword, value: dict): if isinstance(rank, int) and rank < 0: raise ValueError( f"Dimension must have some info about rank which is not " - f"available. Please check arround Line: {dct[line_number]}" + f"available. Please check around Line: {dct[line_number]}" ) dims.set(key, str(val)) rm_key_list.append(key) @@ -477,8 +485,8 @@ def xml_handle_dim_from_dimension_dict( NOTE: The inputs 'keyword' and 'value' are as input for xml_handle_dimensions function. please also read note in xml_handle_dimensions. """ - - possible_dim_attrs = ["ref", "incr", "refindex", "required"] + deprecated_dim_attrs = ["ref", "incr", "refindex"] + possible_dim_attrs = [*deprecated_dim_attrs, "required"] # Some attributes might have equivalent name e.g. 'required' is correct one and # 'optional' could be another name. Then change attribute to the correct one. @@ -502,7 +510,7 @@ def xml_handle_dim_from_dimension_dict( # dim consists of list of [index, value] llist_ind_value = vvalue assert isinstance(llist_ind_value, list), ( - f"Line {value[line_number]}: dim" f"argument not a list !" + f"Line {value[line_number]}: dim argument not a list !" ) xml_handle_comment(dims_obj, line_number, line_loc) if isinstance(rank, int) and rank > 0: @@ -548,6 +556,10 @@ def xml_handle_dim_from_dimension_dict( elif isinstance(vvval, list) and i >= len(vvval): pass else: + if kkkey in deprecated_dim_attrs: + dep_text = (f"Attrbute {kkkey} is deprecated. " + f"Check attributes after line {cmnt_loc}") + warnings.warn(dep_text, DeprecationWarning) for i, dim in enumerate(dim_list): # all atribute of dims comes as list if isinstance(vvval, list) and i < len(vvval): @@ -572,7 +584,7 @@ def xml_handle_dim_from_dimension_dict( for key in rm_key_list: del value[key] - check_for_skiped_attributes("dim", value, possible_dim_attrs, verbose) + check_for_skipped_attributes("dim", value, possible_dim_attrs, verbose) def xml_handle_enumeration(dct, obj, keyword, value, verbose): @@ -642,7 +654,7 @@ def xml_handle_link(dct, obj, keyword, value, verbose): for key in rm_key_list: del value[key] # Check for skipped attrinutes - check_for_skiped_attributes("link", value, possible_attrs, verbose) + check_for_skipped_attributes("link", value, possible_attrs, verbose) if isinstance(value, dict) and value != {}: recursive_build(link_obj, value, verbose=None) @@ -655,8 +667,8 @@ def xml_handle_choice(dct, obj, keyword, value, verbose=False): line_number = f"__line__{keyword}" line_loc = dct[line_number] xml_handle_comment(obj, line_number, line_loc) - # Add attributes in possible if new attributs have been added nexus definition. - possible_attr = [] + # Add to this tuple if new attributs have been added to nexus definition. + possible_attr = () choice_obj = ET.SubElement(obj, "choice") # take care of special attributes name = keyword[:-8] @@ -683,7 +695,7 @@ def xml_handle_choice(dct, obj, keyword, value, verbose=False): for key in rm_key_list: del value[key] # Check for skipped attrinutes - check_for_skiped_attributes("choice", value, possible_attr, verbose) + check_for_skipped_attributes("choice", value, possible_attr, verbose) if isinstance(value, dict) and value != {}: recursive_build(choice_obj, value, verbose=None) @@ -694,7 +706,7 @@ def xml_handle_symbols(dct, obj, keyword, value: dict): line_number = f"__line__{keyword}" line_loc = dct[line_number] assert ( - len(list(value.keys())) >= 1 + len(list(value.keys())) > 0 ), f"Line {line_loc}: symbols table must not be empty !" xml_handle_comment(obj, line_number, line_loc) syms = ET.SubElement(obj, "symbols") @@ -716,7 +728,7 @@ def xml_handle_symbols(dct, obj, keyword, value: dict): vvalue, str ), f"Line {line_loc}: put a comment in doc string !" sym = ET.SubElement(syms, "symbol") - sym.set("name", str(kkeyword)) + sym.set("name", kkeyword) # sym_doc = ET.SubElement(sym, 'doc') xml_handle_doc(sym, vvalue) rm_key_list.append(kkeyword) @@ -733,7 +745,7 @@ def check_keyword_variable(verbose, dct, keyword, value): """ keyword_name, keyword_type = nx_name_type_resolving(keyword) if verbose: - sys.stdout.write( + print( f"{keyword_name}({keyword_type}): value type is {type(value)}\n" ) if keyword_name == "" and keyword_type == "": @@ -764,7 +776,7 @@ def xml_handle_attributes(dct, obj, keyword, value, verbose): line_number = f"__line__{keyword}" line_loc = dct[line_number] xml_handle_comment(obj, line_number, line_loc) - # list of possible attribute of xml attribute elementsa + # list of possible attribute of xml attribute elements attr_attr_list = [ "name", "type", @@ -823,7 +835,7 @@ def xml_handle_attributes(dct, obj, keyword, value, verbose): for key in rm_key_list: del value[key] # Check cor skiped attribute - check_for_skiped_attributes("Attribute", value, attr_attr_list, verbose) + check_for_skipped_attributes("Attribute", value, attr_attr_list, verbose) if value: recursive_build(elemt_obj, value, verbose) @@ -842,7 +854,7 @@ def validate_field_attribute_and_value(v_attr, vval, allowed_attribute, value): f" Please check arround line {value[line_number]}" ) - # The bellow elements might come as child element + # The below elements might come as child element skipped_child_name = ["doc", "dimension", "enumeration", "choice", "exists"] # check for invalid key or attributes if ( @@ -956,7 +968,7 @@ def xml_handle_fields(obj, keyword, value, line_annot, line_loc, verbose=False): for key in rm_key_list: del value[key] # Check for skipped attrinutes - check_for_skiped_attributes("field", value, allowed_attr, verbose) + check_for_skipped_attributes("field", value, allowed_attr, verbose) if isinstance(value, dict) and value != {}: recursive_build(elemt_obj, value, verbose) @@ -971,18 +983,20 @@ def xml_handle_comment( ): """ Add xml comment: check for comments that has the same 'line_annotation' - (e.g. __line__data) and the same line_loc_no (e.g. 30). After that, i - does of three tasks: + (e.g. __line__data) and the same line_loc_no (e.g. 30). After that, it + does following tasks: 1. Returns list of comments texts (multiple members if element has multiple comments) - 2. Rearrange comment element and xml_ele where comment comes first. - 3. Append comment element when no xml_ele will no be provided. + or [] if no intended comments are found + 2. Rearrange comment elements of xml_ele and xml_ele where comment comes first. + 3. Append comment element when no xml_ele found as general comments. """ - + line_info = (line_annotation, int(line_loc_no)) if line_info in COMMENT_BLOCKS: # noqa: F821 cmnt = COMMENT_BLOCKS.get_coment_by_line_info(line_info) # noqa: F821 cmnt_text = cmnt.get_comment_text_list() - + + # Check comment for definition element if is_def_cmnt: return cmnt_text if xml_ele is not None: @@ -993,20 +1007,21 @@ def xml_handle_comment( obj.append(xml_ele) elif not is_def_cmnt and xml_ele is None: for string in cmnt_text: - si_comnt = ET.Comment(string) - obj.append(si_comnt) - else: - raise ValueError("Provied correct parameter values.") - return "" + obj.append(ET.Comment(string)) + return [] def recursive_build(obj, dct, verbose): - """obj is the current node of the XML tree where we want to append to, - dct is a dictionary object which represents the content of a child to obj - dct may contain further dictionary nests, representing NXDL groups, - which trigger recursive processing - NXDL fields may contain attributes but trigger no recursion so attributes are leafs. - + """Walk through nested dictionary. + Parameters: + ----------- + obj : ET.Element + Obj is the current node of the XML tree where we want to append to. + dct : dict + dct is the nested python dictionary which represents the content of a child and + its successors. + + Note: NXDL fields may contain attributes but trigger no recursion so attributes are leafs. """ for keyword, value in iter(dct.items()): if "__line__" in keyword: @@ -1043,10 +1058,8 @@ def recursive_build(obj, dct, verbose): xml_handle_units(obj, value) elif keyword == "enumeration": xml_handle_enumeration(dct, obj, keyword, value, verbose) - elif keyword == "dimensions": xml_handle_dimensions(dct, obj, keyword, value) - elif keyword == "exists": xml_handle_exists(dct, obj, keyword, value) # Handles fileds e.g. AXISNAME @@ -1054,8 +1067,8 @@ def recursive_build(obj, dct, verbose): xml_handle_fields(obj, keyword, value, line_number, line_loc, verbose) else: raise ValueError( - f"An unfamiliar type of element {keyword} has been found which is " - f"not be able to be resolved. Chekc arround line {dct[line_number]}" + f"An unknown type of element {keyword} has been found which is " + f"not be able to be resolved. Chekc around line {dct[line_number]}" ) @@ -1065,12 +1078,12 @@ def pretty_print_xml(xml_root, output_xml, def_comments=None): built-in libraries and preceding XML processing instruction """ dom = minidom.parseString(ET.tostring(xml_root, encoding="utf-8", method="xml")) - proc_instractionn = dom.createProcessingInstruction( + proc_instruction = dom.createProcessingInstruction( "xml-stylesheet", 'type="text/xsl" href="nxdlformat.xsl"' ) dom_comment = dom.createComment(DOM_COMMENT) root = dom.firstChild - dom.insertBefore(proc_instractionn, root) + dom.insertBefore(proc_instruction, root) dom.insertBefore(dom_comment, root) if def_comments: @@ -1105,8 +1118,8 @@ def pretty_print_xml(xml_root, output_xml, def_comments=None): def nyaml2nxdl(input_file: str, out_file, verbose: bool): """ Main of the nyaml2nxdl converter, creates XML tree, namespace and - schema, definitions then evaluates a dictionary nest of groups recursively and - fields or (their) attributes as childs of the groups + schema, definitions then evaluates a nested dictionary of groups recursively and + fields or (their) attributes as children of the groups """ def_attributes = [ @@ -1149,13 +1162,13 @@ def nyaml2nxdl(input_file: str, out_file, verbose: bool): cmnt_text = xml_handle_comment( xml_root, line_number, line_loc_no, is_def_cmnt=True ) - def_cmnt_text += cmnt_text if cmnt_text else [] + def_cmnt_text += cmnt_text del yml_appdef[line_number] del yml_appdef[kkey] # Taking care or name and extends elif "NX" in kkey: - # Tacking the attribute order but the correct value will be stored later + # Taking the attribute order but the correct value will be stored later # check for name first or type first if (NXobject)NXname then type first l_bracket_ind = kkey.rfind("(") r_bracket_ind = kkey.rfind(")") @@ -1188,8 +1201,8 @@ def nyaml2nxdl(input_file: str, out_file, verbose: bool): "xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance", "xsi:schemaLocation": "http://definition.nexusformat.org/nxdl/3.1 ../nxdl.xsd", } - for key, ns_ in namespaces.items(): - xml_root.attrib[key] = ns_ + for key, ns in namespaces.items(): + xml_root.attrib[key] = ns # Taking care of Symbols elements if "symbols" in yml_appdef.keys(): xml_handle_symbols(yml_appdef, xml_root, "symbols", yml_appdef["symbols"]) @@ -1233,7 +1246,7 @@ def nyaml2nxdl(input_file: str, out_file, verbose: bool): (lin_annot, line_loc) = post_comment.get_line_info() xml_handle_comment(xml_root, lin_annot, line_loc) - # Note: Just to keep the functionality if we need this functionality later. + # Note: Just to keep the functionality if we need this later. default_attr = False if default_attr: check_for_default_attribute_and_value(xml_root) diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py b/dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py index dc609770d7..369befa48b 100644 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py @@ -36,8 +36,8 @@ from yaml.resolver import BaseResolver # NOTE: If any one change one of the bellow dict please change it for both -ESCAPE_CHAR_DICT_IN_YAML = {"\t": " ", "':'": ":"} -ESCAPE_CHAR_DICT_IN_XML = {" ": "\t", ":": "':'"} +ESCAPE_CHAR_DICT_IN_YAML = {"\t": " "} +ESCAPE_CHAR_DICT_IN_XML = {" ": "\t"} def remove_namespace_from_tag(tag): From 31d1ce339810da2dc327b2edb85a1ecdc94f2ba0 Mon Sep 17 00:00:00 2001 From: domna Date: Mon, 25 Sep 2023 13:42:29 +0200 Subject: [PATCH 64/93] Improved Makefile from fairmat branch --- Makefile | 41 +++++++++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index eff518f189..4e29846648 100644 --- a/Makefile +++ b/Makefile @@ -6,9 +6,18 @@ PYTHON = python3 SPHINX = sphinx-build BUILD_DIR = "build" -NXDL_DIRS := contributed_definitions applications base_classes +BASE_CLASS_DIR := base_classes +CONTRIB_DIR := contributed_definitions +APPDEF_DIR := applications +NXDL_DIRS := $(BASE_CLASS_DIR) $(CONTRIB_DIR) $(APPDEF_DIR) +NYAML_SUBDIR := nyaml +NYAML_APPENDIX := _parsed -.PHONY: help install style autoformat test clean prepare html pdf impatient-guide all local +NXDL_BC := $(wildcard $(BASE_CLASS_DIR)/*.nxdl.xml) +NXDL_CONTRIB := $(wildcard $(CONTRIB_DIR)/*.nxdl.xml) +NXDL_APPDEF := $(wildcard $(APPDEF_DIR)/*.nxdl.xml) + +.PHONY: help install style autoformat test clean prepare html pdf impatient-guide all local nyaml nxdl help :: @echo "" @@ -50,9 +59,9 @@ test :: clean :: $(RM) -rf $(BUILD_DIR) - for dir in $(NXDL_DIRS); do\ - $(RM) -rf $${dir}/nyaml;\ - done + $(RM) -rf $(BASE_CLASS_DIR)/$(NYAML_SUBDIR) + $(RM) -rf $(APPDEF_DIR)/$(NYAML_SUBDIR) + $(RM) -rf $(CONTRIB_DIR)/$(NYAML_SUBDIR) prepare :: $(PYTHON) -m dev_tools manual --prepare --build-root $(BUILD_DIR) @@ -87,6 +96,18 @@ all :: @echo "HTML built: `ls -lAFgh $(BUILD_DIR)/manual/build/html/index.html`" @echo "PDF built: `ls -lAFgh $(BUILD_DIR)/manual/build/latex/nexus.pdf`" +$(BASE_CLASS_DIR)/%.nxdl.xml : $(BASE_CLASS_DIR)/$(NYAML_SUBDIR)/%.yaml + nyaml2nxdl --input-file $< + mv $(BASE_CLASS_DIR)/$(NYAML_SUBDIR)/$*.nxdl.xml $@ + +$(CONTRIB_DIR)/%.nxdl.xml : $(CONTRIB_DIR)/$(NYAML_SUBDIR)/%.yaml + nyaml2nxdl --input-file $< + mv $(CONTRIB_DIR)/$(NYAML_SUBDIR)/$*.nxdl.xml $@ + +$(APPDEF_DIR)/%.nxdl.xml : $(APPDEF_DIR)/$(NYAML_SUBDIR)/%.yaml + nyaml2nxdl --input-file $< + mv $(APPDEF_DIR)/$(NYAML_SUBDIR)/$*.nxdl.xml $@ + NXDLS := $(foreach dir,$(NXDL_DIRS),$(wildcard $(dir)/*.nxdl.xml)) nyaml : $(DIRS) $(NXDLS) for file in $^; do\ @@ -96,15 +117,7 @@ nyaml : $(DIRS) $(NXDLS) mv -- "$${file%.nxdl.xml}_parsed.yaml" "$${file%/*}/nyaml/$${FNAME%.nxdl.xml}.yaml";\ done -NYAMLS := $(foreach dir,$(NXDL_DIRS),$(wildcard $(dir)/nyaml/*.yaml)) -nxdl : $(DIRS) $(NYAMLS) - for file in $^; do\ - mkdir -p "$${file%/*}/nyaml";\ - nyaml2nxdl --input-file $${file};\ - FNAME=$${file##*/};\ - mv -- "$${file%.yaml}.nxdl.xml" "$${file%/*}/../$${FNAME%.yaml}.nxdl.xml";\ - done - +nxdl: $(NXDL_APPDEF) $(NXDL_CONTRIB) $(NXDL_BC) # NeXus - Neutron and X-ray Common Data Format # From 6ceaebd6d92937fcc98b20c5de415816d37a58b2 Mon Sep 17 00:00:00 2001 From: domna Date: Mon, 25 Sep 2023 14:20:25 +0200 Subject: [PATCH 65/93] Removes unused code --- dev_tools/tests/test_nyaml2nxdl.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/dev_tools/tests/test_nyaml2nxdl.py b/dev_tools/tests/test_nyaml2nxdl.py index 792d8d4626..6d41d1085e 100755 --- a/dev_tools/tests/test_nyaml2nxdl.py +++ b/dev_tools/tests/test_nyaml2nxdl.py @@ -5,16 +5,12 @@ from ..nyaml2nxdl import nyaml2nxdl as conv from ..utils.nxdl_utils import find_definition_file -# import subprocess - def test_conversion(): root = find_definition_file("NXentry") - # subprocess.run(["python3","-m","dev_tools.nyaml2nxdl.nyaml2nxdl","--input-file",root]) result = CliRunner().invoke(conv.launch_tool, ["--input-file", root]) assert result.exit_code == 0 yaml = root[:-9] + "_parsed.yaml" - # subprocess.run(["python3","-m","dev_tools.nyaml2nxdl.nyaml2nxdl","--input-file",yaml]) result = CliRunner().invoke(conv.launch_tool, ["--input-file", yaml]) assert result.exit_code == 0 new_root = yaml[:-4] + "nxdl.xml" From b52cf088c5585566c75eac2f1796482c8458049f Mon Sep 17 00:00:00 2001 From: domna Date: Mon, 25 Sep 2023 14:20:49 +0200 Subject: [PATCH 66/93] Addresses some review comments in nxdl_utils --- dev_tools/utils/nxdl_utils.py | 40 ++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/dev_tools/utils/nxdl_utils.py b/dev_tools/utils/nxdl_utils.py index efba439bec..65d6f99ae2 100644 --- a/dev_tools/utils/nxdl_utils.py +++ b/dev_tools/utils/nxdl_utils.py @@ -9,18 +9,21 @@ from glob import glob -class NxdlAttributeError(Exception): - """An exception for throwing an error when an Nxdl attribute is not found.""" +class NxdlAttributeNotFoundError(Exception): + """An exception to throw when an Nxdl attribute is not found.""" def get_app_defs_names(): """Returns all the AppDef names without their extension: .nxdl.xml""" - app_def_path_glob = ( - f"{get_nexus_definitions_path()}{os.sep}applications{os.sep}*.nxdl*" + app_def_path_glob = os.path.join( + get_nexus_definitions_path(), + "applications", + "*.nxdl*" ) - contrib_def_path_glob = ( - f"{get_nexus_definitions_path()}{os.sep}" - f"contributed_definitions{os.sep}*.nxdl*" + contrib_def_path_glob = os.path.join( + get_nexus_definitions_path(), + "contributed_definitions", + "*.nxdl*" ) files = sorted(glob(app_def_path_glob)) + sorted(glob(contrib_def_path_glob)) return [os.path.basename(file).split(".")[0] for file in files] + ["NXroot"] @@ -40,7 +43,9 @@ def get_nexus_definitions_path(): return os.environ["NEXUS_DEF_PATH"] except KeyError: # or it should be available locally under the dir 'definitions' local_dir = os.path.abspath(os.path.dirname(__file__)) - return os.path.join(local_dir, f"..{os.sep}..") + for _ in range(2): + local_dir = os.path.dirname(local_dir) + return local_dir def get_hdf_root(hdf_node): @@ -60,7 +65,7 @@ def get_hdf_parent(hdf_info): if "hdf_root" not in hdf_info else hdf_info["hdf_root"] ) - for child_name in hdf_info["hdf_path"].split("/"): + for child_name in hdf_info["hdf_path"].rsplit("/"): node = node[child_name] return node @@ -86,24 +91,21 @@ def get_hdf_info_parent(hdf_info): def get_nx_class(nxdl_elem): """Get the nexus class for a NXDL node""" - if "category" in nxdl_elem.attrib.keys(): + if "category" in nxdl_elem.attrib: return None - try: - return nxdl_elem.attrib["type"] - except KeyError: - return "NX_CHAR" + return nxdl_elem.attrib.get("type", "NX_CHAR") def get_nx_namefit(hdf_name, name, name_any=False): """Checks if an HDF5 node name corresponds to a child of the NXDL element - uppercase letters in front can be replaced by arbitraty name, but + uppercase letters in front can be replaced by arbitrary name, but uppercase to lowercase match is preferred, so such match is counted as a measure of the fit""" if name == hdf_name: return len(name) * 2 # count leading capitals counting = 0 - while counting < len(name) and name[counting].upper() == name[counting]: + while counting < len(name) and name[counting].isupper(): counting += 1 if ( name_any @@ -198,7 +200,7 @@ def get_node_name(node): Either as specified by the 'name' or taken from the type (nx_class). Note that if only class name is available, the NX prefix is removed and the string is converted to UPPER case.""" - if "name" in node.attrib.keys(): + if "name" in node.attrib: name = node.attrib["name"] else: name = node.attrib["type"] @@ -837,7 +839,7 @@ def get_node_at_nxdl_path( (class_path, nxdlpath, elist) = get_inherited_nodes(nxdl_path, nx_name, elem) except ValueError as value_error: if exc: - raise NxdlAttributeError( + raise NxdlAttributeNotFoundError( f"Attributes were not found for {nxdl_path}. " "Please check this entry in the template dictionary." ) from value_error @@ -847,7 +849,7 @@ def get_node_at_nxdl_path( else: elem = None if exc: - raise NxdlAttributeError( + raise NxdlAttributeNotFoundError( f"Attributes were not found for {nxdl_path}. " "Please check this entry in the template dictionary." ) From a5233c748dc8c1b29824ee5b6af42b55fac90315 Mon Sep 17 00:00:00 2001 From: domna Date: Mon, 25 Sep 2023 15:20:31 +0200 Subject: [PATCH 67/93] Addresses comments in nxdl_utils --- dev_tools/utils/nxdl_utils.py | 94 ++++++++++++++--------------------- 1 file changed, 38 insertions(+), 56 deletions(-) diff --git a/dev_tools/utils/nxdl_utils.py b/dev_tools/utils/nxdl_utils.py index 65d6f99ae2..9252c46fe8 100644 --- a/dev_tools/utils/nxdl_utils.py +++ b/dev_tools/utils/nxdl_utils.py @@ -25,7 +25,13 @@ def get_app_defs_names(): "contributed_definitions", "*.nxdl*" ) - files = sorted(glob(app_def_path_glob)) + sorted(glob(contrib_def_path_glob)) + + files = sorted(glob(app_def_path_glob)) + for nexus_file in sorted(contrib_def_path_glob): + root = get_xml_root(nexus_file) + if root.attrib["category"] == "application": + files.append(nexus_file) + return [os.path.basename(file).split(".")[0] for file in files] + ["NXroot"] @@ -65,27 +71,21 @@ def get_hdf_parent(hdf_info): if "hdf_root" not in hdf_info else hdf_info["hdf_root"] ) - for child_name in hdf_info["hdf_path"].rsplit("/"): + for child_name in hdf_info["hdf_path"].split("/"): node = node[child_name] return node def get_parent_path(hdf_name): """Get parent path""" - return "/".join(hdf_name.split("/")[:-1]) + return hdf_name.rsplit("/", 1)[0] def get_hdf_info_parent(hdf_info): """Get the hdf_info for the parent of an hdf_node in an hdf_info""" if "hdf_path" not in hdf_info: return {"hdf_node": hdf_info["hdf_node"].parent} - node = ( - get_hdf_root(hdf_info["hdf_node"]) - if "hdf_root" not in hdf_info - else hdf_info["hdf_root"] - ) - for child_name in hdf_info["hdf_path"].split("/")[1:-1]: - node = node[child_name] + node = get_hdf_parent(hdf_info) return {"hdf_node": node, "hdf_path": get_parent_path(hdf_info["hdf_path"])} @@ -142,13 +142,12 @@ def get_nx_classes(): ) ) ) - nx_clss = [] + nx_class = [] for nexus_file in base_classes + applications + contributed: root = get_xml_root(nexus_file) if root.attrib["category"] == "base": - nx_clss.append(str(nexus_file[nexus_file.rindex(os.sep) + 1 :])[:-9]) - nx_clss = sorted(nx_clss) - return nx_clss + nx_class.append(os.path.basename(nexus_file).split(".")[0]) + return sorted(nx_class) def get_nx_units(): @@ -179,9 +178,9 @@ def get_nx_attribute_type(): root = get_xml_root(filepath) units_and_type_list = [] for child in root: - for i in child.attrib.values(): - units_and_type_list.append(i) + units_and_type_list.extend(child.attrib.values()) flag = False + nx_types = [] for line in units_and_type_list: if line == "primitiveType": flag = True @@ -234,7 +233,7 @@ def belongs_to_capital(params): (act_htmlname, chk_name, name_any, nxdl_elem, child, name) = params # or starts with capital and no reserved words used if ( - (name_any or "A" <= act_htmlname[0] <= "Z") + (act_htmlname[0].isalpha() and act_htmlname[0].isupper()) and name != "doc" and name != "enumeration" ): @@ -262,18 +261,19 @@ def belongs_to_capital(params): def get_local_name_from_xml(element): """Helper function to extract the element tag without the namespace.""" - return element.tag[element.tag.rindex("}") + 1 :] + return element.tag.rsplit("}")[-1] def get_own_nxdl_child_reserved_elements(child, name, nxdl_elem): """checking reserved elements, like doc, enumeration""" - if get_local_name_from_xml(child) == "doc" and name == "doc": + local_name = get_local_name_from_xml(child) + if local_name == "doc" and name == "doc": if nxdl_elem.get("nxdlbase"): child.set("nxdlbase", nxdl_elem.get("nxdlbase")) child.set("nxdlbase_class", nxdl_elem.get("nxdlbase_class")) child.set("nxdlpath", nxdl_elem.get("nxdlpath") + "/doc") return child - if get_local_name_from_xml(child) == "enumeration" and name == "enumeration": + if local_name == "enumeration" and name == "enumeration": if nxdl_elem.get("nxdlbase"): child.set("nxdlbase", nxdl_elem.get("nxdlbase")) child.set("nxdlbase_class", nxdl_elem.get("nxdlbase_class")) @@ -283,28 +283,16 @@ def get_own_nxdl_child_reserved_elements(child, name, nxdl_elem): def get_own_nxdl_child_base_types(child, class_type, nxdl_elem, name, hdf_name): - """checking base types of group, field,m attribute""" + """checking base types of group, field, attribute""" if get_local_name_from_xml(child) == "group": if ( class_type is None or (class_type and get_nx_class(child) == class_type) ) and belongs_to(nxdl_elem, child, name, class_type, hdf_name): - if nxdl_elem.get("nxdlbase"): - child.set("nxdlbase", nxdl_elem.get("nxdlbase")) - child.set("nxdlbase_class", nxdl_elem.get("nxdlbase_class")) - child.set( - "nxdlpath", nxdl_elem.get("nxdlpath") + "/" + get_node_name(child) - ) - return child + return set_nxdlpath(child, nxdl_elem) if get_local_name_from_xml(child) == "field" and belongs_to( nxdl_elem, child, name, None, hdf_name ): - if nxdl_elem.get("nxdlbase"): - child.set("nxdlbase", nxdl_elem.get("nxdlbase")) - child.set("nxdlbase_class", nxdl_elem.get("nxdlbase_class")) - child.set( - "nxdlpath", nxdl_elem.get("nxdlpath") + "/" + get_node_name(child) - ) - return child + return set_nxdlpath(child, nxdl_elem) if get_local_name_from_xml(child) == "attribute" and belongs_to( nxdl_elem, child, name, None, hdf_name ): @@ -314,7 +302,7 @@ def get_own_nxdl_child_base_types(child, class_type, nxdl_elem, name, hdf_name): child.set( "nxdlpath", nxdl_elem.get("nxdlpath") + "/" + get_node_name(child) ) - return child + return set_nxdlpath(child, nxdl_elem) return False @@ -335,7 +323,7 @@ def get_own_nxdl_child( ) return child for child in nxdl_elem: - if "name" in child.attrib and child.attrib["name"] == name: + if child.attrib.get("name") == name: child.set("nxdlbase", nxdl_elem.get("nxdlbase")) return child @@ -422,11 +410,8 @@ def get_required_string(nxdl_elem): if is_optional or is_minoccurs: return "<>" # default optionality: in BASE CLASSES is true; in APPLICATIONS is false - try: - if nxdl_elem.get("nxdlbase_class") == "base": - return "<>" - except TypeError: - return "<>" + if nxdl_elem.get("nxdlbase_class") == "base": + return "<>" return "<>" @@ -434,7 +419,7 @@ def get_required_string(nxdl_elem): def write_doc_string(logger, doc, attr): """Simple function that prints a line in the logger if doc exists""" if doc: - logger.debug("@" + attr + " [NX_CHAR]") + logger.debug("@%s [NX_CHAR]", attr) return logger, doc, attr @@ -567,7 +552,7 @@ def other_attrs( def get_node_concept_path(elem): """get the short version of nxdlbase:nxdlpath""" - return str(elem.get("nxdlbase").split("/")[-1] + ":" + elem.get("nxdlpath")) + return f'{elem.get("nxdlbase").split("/")[-1]}:{elem.get("nxdlpath")}' def get_doc(node, ntype, nxhtml, nxpath): @@ -590,7 +575,7 @@ def get_doc(node, ntype, nxhtml, nxpath): if index: enum_str = ( "\n " - + ("Possible values:" if len(enums.split(",")) > 1 else "Obligatory value:") + + ("Possible values:" if enums.count(",") else "Obligatory value:") + "\n " + enums + "\n" @@ -639,8 +624,9 @@ def get_enums(node): def add_base_classes(elist, nx_name=None, elem: ET.Element = None): - """Add the base classes corresponding to the last eleme in elist to the list. Note that if - elist is empty, a nxdl file with the name of nx_name or a rather room elem is used if provided + """ + Add the base classes corresponding to the last elemenent in elist to the list. Note that if + elist is empty, a nxdl file with the name of nx_name or a placeholder elem is used if provided """ if elist and nx_name is None: nx_name = get_nx_class(elist[-1]) @@ -681,7 +667,7 @@ def set_nxdlpath(child, nxdl_elem): def get_direct_child(nxdl_elem, html_name): """returns the child of nxdl_elem which has a name - corresponding to the the html documentation name html_name""" + corresponding to the html documentation name html_name""" for child in nxdl_elem: if get_local_name_from_xml(child) in ( "group", @@ -785,9 +771,7 @@ def walk_elist(elist, html_name): if fitting_child is not None: child = fitting_child break - elist[ind] = child - if elist[ind] is None: - del elist[ind] + if child is None: continue # override: remove low priority inheritance classes if class_type is overriden if len(elist) > ind + 1 and get_nx_class(elist[ind]) != get_nx_class( @@ -814,11 +798,9 @@ def get_inherited_nodes( nxdl_elem_path = [elist[0]] class_path = [] # type: ignore[var-annotated] - html_path = nxdl_path.split("/")[1:] - path = html_path - for pind in range(len(path)): - html_name = html_path[pind] - elist, html_name = walk_elist(elist, html_name) + html_paths = nxdl_path.split("/")[1:] + for html_name in html_paths: + elist, _ = walk_elist(elist, html_name) if elist: class_path.append(get_nx_class(elist[0])) nxdl_elem_path.append(elist[0]) From 6149993b71b7bc51be419550d13a3d246184e074 Mon Sep 17 00:00:00 2001 From: domna Date: Mon, 25 Sep 2023 15:22:49 +0200 Subject: [PATCH 68/93] Fixes typo in readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4f4020aa6f..0fd9beb45b 100755 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ Open the HTML manual in a web brower for visual verification firefox build/manual/build/html/index.html -Convenient editting of definitions is available in nyaml format. For this, definitions need to be converted first from xml to yaml format by the command +Convenient editing of definitions is available in nyaml format. For this, definitions need to be converted first from xml to yaml format by the command make nyaml From e56c7d5d25a1a92508c5fcaf2316fcba1e8a8a91 Mon Sep 17 00:00:00 2001 From: Rubel Date: Mon, 25 Sep 2023 14:45:15 +0200 Subject: [PATCH 69/93] Updating nyam2nxdl converter files. --- Makefile | 2 +- .../nyaml2nxdl/nyaml2nxdl_backward_tools.py | 4 +- .../nyaml2nxdl/nyaml2nxdl_forward_tools.py | 10 +- dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py | 105 ++++++++++-------- 4 files changed, 63 insertions(+), 58 deletions(-) diff --git a/Makefile b/Makefile index 4e29846648..66b4ebc8ce 100644 --- a/Makefile +++ b/Makefile @@ -112,7 +112,7 @@ NXDLS := $(foreach dir,$(NXDL_DIRS),$(wildcard $(dir)/*.nxdl.xml)) nyaml : $(DIRS) $(NXDLS) for file in $^; do\ mkdir -p "$${file%/*}/nyaml";\ - nyaml2nxdl --input-file $${file};\ + nyaml2nxdl --input-file $${file} --do-not-store-nxdl;\ FNAME=$${file##*/};\ mv -- "$${file%.nxdl.xml}_parsed.yaml" "$${file%/*}/nyaml/$${FNAME%.nxdl.xml}.yaml";\ done diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py b/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py index e709ffe4a6..bea2d74e90 100755 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py @@ -27,7 +27,7 @@ from typing import Dict from typing import List -from .nyaml2nxdl_helper import cleaning_empty_lines +from .nyaml2nxdl_helper import clean_empty_lines from .nyaml2nxdl_helper import get_node_parent_info from .nyaml2nxdl_helper import get_yaml_escape_char_dict from .nyaml2nxdl_helper import remove_namespace_from_tag @@ -331,7 +331,7 @@ def handle_not_root_level_doc(self, depth, text, tag="doc", file_out=None): text = handle_mapping_char(text, -1, True) if "\n" in text: # To remove '\n' with non-space character as it will be added before text. - text = cleaning_empty_lines(text.split("\n")) + text = clean_empty_lines(text.split("\n")) text_tmp = [] yaml_indent_n = len((depth + 1) * DEPTH_SIZE) diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py b/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py index 0ccd0cd6e6..e62be14d24 100644 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py @@ -34,11 +34,12 @@ from ..utils import nxdl_utils as pynxtools_nxlib from .comment_collector import CommentCollector from .nyaml2nxdl_helper import LineLoader -from .nyaml2nxdl_helper import cleaning_empty_lines +from .nyaml2nxdl_helper import clean_empty_lines from .nyaml2nxdl_helper import get_yaml_escape_char_reverter_dict from .nyaml2nxdl_helper import nx_name_type_resolving from .nyaml2nxdl_helper import remove_namespace_from_tag + # pylint: disable=too-many-lines, global-statement, invalid-name DOM_COMMENT = ( f"\n" @@ -235,7 +236,7 @@ def format_nxdl_doc(string): formatted_doc = "\n" + f"{string}" else: text_lines = string.split("\n") - text_lines = cleaning_empty_lines(text_lines) + text_lines = clean_empty_lines(text_lines) formatted_doc += "\n" + "\n".join(text_lines) if not formatted_doc.endswith("\n"): formatted_doc += "\n" @@ -1121,7 +1122,6 @@ def nyaml2nxdl(input_file: str, out_file, verbose: bool): schema, definitions then evaluates a nested dictionary of groups recursively and fields or (their) attributes as children of the groups """ - def_attributes = [ "deprecated", "ignoreExtraGroups", @@ -1209,11 +1209,9 @@ def nyaml2nxdl(input_file: str, out_file, verbose: bool): del yml_appdef["symbols"] del yml_appdef["__line__symbols"] - assert ( isinstance(yml_appdef["doc"], str) and yml_appdef["doc"] != "" - ), "Doc \ -has to be a non-empty string!" + ), "Doc has to be a non-empty string!" line_number = "__line__doc" line_loc_no = yml_appdef[line_number] diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py b/dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py index 369befa48b..d62bee2fc8 100644 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 -"""Main file of yaml2nxdl tool. -Users create NeXus instances by writing a YAML file -which details a hierarchy of data/metadata elements +"""File consists of helping functions and variables. +The functions and variables are utilised in the converting tool +to convert from nyaml to nxdl and vice versa. """ # -*- coding: utf-8 -*- # @@ -24,9 +24,9 @@ # -# Yaml library does not except the keys (escapechar "\t" and yaml separator ":") +# Yaml library does not except the keys (escape char "\t" and yaml separator ":") # So the corresponding value is to skip them and -# and also carefull about this order +# and also careful about this order import hashlib from yaml.composer import Composer @@ -35,9 +35,8 @@ from yaml.nodes import ScalarNode from yaml.resolver import BaseResolver -# NOTE: If any one change one of the bellow dict please change it for both ESCAPE_CHAR_DICT_IN_YAML = {"\t": " "} -ESCAPE_CHAR_DICT_IN_XML = {" ": "\t"} +ESCAPE_CHAR_DICT_IN_XML = {val : key for key, val in ESCAPE_CHAR_DICT_IN_YAML.items()} def remove_namespace_from_tag(tag): @@ -47,22 +46,25 @@ def remove_namespace_from_tag(tag): class LineLoader(Loader): # pylint: disable=too-many-ancestors - """ + """Class to load yaml file with extra non yaml items. + LineLoader parses a yaml into a python dictionary extended with extra items. The new items have as keys __line__ and as values the yaml file line number """ def compose_node(self, parent, index): + """Compose node and return node.""" # the line number where the previous token has ended (plus empty lines) node = Composer.compose_node(self, parent, index) node.__line__ = self.line + 1 return node def construct_mapping(self, node, deep=False): - node_pair_lst = node.value + """Construct mapping between node info and line info.""" node_pair_lst_for_appending = [] - for key_node in node_pair_lst: + # Visit through node-pair list + for key_node in node.value: shadow_key_node = ScalarNode( tag=BaseResolver.DEFAULT_SCALAR_TAG, value="__line__" + key_node[0].value, @@ -72,7 +74,7 @@ def construct_mapping(self, node, deep=False): ) node_pair_lst_for_appending.append((shadow_key_node, shadow_value_node)) - node.value = node_pair_lst + node_pair_lst_for_appending + node.value = node.value + node_pair_lst_for_appending return Constructor.construct_mapping(self, node, deep=deep) @@ -88,9 +90,7 @@ def get_yaml_escape_char_reverter_dict(): def type_check(nx_type): - """ - Check for nexus type if type is NX_CHAR get '' or get as it is. - """ + """Check for nexus type if type is NX_CHAR get '' or get as it is.""" if nx_type in ["NX_CHAR", ""]: nx_type = "" @@ -100,48 +100,47 @@ def type_check(nx_type): def get_node_parent_info(tree, node): - """ - Return tuple of (parent, index) where: - parent = node of parent within tree - index = index of node under parent + """Return tuple of (parent, index). + + parent = parent node is the first level node under tree node + index = index of grand child node of tree """ + # map from grand child to child parent of tree parent_map = {c: p for p in tree.iter() for c in p} parent = parent_map[node] return parent, list(parent).index(node) -def cleaning_empty_lines(line_list): - """ - Cleaning up empty lines on top and bottom. - """ +def clean_empty_lines(line_list): + """Clean up empty lines by top part and bottom and part.""" if not isinstance(line_list, list): line_list = line_list.split("\n") if "\n" in line_list else [""] - # Clining up top empty lines - while True: - if line_list[0].strip(): + start_non_empty_line = 0 + ends_non_empty_line = None + # Find the index of first non-empty line + for ind, line in enumerate(line_list): + if not line.isspace(): + start_non_empty_line = ind break - line_list = line_list[1:] - if len(line_list) == 0: - line_list.append("") - return line_list - - # Clining bottom empty lines - while True: - if line_list[-1].strip(): + + # Find the index of the last non-empty line + for ind, line in enumerate(line_list[::-1]): + if not line.isspace(): + ends_non_empty_line = -ind break - line_list = line_list[0:-1] - if len(line_list) == 0: - line_list.append("") - return line_list - return line_list + if ends_non_empty_line == 0 : + ends_non_empty_line = None + + return line_list[start_non_empty_line : ends_non_empty_line] def nx_name_type_resolving(tmp): - """ - extracts the eventually custom name {optional_string} + """Separate name and NeXus type + + Extracts the eventually custom name {optional_string} and type {nexus_type} from a YML section string. YML section string syntax: optional_string(nexus_type) """ @@ -150,6 +149,10 @@ def nx_name_type_resolving(tmp): # either an nx_ (type, base, candidate) class contains only 1 '(' and ')' index_start = tmp.index("(") index_end = tmp.index(")", index_start + 1) + if index_start > index_end: + raise ValueError(f"Check name and type combination {tmp} which can not be resolved.") + if index_end - index_start == 1: + raise ValueError(f"Check name(type) combination {tmp}, properly not defined.") typ = tmp[index_start + 1 : index_end] nam = tmp.replace("(" + typ + ")", "") return nam, typ @@ -191,11 +194,16 @@ def extend_yamlfile_by_nxdl_as_comment( def separate_hash_yaml_and_nxdl(yaml_file, sep_yaml, sep_xml): - """Separate the provided yaml file into yaml, nxdl and hash if yaml was extended with - nxdl at the end of yaml by + """Separate yaml, SHA hash and nxdl parts. + + Separate the provided yaml file into yaml, nxdl and hash if yaml was extended with + nxdl at the end of yaml as + + '\n# ++++++++++++++++++++++++++++++++++ SHA HASH \ ++++++++++++++++++++++++++++++++++\n' # ' + """ sha_hash = "" with open(yaml_file, "r", encoding="utf-8") as inp_file: @@ -204,20 +212,19 @@ def separate_hash_yaml_and_nxdl(yaml_file, sep_yaml, sep_xml): with open(sep_yaml, "w", encoding="utf-8") as yml_f_ob, open( sep_xml, "w", encoding="utf-8" ) as xml_f_ob: - last_line = "" write_on_yaml = True - for ind, line in enumerate(lines): - if ind == 0: - last_line = line - # Write in file when ensured that the nest line is not with '++ SHA HASH ++' - elif "++ SHA HASH ++" not in line and write_on_yaml: + + last_line = lines[0] + for line in lines[1:]: + # Write in file when ensured that the next line is not with '++ SHA HASH ++' + if "++ SHA HASH ++" not in line and write_on_yaml: yml_f_ob.write(last_line) last_line = line elif "++ SHA HASH ++" in line: write_on_yaml = False last_line = "" elif not write_on_yaml and not last_line: - # The first line of xml file has been found. Onward write lines directly + # The first line of xml file has been found so in future write lines directly # into xml file. if not sha_hash: sha_hash = line.split("# ", 1)[-1].strip() From b276962208744178e2ea1848a5817c6fd20f3896 Mon Sep 17 00:00:00 2001 From: domna Date: Mon, 25 Sep 2023 15:36:44 +0200 Subject: [PATCH 70/93] Fix typo in setuptools_scm --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f4377ff3f4..b433177e67 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ dependencies = [ [project.scripts] nyaml2nxdl = "dev_tools.nyaml2nxdl.nyaml2nxdl:launch_tool" -[tools.setuptools_scm] +[tool.setuptools_scm] version_scheme = "guess-next-dev" local_scheme = "node-and-date" From 8b8d4de78eb139a5e6c7280c6baaa6581ed8ea85 Mon Sep 17 00:00:00 2001 From: Rubel Date: Mon, 25 Sep 2023 16:33:29 +0200 Subject: [PATCH 71/93] Resolving requested changes. --- Makefile | 2 +- dev_tools/nyaml2nxdl/README.md | 12 +++---- dev_tools/nyaml2nxdl/comment_collector.py | 32 +++++++++---------- .../nyaml2nxdl/nyaml2nxdl_forward_tools.py | 4 +-- 4 files changed, 23 insertions(+), 27 deletions(-) diff --git a/Makefile b/Makefile index 66b4ebc8ce..4e29846648 100644 --- a/Makefile +++ b/Makefile @@ -112,7 +112,7 @@ NXDLS := $(foreach dir,$(NXDL_DIRS),$(wildcard $(dir)/*.nxdl.xml)) nyaml : $(DIRS) $(NXDLS) for file in $^; do\ mkdir -p "$${file%/*}/nyaml";\ - nyaml2nxdl --input-file $${file} --do-not-store-nxdl;\ + nyaml2nxdl --input-file $${file};\ FNAME=$${file##*/};\ mv -- "$${file%.nxdl.xml}_parsed.yaml" "$${file%/*}/nyaml/$${FNAME%.nxdl.xml}.yaml";\ done diff --git a/dev_tools/nyaml2nxdl/README.md b/dev_tools/nyaml2nxdl/README.md index a8af60ad3e..5a48f49887 100644 --- a/dev_tools/nyaml2nxdl/README.md +++ b/dev_tools/nyaml2nxdl/README.md @@ -45,7 +45,7 @@ Options: **Special keywords**: Several keywords can be used as childs of groups, fields, and attributes to specify the members of these. Groups, fields and attributes are nodes of the XML tree. * **doc**: A human-readable description/docstring -* **exists** Options are recommended, required, [min, 1, max, infty] numbers like here 1 can be replaced by any uint, or infty to indicate no restriction on how frequently the entry can occur inside the NXDL schema at the same hierarchy level. +* **exists** Options are recommended, required, [min, 1, max, `infty`] numbers like here 1 can be replaced by any `uint` (unsigned integer), or `infty` to indicate no restriction on how frequently the entry can occur inside the NXDL schema at the same hierarchy level. * **link** Define links between nodes. * **units** A statement introducing NeXus-compliant NXDL units arguments, like NX_VOLTAGE * **dimensions** Details which dimensional arrays to expect @@ -60,13 +60,9 @@ dimensions: ref: [ref_value_1, ref_value_2, ...] incr: [incr_value_1, incr_value_2, ...] ``` -Keep in mind that length of all the lists must be same. +Keep in mind that length of all the lists must have the **same size**. +**Important Note**: The attributes `ref`, `incr`, `index` are deprecated. ## Next steps -The NOMAD team is currently working on the establishing of a one-to-one mapping between -NeXus definitions and the NOMAD MetaInfo. As soon as this is in place the YAML files will -be annotated with further metadata so that they can serve two purposes. -On the one hand they can serve as an instance for a schema to create a GUI representation -of a NOMAD Oasis ELN schema. On the other hand the YAML to NXDL converter will skip all -those pieces of information which are irrelevant from a NeXus perspective. +The NOMAD team is currently working to establish a one-to-one mapping between NeXus definitions and the NOMAD MetaInfo(scientific data model in nomad). As soon as this is in place the YAML files will be annotated with further metadata so that they can serve two purposes. On the one hand they can serve as an instance for a schema to create a GUI representation of a NOMAD Oasis ELN schema. On the other hand the YAML to NXDL converter will skip all those pieces of information which are irrelevant from a NeXus perspective. diff --git a/dev_tools/nyaml2nxdl/comment_collector.py b/dev_tools/nyaml2nxdl/comment_collector.py index 3a339e7be8..573b02bd6d 100644 --- a/dev_tools/nyaml2nxdl/comment_collector.py +++ b/dev_tools/nyaml2nxdl/comment_collector.py @@ -19,9 +19,9 @@ # """ -Collect comments in a list by CommentCollector class and each comment is a instance of Comment -class. Comment (sometimes refered as 'comment block') instance consists of text and line info or neighbours info where the -comment must be placed. +Collect comments in a list by CommentCollector class and each comment is an instance of Comment +class. Each `Comment` instance(sometimes refered as 'comment block') consists of text and line +info or neighbours info where the comment must be placed. The class Comment is an abstract class for general functions or method to be implemented XMLComment and YAMLComment class. @@ -83,8 +83,9 @@ def __init__(self, input_file: str = None, loaded_obj: Union[object, Dict] = Non raise ValueError("Incorrect inputs for CommentCollector") def extract_all_comment_blocks(self): - """ - Collect all comments. Note here that comment means (comment text + element or line info + """Collect all comments. + + Note here that comment means (comment text + element or line info intended for comment. """ id_ = 0 @@ -125,7 +126,7 @@ def get_next_comment(self): """ return self._comment_chain[self._comment_tracker] - def get_coment_by_line_info(self, comment_locs: Tuple[str, Union[int, str]]): + def get_comment_by_line_info(self, comment_locs: Tuple[str, Union[int, str]]): """ Get comment using line information. """ @@ -376,7 +377,9 @@ def __init__(self, comment_id: int = -1, last_comment: "Comment" = None) -> None ) def process_each_line(self, text, line_num): - """Take care of each line of text. Through which function the text + """Process each line. + + Take care of each line of text. Through which function the text must be passed should be decide here. """ text = text.strip() @@ -402,8 +405,8 @@ def process_each_line(self, text, line_num): self.store_element(line_key, line_num) def has_post_comment(self): - """ - Ensure if this is a post coment or not. + """Ensure if this is a post coment or not. + Post comment means the comment that come at the very end without having any nxdl element(class, group, filed and attribute.) """ @@ -413,7 +416,7 @@ def has_post_comment(self): return False def append_comment(self, text: str) -> None: - """Append comment texts to associated comment. + """Append comment texts to the associated comment. Collects all the line of the same comment and append them with that single comment. @@ -447,9 +450,7 @@ def append_comment(self, text: str) -> None: # pylint: disable=arguments-differ def store_element(self, line_key, line_number): - """ - Store comment contents and information of comment location (for what comment is - created.). + """Store comment contents and information of comment location. """ self._elemt = {} self._elemt[line_key] = int(line_number) @@ -482,8 +483,7 @@ def replace_escape_char(self, text): return text def get_element_location(self): - """ - Return yaml line '__line__KEY' info and and line numner + """Return yaml line '__line__' info and and line numner """ if len(self._elemt) == 1: raise ValueError(f"Comment element should be one but got " f"{self._elemt}") @@ -502,7 +502,7 @@ def collect_yaml_line_info(self, yaml_dict, line_info_dict): self.collect_yaml_line_info(val, line_info_dict) def __contains__(self, line_key): - """For checking whether __line__NAME is in _elemt dict or not.""" + """For checking whether __line__ is in _elemt dict or not.""" return line_key in self._elemt def __eq__(self, comment_obj): diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py b/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py index e62be14d24..f1c227763c 100644 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py @@ -508,7 +508,7 @@ def xml_handle_dim_from_dimension_dict( line_loc = value[line_number] # dim comes in precedence if attr == "dim": - # dim consists of list of [index, value] + # dim consists of [index, value] list llist_ind_value = vvalue assert isinstance(llist_ind_value, list), ( f"Line {value[line_number]}: dim argument not a list !" @@ -994,7 +994,7 @@ def xml_handle_comment( line_info = (line_annotation, int(line_loc_no)) if line_info in COMMENT_BLOCKS: # noqa: F821 - cmnt = COMMENT_BLOCKS.get_coment_by_line_info(line_info) # noqa: F821 + cmnt = COMMENT_BLOCKS.get_comment_by_line_info(line_info) # noqa: F821 cmnt_text = cmnt.get_comment_text_list() # Check comment for definition element From 96b146fba3b5213c41c8047ff05c51398b0f84ef Mon Sep 17 00:00:00 2001 From: Rubel Date: Tue, 26 Sep 2023 08:37:02 +0200 Subject: [PATCH 72/93] Fixing attributes for nxdl and yaml separately. --- .../nyaml2nxdl/nyaml2nxdl_backward_tools.py | 84 ++++++++----------- .../nyaml2nxdl/nyaml2nxdl_forward_tools.py | 72 ++++------------ dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py | 62 ++++++++++++++ 3 files changed, 113 insertions(+), 105 deletions(-) diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py b/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py index bea2d74e90..cdec34f661 100755 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py @@ -31,6 +31,11 @@ from .nyaml2nxdl_helper import get_node_parent_info from .nyaml2nxdl_helper import get_yaml_escape_char_dict from .nyaml2nxdl_helper import remove_namespace_from_tag +from .nyaml2nxdl_helper import (NXDL_FIELD_ATTRIBUTES, + NXDL_GROUP_ATTRIBUTES, + NXDL_ATTRIBUTES_ATTRIBUTES, + NXDL_LINK_ATTRIBUTES) + DEPTH_SIZE = " " CMNT_TAG = "!--" @@ -40,8 +45,8 @@ DEFINITION_CATEGORIES = ("category: application", "category: base") def separate_pi_comments(input_file): - """ - Separate PI comments from ProcessesInstruction (PI) + """Separate PI comments from ProcessesInstruction (PI) + Separeate the comments that comes immediately after XML process instruction part, i.e. copyright comment part. """ @@ -158,42 +163,7 @@ def __init__( # The 'symbol_comments' contains comments for 'symbols doc' and all 'symbol' # 'symbol_comments': [comments]} self.root_level_comment: Dict[str, str] = {} - self.grp_fld_allowed_attr = ( - "optional", - "recommended", - "name", - "type", - "axes", - "axis", - "data_offset", - "interpretation", - "long_name", - "maxOccurs", - "minOccurs", - "nameType", - "optional", - "primary", - "signal", - "stride", - "units", - "required", - "deprecated", - "exists", - ) - self.attr_allowed_attr = ( - "name", - "type", - "units", - "nameType", - "recommended", - "optional", - "minOccurs", - "maxOccurs", - "deprecated", - ) - self.link_allowed_attr = ("name", - "target", - "napimount") + self.optionality_keys = ("minOccurs", "maxOccurs", "optional", @@ -543,7 +513,7 @@ def handle_group_or_field(self, depth, node, file_out): name_type = name_type + val rm_key_list.append(key) elif key == "type": - name_type = name_type + "(%s)" % val + name_type = f"{name_type}({val})" rm_key_list.append(key) if not name_type: raise ValueError( @@ -560,11 +530,8 @@ def handle_group_or_field(self, depth, node, file_out): tmp_dict = {} exists_dict = {} for key, val in node_attr.items(): - if key not in self.grp_fld_allowed_attr: - raise ValueError( - f"An attribute ({key}) in 'field' or 'group' has been found " - f"that is not allowed. The allowed attr is {self.grp_fld_allowed_attr}." - ) + # Check for any unwanted attributes + self.check_for_unwanted_attributes(node=node) # As both 'minOccurs', 'maxOccurs' and optionality move to the 'exists' if key in self.optionality_keys: if "exists" not in tmp_dict: @@ -591,6 +558,25 @@ def handle_group_or_field(self, depth, node, file_out): f"{handle_mapping_char(val, depth_ + 1, False)}\n" ) + def check_for_unwanted_attributes(self, node, allowed_attributes_li=None, tag=None): + """Check for any attributes that NeXus does not allow.""" + node_tag = remove_namespace_from_tag(node.tag) + if node_tag == "field": + for key in node.attrib.keys(): + if key not in NXDL_FIELD_ATTRIBUTES: + raise ValueError(f"Field has got an unwanted attribute {key}." + f"NeXus field allows attributes from {NXDL_FIELD_ATTRIBUTES}") + elif node_tag == "group": + for key in node.attrib.keys(): + if key not in NXDL_GROUP_ATTRIBUTES: + raise ValueError(f"Attribute has got an unwanted attribute {key}." + f"NeXus attribute allows attributes from {NXDL_GROUP_ATTRIBUTES}") + elif node_tag == tag: + for key in node.attrib.keys(): + if key not in allowed_attributes_li: + raise ValueError(f"{tag.capitalized()} has got an unwanted attribute {key}." + f"NeXus {tag.capitalized()} allows attributes from {allowed_attributes_li}") + # pylint: disable=too-many-branches, too-many-locals def handle_dimension(self, depth, node, file_out): """ @@ -776,10 +762,10 @@ def handle_attributes(self, depth, node, file_out): tmp_dict = {} exists_dict = {} for key, val in node_attr.items(): - if key not in self.attr_allowed_attr: + if key not in NXDL_ATTRIBUTES_ATTRIBUTES: raise ValueError( f"An attribute ({key}) has been found that is not allowed." - f"The allowed attr is {self.attr_allowed_attr}." + f"The allowed attr is {NXDL_ATTRIBUTES_ATTRIBUTES}." ) # As both 'minOccurs', 'maxOccurs' and optionality move to the 'exists' if key in self.optionality_keys: @@ -834,13 +820,13 @@ def handle_link(self, depth, node, file_out): depth_ = depth + 1 # Handle general cases for attr_key, val in node_attr.items(): - if attr_key in self.link_allowed_attr: + if attr_key in NXDL_LINK_ATTRIBUTES: indent = depth_ * DEPTH_SIZE file_out.write(f"{indent}{attr_key}: {val}\n") else: raise ValueError( f"An unexpected attribute '{attr_key}' of link has found." - f"At this moment the alloed keys are {self.link_allowed_attr}" + f"At this moment the alloed keys are {NXDL_LINK_ATTRIBUTES}" ) def handle_choice(self, depth, node, file_out): @@ -934,7 +920,7 @@ def xmlparse(self, output_yml, xml_tree, depth, verbose): node.remove(remove_cmnt_n) remove_cmnt_n = None - if tag == ("doc") and depth != 1: + if tag == "doc" and depth != 1: parent = get_node_parent_info(tree, node)[0] doc_parent = remove_namespace_from_tag(parent.tag) if doc_parent != "item": diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py b/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py index f1c227763c..b6e1115627 100644 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py @@ -37,6 +37,10 @@ from .nyaml2nxdl_helper import clean_empty_lines from .nyaml2nxdl_helper import get_yaml_escape_char_reverter_dict from .nyaml2nxdl_helper import nx_name_type_resolving +from .nyaml2nxdl_helper import (YAML_ATTRIBUTES_ATTRIBUTES, + YAML_FIELD_ATTRIBUTES, + YAML_GROUP_ATTRIBUTES, + YAML_LINK_ATTRIBUTES) from .nyaml2nxdl_helper import remove_namespace_from_tag @@ -342,16 +346,7 @@ def xml_handle_group(dct, obj, keyword, value, verbose=False): line_number = f"__line__{keyword}" line_loc = dct[line_number] xml_handle_comment(obj, line_number, line_loc) - list_of_attr = [ - "name", - "type", - "nameType", - "deprecated", - "optional", - "recommended", - "exists", - "unit", - ] + l_bracket = -1 r_bracket = -1 if keyword.count("(") == 1: @@ -396,8 +391,8 @@ def xml_handle_group(dct, obj, keyword, value, verbose=False): elif attr == "unit": xml_handle_units(grp, vval) xml_handle_comment(obj, line_number, line_loc, grp) - elif attr in list_of_attr and not isinstance(vval, dict) and vval: - validate_field_attribute_and_value(attr, vval, list_of_attr, value) + elif attr in YAML_GROUP_ATTRIBUTES and not isinstance(vval, dict) and vval: + validate_field_attribute_and_value(attr, vval, YAML_GROUP_ATTRIBUTES, value) grp.set(attr, check_for_mapping_char_other(vval)) rm_key_list.append(attr) rm_key_list.append(line_number) @@ -406,7 +401,7 @@ def xml_handle_group(dct, obj, keyword, value, verbose=False): for key in rm_key_list: del value[key] # Check for skipped attrinutes - check_for_skipped_attributes("group", value, list_of_attr, verbose) + check_for_skipped_attributes("group", value, YAML_GROUP_ATTRIBUTES, verbose) if isinstance(value, dict) and len(value) != 0: recursive_build(grp, value, verbose) @@ -629,7 +624,6 @@ def xml_handle_link(dct, obj, keyword, value, verbose): line_number = f"__line__{keyword}" line_loc = dct[line_number] xml_handle_comment(obj, line_number, line_loc) - possible_attrs = ["name", "target", "napimount"] name = keyword[:-6] link_obj = ET.SubElement(obj, "link") link_obj.set("name", str(name)) @@ -645,7 +639,7 @@ def xml_handle_link(dct, obj, keyword, value, verbose): xml_handle_doc(link_obj, vval, line_number, line_loc) rm_key_list.append(attr) rm_key_list.append(line_number) - elif attr in possible_attrs and not isinstance(vval, dict): + elif attr in YAML_LINK_ATTRIBUTES and not isinstance(vval, dict): if vval: link_obj.set(attr, str(vval)) rm_key_list.append(attr) @@ -655,7 +649,7 @@ def xml_handle_link(dct, obj, keyword, value, verbose): for key in rm_key_list: del value[key] # Check for skipped attrinutes - check_for_skipped_attributes("link", value, possible_attrs, verbose) + check_for_skipped_attributes("link", value, YAML_LINK_ATTRIBUTES, verbose) if isinstance(value, dict) and value != {}: recursive_build(link_obj, value, verbose=None) @@ -768,7 +762,7 @@ def verbose_flag(verbose, keyword, value): Verbose stdout printing for nested levels of yaml file, if verbose flag is active """ if verbose: - sys.stdout.write(f" key:{keyword}; value type is {type(value)}\n") + print(f"key:{keyword}; value type is {type(value)}\n") def xml_handle_attributes(dct, obj, keyword, value, verbose): @@ -777,19 +771,6 @@ def xml_handle_attributes(dct, obj, keyword, value, verbose): line_number = f"__line__{keyword}" line_loc = dct[line_number] xml_handle_comment(obj, line_number, line_loc) - # list of possible attribute of xml attribute elements - attr_attr_list = [ - "name", - "type", - "unit", - "nameType", - "optional", - "recommended", - "minOccurs", - "maxOccurs", - "deprecated", - "exists", - ] # as an attribute identifier keyword_name, keyword_typ = nx_name_type_resolving(keyword) line_number = f"__line__{keyword}" @@ -810,7 +791,7 @@ def xml_handle_attributes(dct, obj, keyword, value, verbose): continue line_number = f"__line__{attr}" line_loc = value[line_number] - if attr in ["doc", *attr_attr_list] and not isinstance(attr_val, dict): + if attr in ["doc", *YAML_ATTRIBUTES_ATTRIBUTES] and not isinstance(attr_val, dict): if attr == "unit": elemt_obj.set(f"{attr}s", str(value[attr])) rm_key_list.append(attr) @@ -836,7 +817,7 @@ def xml_handle_attributes(dct, obj, keyword, value, verbose): for key in rm_key_list: del value[key] # Check cor skiped attribute - check_for_skipped_attributes("Attribute", value, attr_attr_list, verbose) + check_for_skipped_attributes("Attribute", value, YAML_ATTRIBUTES_ATTRIBUTES, verbose) if value: recursive_build(elemt_obj, value, verbose) @@ -886,27 +867,6 @@ def xml_handle_fields(obj, keyword, value, line_annot, line_loc, verbose=False): then the not empty keyword_name is a field! This simple function will define a new node of xml tree """ - # List of possible attributes of xml elements - allowed_attr = [ - "name", - "type", - "nameType", - "unit", - "minOccurs", - "long_name", - "axis", - "signal", - "deprecated", - "axes", - "exists", - "data_offset", - "interpretation", - "maxOccurs", - "primary", - "recommended", - "optional", - "stride", - ] xml_handle_comment(obj, line_annot, line_loc) l_bracket = -1 @@ -959,8 +919,8 @@ def xml_handle_fields(obj, keyword, value, line_annot, line_loc, verbose=False): elif attr == "unit": xml_handle_units(elemt_obj, vval) xml_handle_comment(obj, line_number, line_loc, elemt_obj) - elif attr in allowed_attr and not isinstance(vval, dict) and vval: - validate_field_attribute_and_value(attr, vval, allowed_attr, value) + elif attr in YAML_FIELD_ATTRIBUTES and not isinstance(vval, dict) and vval: + validate_field_attribute_and_value(attr, vval, YAML_FIELD_ATTRIBUTES, value) elemt_obj.set(attr, check_for_mapping_char_other(vval)) rm_key_list.append(attr) rm_key_list.append(line_number) @@ -969,7 +929,7 @@ def xml_handle_fields(obj, keyword, value, line_annot, line_loc, verbose=False): for key in rm_key_list: del value[key] # Check for skipped attrinutes - check_for_skipped_attributes("field", value, allowed_attr, verbose) + check_for_skipped_attributes("field", value, YAML_FIELD_ATTRIBUTES, verbose) if isinstance(value, dict) and value != {}: recursive_build(elemt_obj, value, verbose) diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py b/dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py index d62bee2fc8..10e1843703 100644 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py @@ -38,6 +38,68 @@ ESCAPE_CHAR_DICT_IN_YAML = {"\t": " "} ESCAPE_CHAR_DICT_IN_XML = {val : key for key, val in ESCAPE_CHAR_DICT_IN_YAML.items()} +# Set up attributes for nxdl version +NXDL_GROUP_ATTRIBUTES = ( + "optional", + "recommended", + "name", + "type", + "maxOccurs", + "minOccurs", + "deprecated") +NXDL_FIELD_ATTRIBUTES = ( + "optional", + "recommended", + "name", + "type", + "axes", + "axis", + "data_offset", + "interpretation", + "long_name", + "maxOccurs", + "minOccurs", + "nameType", + "primary", + "signal", + "stride", + "required", + "deprecated", + "units", +) + +NXDL_ATTRIBUTES_ATTRIBUTES = ( + "name", + "type", + "recommended", + "optional", + "deprecated", +) + +NXDL_LINK_ATTRIBUTES = ("name", + "target", + "napimount") + +# Set up attributes for yaml version +YAML_GROUP_ATTRIBUTES = ( + *NXDL_GROUP_ATTRIBUTES, + "exists" + ) + +YAML_FIELD_ATTRIBUTES = ( + *NXDL_FIELD_ATTRIBUTES[0:-1], + "unit", + "exists" +) + +YAML_ATTRIBUTES_ATTRIBUTES = ( + *NXDL_ATTRIBUTES_ATTRIBUTES, + "minOccurs", + "maxOccurs", + "exists", +) + +YAML_LINK_ATTRIBUTES = NXDL_LINK_ATTRIBUTES def remove_namespace_from_tag(tag): """Helper function to remove the namespace from an XML tag.""" From 9c49d8a12c22f17f4abc587e915a985d0d5548df Mon Sep 17 00:00:00 2001 From: Rubel Date: Wed, 27 Sep 2023 13:22:26 +0200 Subject: [PATCH 73/93] REsolving PR comments --- dev_tools/nyaml2nxdl/README.md | 2 +- .../nyaml2nxdl/nyaml2nxdl_backward_tools.py | 164 ++++++++---------- .../nyaml2nxdl/nyaml2nxdl_forward_tools.py | 4 +- 3 files changed, 79 insertions(+), 91 deletions(-) diff --git a/dev_tools/nyaml2nxdl/README.md b/dev_tools/nyaml2nxdl/README.md index 5a48f49887..4b316fc5d0 100644 --- a/dev_tools/nyaml2nxdl/README.md +++ b/dev_tools/nyaml2nxdl/README.md @@ -51,7 +51,7 @@ Options: * **dimensions** Details which dimensional arrays to expect * **enumeration** Python list of strings which are considered as recommended entries to choose from. * **dim_parameters** `dim` which is a child of `dimension` and the `dim` might have several attributes `ref`, -`incr` including `index` and `value`. So while writting `yaml` file schema definition please following structure: +`incr` including `index` and `value`. So while writing `yaml` file schema definition please following structure: ``` dimensions: rank: integer value diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py b/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py index cdec34f661..0c3e1dd375 100755 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -"""This file collects the function used in the reverse tool nxdl2yaml. - +"""This file collects the functions for conversion from nxdl.xml to yaml version. """ import os @@ -22,7 +21,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import sys + import xml.etree.ElementTree as ET from typing import Dict from typing import List @@ -31,7 +30,7 @@ from .nyaml2nxdl_helper import get_node_parent_info from .nyaml2nxdl_helper import get_yaml_escape_char_dict from .nyaml2nxdl_helper import remove_namespace_from_tag -from .nyaml2nxdl_helper import (NXDL_FIELD_ATTRIBUTES, +from .nyaml2nxdl_helper import (NXDL_FIELD_ATTRIBUTES, NXDL_GROUP_ATTRIBUTES, NXDL_ATTRIBUTES_ATTRIBUTES, NXDL_LINK_ATTRIBUTES) @@ -44,10 +43,12 @@ CMNT_END = "-->" DEFINITION_CATEGORIES = ("category: application", "category: base") + def separate_pi_comments(input_file): - """Separate PI comments from ProcessesInstruction (PI) + """Separate PI comments from ProcessesInstruction (PI). - Separeate the comments that comes immediately after XML process instruction part, + ProcessesInstruction refers xml element process and version + Separate the comments that comes immediately after XML process instruction part, i.e. copyright comment part. """ comments_list = [] @@ -58,8 +59,6 @@ def separate_pi_comments(input_file): lines = file.readlines() has_pi = True for line in lines: - # c_start = "" def_tag = "Text liness + # Text lines # text continues - # So no indentaion at the staring or doc. So doc group will come along general + # So no indentation at the start of doc. So doc group will come along general # alignment if first_line_indent_n == 0: first_line_indent_n = yaml_indent_n - # for indent_diff -ve all lines will move left by the same ammout - # for indect_diff +ve all lines will move right the same amount + # for indent_diff -ve all lines will move left by the same amount + # for indect_diff +ve all lines will move right by the same amount indent_diff = yaml_indent_n - first_line_indent_n # CHeck for first line empty if not keep first line empty for line in text: line_indent_n = 0 - # count first empty spaces without alphabate + # count first empty spaces without alphabet line_indent_n = len(line) - len(line.lstrip()) line_indent_n = line_indent_n + indent_diff @@ -368,22 +364,19 @@ def write_out(self, indent, text, file_out): file_out.write(line_string) def print_root_level_doc(self, file_out): - """ - Print at the root level of YML file \ - the general documentation field found in XML file + """Print at the root level of YML file the general documentation field found in XML file """ indent = 0 * DEPTH_SIZE text = self.root_level_comment.get("root_doc") if text: - text = self.root_level_comment["root_doc"] self.write_out(indent, text, file_out) text = self.root_level_doc self.write_out(indent, text, file_out) self.root_level_doc = "" - def comvert_to_ymal_comment(self, indent, text): + def convert_to_ymal_comment(self, indent, text): """ Convert into yaml comment by adding exta '#' char in front of comment lines """ @@ -405,26 +398,23 @@ def print_root_level_info(self, depth, file_out): the information stored as definition attributes in the XML file """ if depth < 0: - raise ValueError("Somthing wrong with indentaion in root level.") + raise ValueError("Somthing wrong with indentation in root level.") - has_categoty = False + has_category = False for def_line in self.root_level_definition: if def_line in DEFINITION_CATEGORIES: self.write_out(indent=0 * DEPTH_SIZE, text=def_line, file_out=file_out) # file_out.write(f"{def_line}\n") - has_categoty = True + has_category = True - if not has_categoty: + if not has_category: raise ValueError( "Definition dose not get any category from 'base or application'." ) self.print_root_level_doc(file_out) - if ( - "symbols" in self.root_level_comment - and self.root_level_comment["symbols"] != "" - ): + text = self.root_level_comment.get("symbols", "") + if text: indent = depth * DEPTH_SIZE - text = self.root_level_comment["symbols"] self.write_out(indent, text, file_out) if self.root_level_symbols: self.write_out( @@ -432,7 +422,7 @@ def print_root_level_info(self, depth, file_out): ) # symbol_list include 'symbols doc', and all 'symbol' for ind, symbol in enumerate(self.symbol_list): - # Taking care of comments that come on to of 'symbols doc' and 'symbol' + # Taking care of comments that come on top of 'symbols doc' and 'symbol' if ( "symbol_comments" in self.root_level_comment and self.root_level_comment["symbol_comments"][ind] != "" @@ -460,10 +450,10 @@ def print_root_level_info(self, depth, file_out): # The first comment is top level copyright doc string for comment in self.pi_comments[1:]: self.write_out( - indent, self.comvert_to_ymal_comment(indent, comment), file_out + indent, self.convert_to_ymal_comment(indent, comment), file_out ) if self.root_level_definition: - # Soring NXname for writing end of the definition attributes + # Store NXname for writing at end of definition attributes nx_name = "" for defs in self.root_level_definition: if "NX" in defs and defs[-1] == ":": @@ -517,8 +507,8 @@ def handle_group_or_field(self, depth, node, file_out): rm_key_list.append(key) if not name_type: raise ValueError( - f"No 'name' or 'type' hase been found. But, 'group' or 'field' " - f"must have at list a nme.We got attributes: {node_attr}" + f"No 'name' or 'type' has been found. But, 'group' or 'field' " + f"must have at least a name.We have attributes: {node_attr}" ) indent = depth * DEPTH_SIZE file_out.write(f"{indent}{name_type}:\n") @@ -526,7 +516,7 @@ def handle_group_or_field(self, depth, node, file_out): for key in rm_key_list: del node_attr[key] - # tmp_dict intended to persevere order of attribnutes + # tmp_dict intended to perserve order of attributes tmp_dict = {} exists_dict = {} for key, val in node_attr.items(): @@ -551,7 +541,7 @@ def handle_group_or_field(self, depth, node, file_out): depth_ = depth + 1 for key, val in tmp_dict.items(): - # Increase depth size inside handle_map...() for writting text with one + # Increase depth size inside handle_map...() for writing text with one # more indentation. file_out.write( f"{depth_ * DEPTH_SIZE}{key}: " @@ -564,7 +554,7 @@ def check_for_unwanted_attributes(self, node, allowed_attributes_li=None, tag=No if node_tag == "field": for key in node.attrib.keys(): if key not in NXDL_FIELD_ATTRIBUTES: - raise ValueError(f"Field has got an unwanted attribute {key}." + raise ValueError(f"Field has an unwanted attribute {key}." f"NeXus field allows attributes from {NXDL_FIELD_ATTRIBUTES}") elif node_tag == "group": for key in node.attrib.keys(): @@ -579,10 +569,10 @@ def check_for_unwanted_attributes(self, node, allowed_attributes_li=None, tag=No # pylint: disable=too-many-branches, too-many-locals def handle_dimension(self, depth, node, file_out): - """ - Handle the dimension field. + """Handle the dimension field. + NOTE: Usually we take care of any xml element in xmlparse(...) and - recursion_in_xml_tree(...) functions. But Here it is a bit different. The doc dimension + recursion_in_xml_tree(...) functions. But here it is a bit different: the doc dimension and attributes of dim has been handled inside this function here. """ possible_dim_attrs = ["ref", "required", "incr", "refindex"] @@ -602,7 +592,7 @@ def handle_dimension(self, depth, node, file_out): f"Current the allowd atributes are {possible_dimemsion_attrs}." f" Please have a look" ) - # taking carew of dimension doc + # taking care of dimension doc for child in list(node): tag = remove_namespace_from_tag(child.tag) if tag == "doc": @@ -613,7 +603,7 @@ def handle_dimension(self, depth, node, file_out): dim_index_value = "" dim_other_parts = {} dim_cmnt_node = [] - # taking care of dim and doc childs of dimension + # taking care of dim and doc children of dimension for child in list(node): tag = remove_namespace_from_tag(child.tag) child_attrs = child.attrib @@ -623,10 +613,8 @@ def handle_dimension(self, depth, node, file_out): index = child_attrs.get("index", "") value = child_attrs.get("value", "") dim_index_value = f"{dim_index_value}[{index}, {value}], " - if "index" in child_attrs: - del child_attrs["index"] - if "value" in child_attrs: - del child_attrs["value"] + child_attrs.pop("index", None) + child_attrs.pop("value", None) # Taking care of doc comes as child of dim for cchild in list(child): @@ -664,7 +652,7 @@ def handle_dimension(self, depth, node, file_out): if dim_other_parts: indent = (depth + 1) * DEPTH_SIZE file_out.write(f"{indent}dim_parameters:\n") - # depth = depth + 2 dim_paramerter has child such as doc of dim + # depth = depth + 2 dim_parameter has child such as doc of dim indent = (depth + 2) * DEPTH_SIZE for key, value in dim_other_parts.items(): if key == "doc": @@ -672,7 +660,7 @@ def handle_dimension(self, depth, node, file_out): depth + 2, str(value), key, file_out ) else: - # Increase depth size inside handle_map...() for writting text with one + # Increase depth size inside handle_map...() for writing text with one # more indentation. file_out.write( f"{indent}{key}: " @@ -692,7 +680,8 @@ def handle_enumeration(self, depth, node, file_out): """ check_doc = [] - for child in list(node): + node_children = list(node) + for child in node_children: if list(child): check_doc.append(list(child)) # pylint: disable=too-many-nested-blocks @@ -700,15 +689,16 @@ def handle_enumeration(self, depth, node, file_out): indent = depth * DEPTH_SIZE tag = remove_namespace_from_tag(node.tag) file_out.write(f"{indent}{tag}: \n") - for child in list(node): + for child in node_children: tag = remove_namespace_from_tag(child.tag) itm_depth = depth + 1 if tag == "item": indent = itm_depth * DEPTH_SIZE value = child.attrib["value"] file_out.write(f"{indent}{value}: \n") - if list(child): - for item_doc in list(child): + child_children = list(child) + if child_children: + for item_doc in child_children: if remove_namespace_from_tag(item_doc.tag) == "doc": item_doc_depth = itm_depth + 1 self.handle_not_root_level_doc( @@ -725,13 +715,13 @@ def handle_enumeration(self, depth, node, file_out): if tag == CMNT_TAG and self.include_comment: self.handle_comment(itm_depth + 1, child, file_out) else: - enum_list = "" + enum_list = [] remove_nodes = [] - for item_child in list(node): + for item_child in node_children: tag = remove_namespace_from_tag(item_child.tag) if tag == "item": value = item_child.attrib["value"] - enum_list = f"{enum_list}{value}, " + enum_list.append(value) if tag == CMNT_TAG and self.include_comment: self.handle_comment(depth, item_child, file_out) remove_nodes.append(item_child) @@ -740,7 +730,7 @@ def handle_enumeration(self, depth, node, file_out): indent = depth * DEPTH_SIZE tag = remove_namespace_from_tag(node.tag) - enum_list = enum_list[:-2] or "" + enum_list = ', '.join(enum_list) file_out.write(f"{indent}{tag}: [{enum_list}]\n") def handle_attributes(self, depth, node, file_out): @@ -805,9 +795,7 @@ def handle_attributes(self, depth, node, file_out): ) def handle_link(self, depth, node, file_out): - """ - Handle link elements of nxdl - """ + """Handle link elements of nxdl""" node_attr = node.attrib # Handle special cases @@ -826,7 +814,7 @@ def handle_link(self, depth, node, file_out): else: raise ValueError( f"An unexpected attribute '{attr_key}' of link has found." - f"At this moment the alloed keys are {NXDL_LINK_ATTRIBUTES}" + f"At this moment the allowed keys are {NXDL_LINK_ATTRIBUTES}" ) def handle_choice(self, depth, node, file_out): @@ -844,7 +832,7 @@ def handle_choice(self, depth, node, file_out): del node_attr["name"] depth_ = depth + 1 - # Take care of attributes of choice element, attrinutes may come in future. + # Take care of attributes of choice element, attributes may come in future. for attr in node_attr.items(): if attr in self.choice_allowed_attr: indent = depth_ * DEPTH_SIZE @@ -853,7 +841,7 @@ def handle_choice(self, depth, node, file_out): else: raise ValueError( f"An unexpected attribute '{attr}' of 'choice' has been found." - f"At this moment attributes for choice {self.choice_allowed_attr}" + f"At this moment allowed attributes for choice {self.choice_allowed_attr}" ) def handle_comment(self, depth, node, file_out): @@ -861,7 +849,7 @@ def handle_comment(self, depth, node, file_out): Collect comment element and pass to write_out function """ indent = depth * DEPTH_SIZE - text = self.comvert_to_ymal_comment(indent, node.text) + text = self.convert_to_ymal_comment(indent, node.text) self.write_out(indent, text, file_out) @@ -899,7 +887,7 @@ def xmlparse(self, output_yml, xml_tree, depth, verbose): for child in list(node): tag_tmp = remove_namespace_from_tag(child.tag) if tag_tmp == CMNT_TAG and self.include_comment: - last_comment = self.comvert_to_ymal_comment( + last_comment = self.convert_to_ymal_comment( depth * DEPTH_SIZE, child.text ) remove_cmnt_n = child diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py b/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py index b6e1115627..8b9c9f3c4f 100644 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py @@ -259,8 +259,8 @@ def check_for_mapping_char_other(text): text = "true" if text == "False": text = "false" - # Some escape char is not valid in yaml libray which is written while writting - # yaml file. In the time of writting nxdl revert to that escape char. + # Some escape char is not valid in yaml libray which is written while writing + # yaml file. In the time of writing nxdl revert to that escape char. escape_reverter = get_yaml_escape_char_reverter_dict() for key, val in escape_reverter.items(): if key in text: From 1fa5665105371b03616f47f7241259b687a28314 Mon Sep 17 00:00:00 2001 From: domna Date: Thu, 28 Sep 2023 08:17:10 +0200 Subject: [PATCH 74/93] Fixes Makefile pickung up nxdl rules for `make nyaml` --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 4e29846648..8e92bbbf54 100644 --- a/Makefile +++ b/Makefile @@ -109,8 +109,8 @@ $(APPDEF_DIR)/%.nxdl.xml : $(APPDEF_DIR)/$(NYAML_SUBDIR)/%.yaml mv $(APPDEF_DIR)/$(NYAML_SUBDIR)/$*.nxdl.xml $@ NXDLS := $(foreach dir,$(NXDL_DIRS),$(wildcard $(dir)/*.nxdl.xml)) -nyaml : $(DIRS) $(NXDLS) - for file in $^; do\ +nyaml : $(DIRS) + for file in $(NXDLS); do\ mkdir -p "$${file%/*}/nyaml";\ nyaml2nxdl --input-file $${file};\ FNAME=$${file##*/};\ From b6761e9af4733e3b9444876bfdcb65f539404117 Mon Sep 17 00:00:00 2001 From: Rubel Date: Thu, 28 Sep 2023 09:30:20 +0200 Subject: [PATCH 75/93] Updating PR comments. --- .../nyaml2nxdl/nyaml2nxdl_backward_tools.py | 34 ++-- .../nyaml2nxdl/nyaml2nxdl_forward_tools.py | 169 ++++++------------ dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py | 14 +- 3 files changed, 71 insertions(+), 146 deletions(-) diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py b/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py index 0c3e1dd375..78102e013f 100755 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py @@ -196,7 +196,7 @@ def handle_symbols(self, depth, node): for child in list(node): tag = remove_namespace_from_tag(child.tag) if tag == CMNT_TAG and self.include_comment: - last_comment = self.convert_to_ymal_comment( + last_comment = self.convert_to_yaml_comment( depth * DEPTH_SIZE, child.text ) if tag == "doc": @@ -223,7 +223,7 @@ def handle_symbols(self, depth, node): for symbol_doc in list(child): tag = remove_namespace_from_tag(symbol_doc.tag) if tag == CMNT_TAG and self.include_comment: - last_comment = self.convert_to_ymal_comment( + last_comment = self.convert_to_yaml_comment( depth * DEPTH_SIZE, symbol_doc.text ) if tag == "doc": @@ -310,7 +310,7 @@ def handle_not_root_level_doc(self, depth, text, tag="doc", file_out=None): if len(line.lstrip()) != 0: first_line_indent_n = len(line) - len(line.lstrip()) break - # Taking care of doc like bellow: + # Taking care of doc like below: # Text lines # text continues # So no indentation at the start of doc. So doc group will come along general @@ -376,7 +376,7 @@ def print_root_level_doc(self, file_out): self.write_out(indent, text, file_out) self.root_level_doc = "" - def convert_to_ymal_comment(self, indent, text): + def convert_to_yaml_comment(self, indent, text): """ Convert into yaml comment by adding exta '#' char in front of comment lines """ @@ -404,7 +404,6 @@ def print_root_level_info(self, depth, file_out): for def_line in self.root_level_definition: if def_line in DEFINITION_CATEGORIES: self.write_out(indent=0 * DEPTH_SIZE, text=def_line, file_out=file_out) - # file_out.write(f"{def_line}\n") has_category = True if not has_category: @@ -450,7 +449,7 @@ def print_root_level_info(self, depth, file_out): # The first comment is top level copyright doc string for comment in self.pi_comments[1:]: self.write_out( - indent, self.convert_to_ymal_comment(indent, comment), file_out + indent, self.convert_to_yaml_comment(indent, comment), file_out ) if self.root_level_definition: # Store NXname for writing at end of definition attributes @@ -516,7 +515,7 @@ def handle_group_or_field(self, depth, node, file_out): for key in rm_key_list: del node_attr[key] - # tmp_dict intended to perserve order of attributes + # tmp_dict intended to preserve order of attributes tmp_dict = {} exists_dict = {} for key, val in node_attr.items(): @@ -559,12 +558,12 @@ def check_for_unwanted_attributes(self, node, allowed_attributes_li=None, tag=No elif node_tag == "group": for key in node.attrib.keys(): if key not in NXDL_GROUP_ATTRIBUTES: - raise ValueError(f"Attribute has got an unwanted attribute {key}." + raise ValueError(f"Attribute has an unwanted attribute {key}." f"NeXus attribute allows attributes from {NXDL_GROUP_ATTRIBUTES}") elif node_tag == tag: for key in node.attrib.keys(): if key not in allowed_attributes_li: - raise ValueError(f"{tag.capitalized()} has got an unwanted attribute {key}." + raise ValueError(f"{tag.capitalized()} has an unwanted attribute {key}." f"NeXus {tag.capitalized()} allows attributes from {allowed_attributes_li}") # pylint: disable=too-many-branches, too-many-locals @@ -588,7 +587,7 @@ def handle_dimension(self, depth, node, file_out): file_out.write(f"{indent}{attr}: {value}\n") else: raise ValueError( - f"Dimension has got an attribute {attr} that is not valid." + f"Dimension has an attribute {attr} that is not valid." f"Current the allowd atributes are {possible_dimemsion_attrs}." f" Please have a look" ) @@ -610,11 +609,9 @@ def handle_dimension(self, depth, node, file_out): # taking care of index and value attributes if tag == "dim": # taking care of index and value in format [[index, value]] - index = child_attrs.get("index", "") - value = child_attrs.get("value", "") + index = child_attrs.pop("index", "") + value = child_attrs.pop("value", "") dim_index_value = f"{dim_index_value}[{index}, {value}], " - child_attrs.pop("index", None) - child_attrs.pop("value", None) # Taking care of doc comes as child of dim for cchild in list(child): @@ -679,11 +676,12 @@ def handle_enumeration(self, depth, node, file_out): """ - check_doc = [] + check_doc = False node_children = list(node) for child in node_children: if list(child): - check_doc.append(list(child)) + check_doc = True + break # pylint: disable=too-many-nested-blocks if check_doc: indent = depth * DEPTH_SIZE @@ -849,7 +847,7 @@ def handle_comment(self, depth, node, file_out): Collect comment element and pass to write_out function """ indent = depth * DEPTH_SIZE - text = self.convert_to_ymal_comment(indent, node.text) + text = self.convert_to_yaml_comment(indent, node.text) self.write_out(indent, text, file_out) @@ -887,7 +885,7 @@ def xmlparse(self, output_yml, xml_tree, depth, verbose): for child in list(node): tag_tmp = remove_namespace_from_tag(child.tag) if tag_tmp == CMNT_TAG and self.include_comment: - last_comment = self.convert_to_ymal_comment( + last_comment = self.convert_to_yaml_comment( depth * DEPTH_SIZE, child.text ) remove_cmnt_n = child diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py b/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py index 8b9c9f3c4f..896161e2fc 100644 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py @@ -22,7 +22,6 @@ # import os -import sys import textwrap import datetime import xml.etree.ElementTree as ET @@ -307,7 +306,7 @@ def xml_handle_exists(dct, obj, keyword, value): obj.set("maxOccurs", str(value[3])) else: obj.set("maxOccurs", "unbounded") - elif value[0] != "min" or value[2] != "max": + else: raise ValueError( f"Line {dct[line_number]}: exists keyword" f"needs to go either with an optional [recommended] list with two " @@ -338,74 +337,6 @@ def xml_handle_exists(dct, obj, keyword, value): obj.set("minOccurs", "0") -# pylint: disable=too-many-branches, too-many-locals, too-many-statements -def xml_handle_group(dct, obj, keyword, value, verbose=False): - """ - The function deals with group instances - """ - line_number = f"__line__{keyword}" - line_loc = dct[line_number] - xml_handle_comment(obj, line_number, line_loc) - - l_bracket = -1 - r_bracket = -1 - if keyword.count("(") == 1: - l_bracket = keyword.index("(") - if keyword.count(")") == 1: - r_bracket = keyword.index(")") - - keyword_name, keyword_type = nx_name_type_resolving(keyword) - if not keyword_type: - # if not keyword_name and not keyword_type: - raise ValueError(f"A group must have a type class from base, application" - f" or contributed class. Check around line: {line_number}.") - grp = ET.SubElement(obj, "group") - - if l_bracket == 0 and r_bracket > 0: - grp.set("type", keyword_type) - if keyword_name: - grp.set("name", keyword_name) - elif l_bracket > 0: - grp.set("name", keyword_name) - if keyword_type: - grp.set("type", keyword_type) - else: - grp.set("name", keyword_name) - - if value: - rm_key_list = [] - for attr, vval in value.items(): - if "__line__" in attr: - continue - line_number = f"__line__{attr}" - line_loc = value[line_number] - if attr == "doc": - xml_handle_doc(grp, vval, line_number, line_loc) - rm_key_list.append(attr) - rm_key_list.append(line_number) - elif attr == "exists" and vval: - xml_handle_exists(value, grp, attr, vval) - rm_key_list.append(attr) - rm_key_list.append(line_number) - xml_handle_comment(obj, line_number, line_loc, grp) - elif attr == "unit": - xml_handle_units(grp, vval) - xml_handle_comment(obj, line_number, line_loc, grp) - elif attr in YAML_GROUP_ATTRIBUTES and not isinstance(vval, dict) and vval: - validate_field_attribute_and_value(attr, vval, YAML_GROUP_ATTRIBUTES, value) - grp.set(attr, check_for_mapping_char_other(vval)) - rm_key_list.append(attr) - rm_key_list.append(line_number) - xml_handle_comment(obj, line_number, line_loc, grp) - - for key in rm_key_list: - del value[key] - # Check for skipped attrinutes - check_for_skipped_attributes("group", value, YAML_GROUP_ATTRIBUTES, verbose) - if isinstance(value, dict) and len(value) != 0: - recursive_build(grp, value, verbose) - - def xml_handle_dimensions(dct, obj, keyword, value: dict): """ This function creates a 'dimensions' element instance, and appends it to an existing element @@ -574,7 +505,7 @@ def xml_handle_dim_from_dimension_dict( else: raise ValueError( f"Got unexpected block except 'dim' and 'dim_parameters'." - f"Please check arround line {line_number}" + f"Please check around line {line_number}" ) for key in rm_key_list: @@ -648,7 +579,7 @@ def xml_handle_link(dct, obj, keyword, value, verbose): for key in rm_key_list: del value[key] - # Check for skipped attrinutes + # Check for skipped attributes check_for_skipped_attributes("link", value, YAML_LINK_ATTRIBUTES, verbose) if isinstance(value, dict) and value != {}: @@ -662,7 +593,7 @@ def xml_handle_choice(dct, obj, keyword, value, verbose=False): line_number = f"__line__{keyword}" line_loc = dct[line_number] xml_handle_comment(obj, line_number, line_loc) - # Add to this tuple if new attributs have been added to nexus definition. + # Add to this tuple if new attributes have been added to nexus definition. possible_attr = () choice_obj = ET.SubElement(obj, "choice") # take care of special attributes @@ -689,7 +620,7 @@ def xml_handle_choice(dct, obj, keyword, value, verbose=False): for key in rm_key_list: del value[key] - # Check for skipped attrinutes + # Check for skipped attributes check_for_skipped_attributes("choice", value, possible_attr, verbose) if isinstance(value, dict) and value != {}: @@ -724,11 +655,9 @@ def xml_handle_symbols(dct, obj, keyword, value: dict): ), f"Line {line_loc}: put a comment in doc string !" sym = ET.SubElement(syms, "symbol") sym.set("name", kkeyword) - # sym_doc = ET.SubElement(sym, 'doc') xml_handle_doc(sym, vvalue) rm_key_list.append(kkeyword) rm_key_list.append(line_number) - # sym_doc.text = '\n' + textwrap.fill(vvalue, width=70) + '\n' for key in rm_key_list: del value[key] @@ -750,7 +679,7 @@ def check_keyword_variable(verbose, dct, keyword, value): def helper_keyword_type(kkeyword_type): """ - This function is returning a value of keyword_type if it belong to NX_TYPE_KEYS + Return a value of keyword_type if it belong to NX_TYPE_KEYS """ if kkeyword_type in NX_TYPE_KEYS: return kkeyword_type @@ -816,7 +745,7 @@ def xml_handle_attributes(dct, obj, keyword, value, verbose): for key in rm_key_list: del value[key] - # Check cor skiped attribute + # Check cor skipped attribute check_for_skipped_attributes("Attribute", value, YAML_ATTRIBUTES_ATTRIBUTES, verbose) if value: recursive_build(elemt_obj, value, verbose) @@ -828,12 +757,11 @@ def validate_field_attribute_and_value(v_attr, vval, allowed_attribute, value): and invalid value. """ - # check for empty val if not isinstance(vval, dict) and not str(vval): # check for empty value line_number = f"__line__{v_attr}" raise ValueError( f"In a field a valid attrbute ('{v_attr}') found that is not stored." - f" Please check arround line {value[line_number]}" + f" Please check around line {value[line_number]}" ) # The below elements might come as child element @@ -849,25 +777,15 @@ def validate_field_attribute_and_value(v_attr, vval, allowed_attribute, value): line_number = f"__line__{v_attr}" raise ValueError( f"In a field or group a invalid attribute ('{v_attr}') or child has found." - f" Please check arround line {value[line_number]}." + f" Please check around line {value[line_number]}." ) -def xml_handle_fields(obj, keyword, value, line_annot, line_loc, verbose=False): - """ - Handle a field in yaml file. - When a keyword is NOT: - symbol, - NX baseclass member, - attribute (\\@), - doc, - enumerations, - dimension, - exists, - then the not empty keyword_name is a field! - This simple function will define a new node of xml tree +def xml_handle_fields_or_group(dct, obj, keyword, value, ele_type, allowed_attr, verbose=False): + """Handle a field or group in yaml file. """ - + line_annot = f"__line__{keyword}" + line_loc = dct[line_annot] xml_handle_comment(obj, line_annot, line_loc) l_bracket = -1 r_bracket = -1 @@ -877,9 +795,14 @@ def xml_handle_fields(obj, keyword, value, line_annot, line_loc, verbose=False): r_bracket = keyword.index(")") keyword_name, keyword_type = nx_name_type_resolving(keyword) - if not keyword_type and not keyword_name: - raise ValueError("Check for name or type in field.") - elemt_obj = ET.SubElement(obj, "field") + if ele_type == 'field' and not keyword_name: + raise ValueError(f"No name for NeXus {ele_type} has been found." + f"Check around line:{line_loc}") + elif not keyword_type and not keyword_name: + raise ValueError(f"No name or type for NeXus {ele_type} has been found." + f"Check around line: {line_loc}") + + elemt_obj = ET.SubElement(obj, ele_type) # type come first if l_bracket == 0 and r_bracket > 0: @@ -916,11 +839,11 @@ def xml_handle_fields(obj, keyword, value, line_annot, line_loc, verbose=False): rm_key_list.append(attr) rm_key_list.append(line_number) xml_handle_comment(obj, line_number, line_loc, elemt_obj) - elif attr == "unit": + elif ele_type == "field" and attr == "unit": xml_handle_units(elemt_obj, vval) xml_handle_comment(obj, line_number, line_loc, elemt_obj) - elif attr in YAML_FIELD_ATTRIBUTES and not isinstance(vval, dict) and vval: - validate_field_attribute_and_value(attr, vval, YAML_FIELD_ATTRIBUTES, value) + elif attr in allowed_attr and not isinstance(vval, dict) and vval: + validate_field_attribute_and_value(attr, vval, allowed_attr, value) elemt_obj.set(attr, check_for_mapping_char_other(vval)) rm_key_list.append(attr) rm_key_list.append(line_number) @@ -928,8 +851,8 @@ def xml_handle_fields(obj, keyword, value, line_annot, line_loc, verbose=False): for key in rm_key_list: del value[key] - # Check for skipped attrinutes - check_for_skipped_attributes("field", value, YAML_FIELD_ATTRIBUTES, verbose) + # Check for skipped attributes + check_for_skipped_attributes(ele_type, value, allowed_attr, verbose) if isinstance(value, dict) and value != {}: recursive_build(elemt_obj, value, verbose) @@ -942,12 +865,14 @@ def xml_handle_comment( xml_ele: ET.Element = None, is_def_cmnt: bool = False, ): - """ - Add xml comment: check for comments that has the same 'line_annotation' - (e.g. __line__data) and the same line_loc_no (e.g. 30). After that, it + """Handle comment. + + Add xml comment: check for comments searched by 'line_annotation' + (e.g. __line__data) and line_loc_no (e.g. 30). It does following tasks: - 1. Returns list of comments texts (multiple members if element has multiple comments) - or [] if no intended comments are found + + 1. Returns list of comments texts, if comment is intended for definition element. + other return always []. 2. Rearrange comment elements of xml_ele and xml_ele where comment comes first. 3. Append comment element when no xml_ele found as general comments. """ @@ -957,7 +882,7 @@ def xml_handle_comment( cmnt = COMMENT_BLOCKS.get_comment_by_line_info(line_info) # noqa: F821 cmnt_text = cmnt.get_comment_text_list() - # Check comment for definition element + # Check comment for definition element and return if is_def_cmnt: return cmnt_text if xml_ele is not None: @@ -969,6 +894,8 @@ def xml_handle_comment( elif not is_def_cmnt and xml_ele is None: for string in cmnt_text: obj.append(ET.Comment(string)) + + # The searched comment is not related with definition element return [] @@ -992,7 +919,7 @@ def recursive_build(obj, dct, verbose): keyword_name, keyword_type = nx_name_type_resolving(keyword) check_keyword_variable(verbose, dct, keyword, value) if verbose: - sys.stdout.write( + print( f"keyword_name:{keyword_name} keyword_type {keyword_type}\n" ) @@ -1000,7 +927,7 @@ def recursive_build(obj, dct, verbose): xml_handle_link(dct, obj, keyword, value, verbose) elif keyword[-8:] == "(choice)": xml_handle_choice(dct, obj, keyword, value) - # The bellow xml_symbol clause is for the symbols that come ubde filed or attributes + # The below xml_symbol clause is for the symbols that come ubde filed or attributes # Root level symbols has been inside nyaml2nxdl() elif keyword_type == "" and keyword_name == "symbols": xml_handle_symbols(dct, obj, keyword, value) @@ -1008,8 +935,10 @@ def recursive_build(obj, dct, verbose): elif (keyword_type in NX_CLSS) or ( keyword_type not in [*NX_TYPE_KEYS, "", *NX_NEW_DEFINED_CLASSES] ): + elem_type = 'group' # we can be sure we need to instantiate a new group - xml_handle_group(dct, obj, keyword, value, verbose) + xml_handle_fields_or_group(dct, obj, keyword, value, elem_type, + YAML_GROUP_ATTRIBUTES, verbose=False) elif keyword_name[0:2] == NX_ATTR_IDNT: # check if obj qualifies xml_handle_attributes(dct, obj, keyword, value, verbose) @@ -1025,11 +954,13 @@ def recursive_build(obj, dct, verbose): xml_handle_exists(dct, obj, keyword, value) # Handles fileds e.g. AXISNAME elif keyword_name != "" and "__line__" not in keyword_name: - xml_handle_fields(obj, keyword, value, line_number, line_loc, verbose) + elem_type = 'field' + xml_handle_fields_or_group(dct, obj, keyword, value, elem_type, + YAML_FIELD_ATTRIBUTES, verbose=False) else: raise ValueError( f"An unknown type of element {keyword} has been found which is " - f"not be able to be resolved. Chekc around line {dct[line_number]}" + f"not be able to be resolved. Check around line {dct[line_number]}" ) @@ -1094,11 +1025,11 @@ def nyaml2nxdl(input_file: str, out_file, verbose: bool): yml_appdef = yml_reader(input_file) def_cmnt_text = [] if verbose: - sys.stdout.write(f"input-file: {input_file}\n") - sys.stdout.write( + print(f"input-file: {input_file}\n") + print( "application/base contains the following root-level entries:\n" ) - sys.stdout.write(str(yml_appdef.keys())) + print(str(yml_appdef.keys())) xml_root = ET.Element("definition", {}) assert ( "category" in yml_appdef.keys() @@ -1126,7 +1057,7 @@ def nyaml2nxdl(input_file: str, out_file, verbose: bool): del yml_appdef[line_number] del yml_appdef[kkey] - # Taking care or name and extends + # Taking care of name and extends elif "NX" in kkey: # Taking the attribute order but the correct value will be stored later # check for name first or type first if (NXobject)NXname then type first @@ -1210,4 +1141,4 @@ def nyaml2nxdl(input_file: str, out_file, verbose: bool): check_for_default_attribute_and_value(xml_root) pretty_print_xml(xml_root, out_file, def_cmnt_text) if verbose: - sys.stdout.write("Parsed YAML to NXDL successfully\n") + print("Parsed YAML to NXDL successfully\n") diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py b/dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py index 10e1843703..87ab12973b 100644 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py @@ -23,10 +23,6 @@ # limitations under the License. # - -# Yaml library does not except the keys (escape char "\t" and yaml separator ":") -# So the corresponding value is to skip them and -# and also careful about this order import hashlib from yaml.composer import Composer @@ -35,6 +31,7 @@ from yaml.nodes import ScalarNode from yaml.resolver import BaseResolver +# Yaml library does not except the keys (escape char "\t" and yaml separator ":") ESCAPE_CHAR_DICT_IN_YAML = {"\t": " "} ESCAPE_CHAR_DICT_IN_XML = {val : key for key, val in ESCAPE_CHAR_DICT_IN_YAML.items()} @@ -168,7 +165,7 @@ def get_node_parent_info(tree, node): index = index of grand child node of tree """ - # map from grand child to child parent of tree + # map from grand child to parent which is child tree parent_map = {c: p for p in tree.iter() for c in p} parent = parent_map[node] return parent, list(parent).index(node) @@ -188,7 +185,7 @@ def clean_empty_lines(line_list): break # Find the index of the last non-empty line - for ind, line in enumerate(line_list[::-1]): + for ind, line in enumerate(reversed(line_list)): if not line.isspace(): ends_non_empty_line = -ind break @@ -202,7 +199,7 @@ def clean_empty_lines(line_list): def nx_name_type_resolving(tmp): """Separate name and NeXus type - Extracts the eventually custom name {optional_string} + Extracts the eventual custom name {optional_string} and type {nexus_type} from a YML section string. YML section string syntax: optional_string(nexus_type) """ @@ -250,8 +247,7 @@ def extend_yamlfile_by_nxdl_as_comment( f1_obj.write(line) with open(file_to_be_appended, mode="r", encoding="utf-8") as f2_obj: - lines = f2_obj.readlines() - for line in lines: + for line in f2_obj: f1_obj.write(f"# {line}") From a0f6a3f89706a94e8908d1041bdfd1eb04ec8a9e Mon Sep 17 00:00:00 2001 From: Rubel Date: Thu, 28 Sep 2023 15:54:49 +0200 Subject: [PATCH 76/93] updating PR comments. --- dev_tools/nyaml2nxdl/nyaml2nxdl.py | 10 +-- .../nyaml2nxdl/nyaml2nxdl_backward_tools.py | 7 +- .../nyaml2nxdl/nyaml2nxdl_forward_tools.py | 15 +++-- dev_tools/utils/nxdl_utils.py | 64 ++++++------------- 4 files changed, 36 insertions(+), 60 deletions(-) diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl.py b/dev_tools/nyaml2nxdl/nyaml2nxdl.py index 52a60b79e3..9449d4487b 100755 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl.py @@ -46,7 +46,7 @@ def generate_nxdl_or_retrieve_nxdl(yaml_file, out_xml_file, verbose): """ Generate yaml, nxdl and hash. - If the extracted hash is exactly the same as producd from generated yaml then + If the extracted hash is exactly the same as produced from generated yaml then retrieve the nxdl part from provided yaml. Else, generate nxdl from separated yaml with the help of nyaml2nxdl function """ @@ -58,12 +58,10 @@ def generate_nxdl_or_retrieve_nxdl(yaml_file, out_xml_file, verbose): gen_hash = get_sha256_hash(sep_yaml) if hash_found == gen_hash: Path(sep_yaml).unlink() - # os.remove(sep_yaml) return nyaml2nxdl(sep_yaml, out_xml_file, verbose) Path(sep_yaml).unlink() - # os.remove(sep_yaml) def split_name_and_extension(file_path): @@ -92,9 +90,8 @@ def split_name_and_extension(file_path): is_flag=True, default=False, help=( - "Check if yaml or nxdl has followed general rules of whether schema or not" - "check whether your comment in the right place or not. The option render an " - "output file of the same extension(*_consistency.yaml or *_consistency.nxdl.xml)" + "Check if yaml and nxdl can be convertered from one to another version recursively and" + " get the same version of file. E.g. from NXexample.nxdl.xml to NXexample_consistency.nxdl.xml." ), ) @click.option( @@ -133,7 +130,6 @@ def launch_tool(input_file, verbose, do_not_store_nxdl, check_consistency): converter = Nxdl2yaml([], []) converter.print_yml(xml_out_file, yaml_out_file, verbose) Path(xml_out_file).unlink() - # os.remove(xml_out_file) elif ext == "nxdl.xml": # if not append: yaml_out_file = raw_name + "_parsed" + ".yaml" diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py b/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py index 78102e013f..d2590e2021 100755 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 """This file collects the functions for conversion from nxdl.xml to yaml version. """ -import os # -*- coding: utf-8 -*- # @@ -25,6 +24,7 @@ import xml.etree.ElementTree as ET from typing import Dict from typing import List +from pathlib import Path from .nyaml2nxdl_helper import clean_empty_lines from .nyaml2nxdl_helper import get_node_parent_info @@ -172,8 +172,9 @@ def __init__( def print_yml(self, input_file, output_yml, verbose): """Parse an XML file provided as input and print a YML file. """ - if os.path.isfile(output_yml): - os.remove(output_yml) + output_file_path = Path(output_yml) + if output_file_path.is_file(): + output_file_path.unlink() depth = 0 diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py b/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py index 896161e2fc..50301b3e11 100644 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py @@ -21,7 +21,7 @@ # limitations under the License. # -import os +import pathlib import textwrap import datetime import xml.etree.ElementTree as ET @@ -89,10 +89,10 @@ def check_for_dom_comment_in_yaml(): "WITHOUT ANY WARRANTY", ] - # Check for dom comments in first three comments + # Check for dom comments in first five comments dom_comment = "" dom_comment_ind = 1 - for ind, comnt in enumerate(COMMENT_BLOCKS[0:3]): + for ind, comnt in enumerate(COMMENT_BLOCKS[0:5]): cmnt_list = comnt.get_comment_text_list() if len(cmnt_list) == 1: text = cmnt_list[0] @@ -982,12 +982,12 @@ def pretty_print_xml(xml_root, output_xml, def_comments=None): for string in def_comments: def_comt_ele = dom.createComment(string) dom.insertBefore(def_comt_ele, root) - + tmp_xml = "tmp.xml" xml_string = dom.toprettyxml(indent=1 * DEPTH_SIZE, newl="\n", encoding="UTF-8") - with open("tmp.xml", "wb") as file_tmp: + with open(tmp_xml, "wb") as file_tmp: file_tmp.write(xml_string) flag = False - with open("tmp.xml", "r", encoding="utf-8") as file_out: + with open(tmp_xml, "r", encoding="utf-8") as file_out: with open(output_xml, "w", encoding="utf-8") as file_out_mod: for i in file_out.readlines(): if "" not in i and "" not in i and flag is False: @@ -1003,7 +1003,8 @@ def pretty_print_xml(xml_root, output_xml, def_comments=None): elif "" not in i and "" in i and flag is True: file_out_mod.write(white_spaces * " " + i) flag = False - os.remove("tmp.xml") + tmp_xml_path = pathlib.Path(tmp_xml) + pathlib.Path.unlink(tmp_xml_path) # pylint: disable=too-many-statements diff --git a/dev_tools/utils/nxdl_utils.py b/dev_tools/utils/nxdl_utils.py index 9252c46fe8..9d3f55c2d7 100644 --- a/dev_tools/utils/nxdl_utils.py +++ b/dev_tools/utils/nxdl_utils.py @@ -156,19 +156,17 @@ def get_nx_units(): root = get_xml_root(filepath) units_and_type_list = [] for child in root: - for i in child.attrib.values(): - units_and_type_list.append(i) + units_and_type_list.extend(child.attrib.values()) flag = False + nx_units = [] for line in units_and_type_list: if line == "anyUnitsAttr": flag = True - nx_units = [] - elif "NX" in line and flag is True: + elif "NX" in line and flag: nx_units.append(line) elif line == "primitiveType": flag = False - else: - pass + return nx_units @@ -184,7 +182,6 @@ def get_nx_attribute_type(): for line in units_and_type_list: if line == "primitiveType": flag = True - nx_types = [] elif "NX" in line and flag is True: nx_types.append(line) elif line == "anyUnitsAttr": @@ -247,10 +244,7 @@ def belongs_to_capital(params): ): continue # check if the name of another sibling fits better - name_any2 = ( - "nameType" in child2.attrib.keys() - and child2.attrib["nameType"] == "any" - ) + name_any2 = child2.attrib.get("nameType") == "any" fit2 = get_nx_namefit(chk_name, get_node_name(child2), name_any2) if fit2 > fit: return False @@ -268,17 +262,10 @@ def get_own_nxdl_child_reserved_elements(child, name, nxdl_elem): """checking reserved elements, like doc, enumeration""" local_name = get_local_name_from_xml(child) if local_name == "doc" and name == "doc": - if nxdl_elem.get("nxdlbase"): - child.set("nxdlbase", nxdl_elem.get("nxdlbase")) - child.set("nxdlbase_class", nxdl_elem.get("nxdlbase_class")) - child.set("nxdlpath", nxdl_elem.get("nxdlpath") + "/doc") - return child + return set_nxdlpath(child, nxdl_elem, tag_name=name) + if local_name == "enumeration" and name == "enumeration": - if nxdl_elem.get("nxdlbase"): - child.set("nxdlbase", nxdl_elem.get("nxdlbase")) - child.set("nxdlbase_class", nxdl_elem.get("nxdlbase_class")) - child.set("nxdlpath", nxdl_elem.get("nxdlpath") + "/enumeration") - return child + return set_nxdlpath(child, nxdl_elem, tag_name=name) return False @@ -296,12 +283,6 @@ def get_own_nxdl_child_base_types(child, class_type, nxdl_elem, name, hdf_name): if get_local_name_from_xml(child) == "attribute" and belongs_to( nxdl_elem, child, name, None, hdf_name ): - if nxdl_elem.get("nxdlbase"): - child.set("nxdlbase", nxdl_elem.get("nxdlbase")) - child.set("nxdlbase_class", nxdl_elem.get("nxdlbase_class")) - child.set( - "nxdlpath", nxdl_elem.get("nxdlpath") + "/" + get_node_name(child) - ) return set_nxdlpath(child, nxdl_elem) return False @@ -314,14 +295,8 @@ def get_own_nxdl_child( class_type - nxdl type or hdf classname (for groups, it is obligatory) hdf_name - hdf name""" for child in nxdl_elem: - if "name" in child.attrib and child.attrib["name"] == name: - if nxdl_elem.get("nxdlbase"): - child.set("nxdlbase", nxdl_elem.get("nxdlbase")) - child.set("nxdlbase_class", nxdl_elem.get("nxdlbase_class")) - child.set( - "nxdlpath", nxdl_elem.get("nxdlpath") + "/" + get_node_name(child) - ) - return child + if child.attrib.get("name") == name: + return set_nxdlpath(child, nxdl_elem) for child in nxdl_elem: if child.attrib.get("name") == name: child.set("nxdlbase", nxdl_elem.get("nxdlbase")) @@ -625,7 +600,7 @@ def get_enums(node): def add_base_classes(elist, nx_name=None, elem: ET.Element = None): """ - Add the base classes corresponding to the last elemenent in elist to the list. Note that if + Add the base classes corresponding to the last element in elist to the list. Note that if elist is empty, a nxdl file with the name of nx_name or a placeholder elem is used if provided """ if elist and nx_name is None: @@ -654,14 +629,18 @@ def add_base_classes(elist, nx_name=None, elem: ET.Element = None): add_base_classes(elist) -def set_nxdlpath(child, nxdl_elem): - """ - Setting up child nxdlbase, nxdlpath and nxdlbase_class from nxdl_element. +def set_nxdlpath(child, nxdl_elem, tag_name=None): + """Setting up child nxdlbase, nxdlpath and nxdlbase_class from nxdl_element. """ if nxdl_elem.get("nxdlbase"): child.set("nxdlbase", nxdl_elem.get("nxdlbase")) child.set("nxdlbase_class", nxdl_elem.get("nxdlbase_class")) - child.set("nxdlpath", nxdl_elem.get("nxdlpath") + "/" + get_node_name(child)) + # Handle element that does not has 'name' attr e.g. doc, enumeration + if tag_name: + child.set("nxdlpath", nxdl_elem.get("nxdlpath") + "/" + tag_name) + else: + child.set("nxdlpath", nxdl_elem.get("nxdlpath") + "/" + get_node_name(child)) + return child @@ -722,7 +701,7 @@ def get_best_nxdata_child(nxdl_elem, hdf_node, hdf_name): def get_best_child(nxdl_elem, hdf_node, hdf_name, hdf_class_name, nexus_type): """returns the child of nxdl_elem which has a name - corresponding to the the html documentation name html_name""" + corresponding to the html documentation name html_name""" bestfit = -1 bestchild = None if ( @@ -779,7 +758,7 @@ def walk_elist(elist, html_name): ): del elist[ind + 1 :] # add new base class(es) if new element brings such (and not a primitive type) - if len(elist) == ind + 1 and get_nx_class(elist[ind])[0:3] != "NX_": + if len(elist) == ind + 1 and get_nx_class(elist[ind]).startswith("NX_"): add_base_classes(elist) return elist, html_name @@ -789,7 +768,6 @@ def get_inherited_nodes( nxdl_path: str = None, # pylint: disable=too-many-arguments,too-many-locals nx_name: str = None, elem: ET.Element = None, - attr=False, ): """Returns a list of ET.Element for the given path.""" # let us start with the given definition file From 9516065947e26b35b950c7ca0cdfda4e360e5aa9 Mon Sep 17 00:00:00 2001 From: Rubel Date: Thu, 28 Sep 2023 15:58:58 +0200 Subject: [PATCH 77/93] updating PR comments. --- dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py b/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py index 50301b3e11..5d14c0264b 100644 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py @@ -888,8 +888,7 @@ def xml_handle_comment( if xml_ele is not None: obj.remove(xml_ele) for string in cmnt_text: - si_comnt = ET.Comment(string) - obj.append(si_comnt) + obj.append(ET.Comment(string)) obj.append(xml_ele) elif not is_def_cmnt and xml_ele is None: for string in cmnt_text: From d315dab6b8ed091d3ef94682dca87112112bfcd2 Mon Sep 17 00:00:00 2001 From: Rubel Date: Thu, 28 Sep 2023 23:39:09 +0200 Subject: [PATCH 78/93] updating PR comments. --- dev_tools/nyaml2nxdl/comment_collector.py | 6 +++--- dev_tools/nyaml2nxdl/nyaml2nxdl.py | 2 +- dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py | 1 - dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py | 5 ++--- dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py | 7 +++---- 5 files changed, 9 insertions(+), 12 deletions(-) diff --git a/dev_tools/nyaml2nxdl/comment_collector.py b/dev_tools/nyaml2nxdl/comment_collector.py index 573b02bd6d..46e3f0f355 100644 --- a/dev_tools/nyaml2nxdl/comment_collector.py +++ b/dev_tools/nyaml2nxdl/comment_collector.py @@ -204,7 +204,7 @@ def __init__(self, comment_id: int = -1, last_comment: "Comment" = None) -> None """Comment object can be considered as a block element that includes document element (an entity for what the comment is written). """ - self._elemt: Any = None + self._elemt: Dict[str, Any] = {} self._elemt_text: str = None self._is_elemt_found: bool = None self._is_elemt_stored: bool = None @@ -485,8 +485,8 @@ def replace_escape_char(self, text): def get_element_location(self): """Return yaml line '__line__' info and and line numner """ - if len(self._elemt) == 1: - raise ValueError(f"Comment element should be one but got " f"{self._elemt}") + if len(self._elemt) == 0: + raise ValueError(f"Comment element should be one or more but got {self._elemt}") return next(self._elemt.items()) def collect_yaml_line_info(self, yaml_dict, line_info_dict): diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl.py b/dev_tools/nyaml2nxdl/nyaml2nxdl.py index 9449d4487b..62e907dfaf 100755 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl.py @@ -90,7 +90,7 @@ def split_name_and_extension(file_path): is_flag=True, default=False, help=( - "Check if yaml and nxdl can be convertered from one to another version recursively and" + "Check if yaml and nxdl can be converted from one to another version recursively and" " get the same version of file. E.g. from NXexample.nxdl.xml to NXexample_consistency.nxdl.xml." ), ) diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py b/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py index d2590e2021..9ae0a55464 100755 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py @@ -346,7 +346,6 @@ def handle_not_root_level_doc(self, depth, text, tag="doc", file_out=None): tag = remove_namespace_from_tag(tag) indent = depth * DEPTH_SIZE else: - text = "" if "}" in tag: tag = remove_namespace_from_tag(tag) indent = depth * DEPTH_SIZE diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py b/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py index 5d14c0264b..fc06712398 100644 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py @@ -228,7 +228,6 @@ def check_for_skipped_attributes(component, value, allowed_attr=None, verbose=Fa def format_nxdl_doc(string): """NeXus format for doc string""" string = check_for_mapping_char_other(string) - formatted_doc = "" string_len = 80 if "\n" not in string: if len(string) > string_len: @@ -301,11 +300,11 @@ def xml_handle_exists(dct, obj, keyword, value): else: obj.set("maxOccurs", "unbounded") elif value[0] == "max" and value[2] == "min": - obj.set("minOccurs", str(value[3])) if str(value[1]) != "infty": - obj.set("maxOccurs", str(value[3])) + obj.set("maxOccurs", str(value[1])) else: obj.set("maxOccurs", "unbounded") + obj.set("minOccurs", str(value[3])) else: raise ValueError( f"Line {dct[line_number]}: exists keyword" diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py b/dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py index 87ab12973b..5aeae54279 100644 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py @@ -176,23 +176,22 @@ def clean_empty_lines(line_list): if not isinstance(line_list, list): line_list = line_list.split("\n") if "\n" in line_list else [""] - start_non_empty_line = 0 + start_non_empty_line = -1 ends_non_empty_line = None # Find the index of first non-empty line for ind, line in enumerate(line_list): - if not line.isspace(): + if len(line.strip()) > 1 : start_non_empty_line = ind break # Find the index of the last non-empty line for ind, line in enumerate(reversed(line_list)): - if not line.isspace(): + if len(line.strip()) > 1: ends_non_empty_line = -ind break if ends_non_empty_line == 0 : ends_non_empty_line = None - return line_list[start_non_empty_line : ends_non_empty_line] From 330975b54142b568846d2b9a9950fc372d620609 Mon Sep 17 00:00:00 2001 From: Rubel Date: Wed, 4 Oct 2023 13:20:23 +0200 Subject: [PATCH 79/93] Replacing os, and xml from dependencies and add lxml and pathlib as dependencies. --- dev_tools/nyaml2nxdl/comment_collector.py | 6 +- dev_tools/nyaml2nxdl/nyaml2nxdl.py | 10 +- .../nyaml2nxdl/nyaml2nxdl_backward_tools.py | 133 +++++++++--------- .../nyaml2nxdl/nyaml2nxdl_forward_tools.py | 91 ++++++------ dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py | 28 +++- dev_tools/utils/nxdl_utils.py | 65 ++++----- 6 files changed, 176 insertions(+), 157 deletions(-) diff --git a/dev_tools/nyaml2nxdl/comment_collector.py b/dev_tools/nyaml2nxdl/comment_collector.py index 46e3f0f355..7b63ececfa 100644 --- a/dev_tools/nyaml2nxdl/comment_collector.py +++ b/dev_tools/nyaml2nxdl/comment_collector.py @@ -405,10 +405,10 @@ def process_each_line(self, text, line_num): self.store_element(line_key, line_num) def has_post_comment(self): - """Ensure if this is a post coment or not. + """Ensure if this is a post comment or not. - Post comment means the comment that come at the very end without having any - nxdl element(class, group, filed and attribute.) + Post comment means the comment that comes at the very end without having any + nxdl element(class, group, filed and attribute.). """ for key, _ in self._elemt.items(): if "__line__post_comment" == key: diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl.py b/dev_tools/nyaml2nxdl/nyaml2nxdl.py index 62e907dfaf..2c092bee88 100755 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl.py @@ -24,7 +24,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import os from pathlib import Path import click @@ -50,8 +49,9 @@ def generate_nxdl_or_retrieve_nxdl(yaml_file, out_xml_file, verbose): retrieve the nxdl part from provided yaml. Else, generate nxdl from separated yaml with the help of nyaml2nxdl function """ - pa_path, rel_file = os.path.split(yaml_file) - sep_yaml = os.path.join(pa_path, f"temp_{rel_file}") + file_path = Path(yaml_file) + pa_path, rel_file = file_path.parent, file_path.name + sep_yaml = (pa_path / f"temp_{rel_file}").as_posix() hash_found = separate_hash_yaml_and_nxdl(yaml_file, sep_yaml, out_xml_file) if hash_found: @@ -115,11 +115,11 @@ def launch_tool(input_file, verbose, do_not_store_nxdl, check_consistency): """ Main function that distinguishes the input file format and launches the tools. """ - if os.path.isfile(input_file): + + if Path(input_file).is_file(): raw_name, ext = split_name_and_extension(input_file) else: raise ValueError("Need a valid input file.") - if ext == "yaml": xml_out_file = raw_name + _nxdl generate_nxdl_or_retrieve_nxdl(input_file, xml_out_file, verbose) diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py b/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py index 9ae0a55464..191eb58138 100755 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py @@ -21,15 +21,16 @@ # limitations under the License. # -import xml.etree.ElementTree as ET -from typing import Dict +from typing import Dict, Callable from typing import List from pathlib import Path +import lxml.etree as ET from .nyaml2nxdl_helper import clean_empty_lines from .nyaml2nxdl_helper import get_node_parent_info from .nyaml2nxdl_helper import get_yaml_escape_char_dict -from .nyaml2nxdl_helper import remove_namespace_from_tag +from .nyaml2nxdl_helper import (remove_namespace_from_tag, + is_dom_comment) from .nyaml2nxdl_helper import (NXDL_FIELD_ATTRIBUTES, NXDL_GROUP_ATTRIBUTES, NXDL_ATTRIBUTES_ATTRIBUTES, @@ -53,39 +54,37 @@ def separate_pi_comments(input_file): """ comments_list = [] comment = [] - xml_lines = [] with open(input_file, "r", encoding="utf-8") as file: lines = file.readlines() - has_pi = True + def_tag = " 0 and has_pi: + elif CMNT_END in line and len(comment) > 0: comment.append(line.replace(CMNT_END, "")) comments_list.append("".join(comment)) comment.clear() - elif def_tag in line or not has_pi: - has_pi = False - xml_lines.append(line) - elif len(comment) > 0 and has_pi: + elif len(comment) > 0: comment.append(line) - else: - xml_lines.append(line) - return comments_list, "".join(xml_lines) + elif def_tag in line: + break + return comments_list # Collected: https://dustinoprea.com/2019/01/22/python-parsing-xml-and-retaining-the-comments/ class _CommentedTreeBuilder(ET.TreeBuilder): - def comment(self, text): - """defining comment builder in TreeBuilder + def start(self, tag, attrs): + super().start(tag=tag, attrs=attrs) + + def Comment(self, text): + """Defining comment builder in TreeBuilder """ self.start(CMNT_TAG, {}) self.data(text) @@ -98,10 +97,10 @@ def parse(filepath): Construct parser function for modified tree builder for including modified TreeBuilder and rebuilding XMLParser. """ - comments, xml_str = separate_pi_comments(filepath) + comments = separate_pi_comments(filepath) ctb = _CommentedTreeBuilder() - xp_parser = ET.XMLParser(target=ctb) - root = ET.fromstring(xml_str, parser=xp_parser) + xp_parser = ET.XMLParser(target=ctb, encoding='UTF-8') + root = ET.parse(filepath, xp_parser) return comments, root @@ -198,7 +197,7 @@ def handle_symbols(self, depth, node): tag = remove_namespace_from_tag(child.tag) if tag == CMNT_TAG and self.include_comment: last_comment = self.convert_to_yaml_comment( - depth * DEPTH_SIZE, child.text + depth, child.text ) if tag == "doc": symbol_cmnt_list.append(last_comment) @@ -225,7 +224,7 @@ def handle_symbols(self, depth, node): tag = remove_namespace_from_tag(symbol_doc.tag) if tag == CMNT_TAG and self.include_comment: last_comment = self.convert_to_yaml_comment( - depth * DEPTH_SIZE, symbol_doc.text + depth, symbol_doc.text ) if tag == "doc": sbl_doc_cmnt_list.append(last_comment) @@ -280,17 +279,12 @@ def handle_root_level_doc(self, node): text = node.text text = self.handle_not_root_level_doc(depth=0, text=text) self.root_level_doc = text + + def clean_and_organise_text(self, text, depth): + """Reconstruct text from doc and comment. - # pylint: disable=too-many-branches - def handle_not_root_level_doc(self, depth, text, tag="doc", file_out=None): - """Handle docs field of group and field but not root. - - Handle docs field along the yaml file. In this function we also tried to keep - the track of indentation. E.g. the below doc block. - * Topic name - Description of topic + Cleaninig up unintentional and accidental empty lines and spaces. """ - # Handling empty doc if not text: text = "" @@ -320,7 +314,7 @@ def handle_not_root_level_doc(self, depth, text, tag="doc", file_out=None): first_line_indent_n = yaml_indent_n # for indent_diff -ve all lines will move left by the same amount - # for indect_diff +ve all lines will move right by the same amount + # for indent_diff +ve all lines will move right by the same amount indent_diff = yaml_indent_n - first_line_indent_n # CHeck for first line empty if not keep first line empty @@ -328,7 +322,6 @@ def handle_not_root_level_doc(self, depth, text, tag="doc", file_out=None): line_indent_n = 0 # count first empty spaces without alphabet line_indent_n = len(line) - len(line.lstrip()) - line_indent_n = line_indent_n + indent_diff if line_indent_n < yaml_indent_n: # if line still under yaml identation @@ -337,18 +330,26 @@ def handle_not_root_level_doc(self, depth, text, tag="doc", file_out=None): text_tmp.append(line_indent_n * " " + line.strip()) text = "\n" + "\n".join(text_tmp) - if "}" in tag: - tag = remove_namespace_from_tag(tag) - indent = depth * DEPTH_SIZE + elif text: text = "\n" + (depth + 1) * DEPTH_SIZE + text.strip() - if "}" in tag: - tag = remove_namespace_from_tag(tag) - indent = depth * DEPTH_SIZE - else: - if "}" in tag: - tag = remove_namespace_from_tag(tag) - indent = depth * DEPTH_SIZE + + return text + + # pylint: disable=too-many-branches + def handle_not_root_level_doc(self, depth, text, tag="doc", file_out=None): + """Handle docs field of group and field but not root. + + Handle docs field along the yaml file. In this function we also tried to keep + the track of indentation. E.g. the below doc block. + * Topic name + Description of topic + """ + + text = self.clean_and_organise_text(text, depth) + if "}" in tag: + tag = remove_namespace_from_tag(tag) + indent = depth * DEPTH_SIZE doc_str = f"{indent}{tag}: |{text}\n" if file_out: @@ -376,18 +377,18 @@ def print_root_level_doc(self, file_out): self.write_out(indent, text, file_out) self.root_level_doc = "" - def convert_to_yaml_comment(self, indent, text): + def convert_to_yaml_comment(self, depth, text): """ Convert into yaml comment by adding exta '#' char in front of comment lines """ - lines = text.split("\n") + # To store indentation text from comment + lines = self.clean_and_organise_text(text, depth).split("\n") mod_lines = [] for line in lines: line = line.strip() if line: if line[0] != "#": line = "# " + line - line = indent + line mod_lines.append(line) # The starting '\n' to keep multiple comments separate return "\n" + "\n".join(mod_lines) @@ -444,12 +445,12 @@ def print_root_level_info(self, depth, file_out): ) self.write_out(indent=(0 * DEPTH_SIZE), text=symbol, file_out=file_out) - if len(self.pi_comments) > 1: - indent = DEPTH_SIZE * depth - # The first comment is top level copyright doc string - for comment in self.pi_comments[1:]: + indent = depth * DEPTH_SIZE + # Take care copyright doc string + for comment in self.pi_comments: + if comment and not is_dom_comment(comment): self.write_out( - indent, self.convert_to_yaml_comment(indent, comment), file_out + indent, self.convert_to_yaml_comment(depth, comment), file_out ) if self.root_level_definition: # Store NXname for writing at end of definition attributes @@ -738,10 +739,9 @@ def handle_attributes(self, depth, node, file_out): nm_attr = "name" node_attr = node.attrib # Maintain order: name and type in form name(type) or (type)name that come first - name = node_attr.get(nm_attr, "") + name = node_attr.pop(nm_attr, "") if not name: raise ValueError("Attribute must have an name key.") - del node_attr[nm_attr] indent = depth * DEPTH_SIZE escapesymbol = r"\@" @@ -797,11 +797,10 @@ def handle_link(self, depth, node, file_out): node_attr = node.attrib # Handle special cases - if "name" in node_attr: + name = node_attr.pop("name", "") + if name: indent = depth * DEPTH_SIZE - name = node_attr["name"] or "" file_out.write(f"{indent}{name}(link):\n") - del node_attr["name"] depth_ = depth + 1 # Handle general cases @@ -821,13 +820,11 @@ def handle_choice(self, depth, node, file_out): """ node_attr = node.attrib + name = node_attr.pop("name", "") # Handle special casees - if "name" in node_attr: + if name: indent = depth * DEPTH_SIZE - attr = node_attr["name"] - file_out.write(f"{indent}{attr}(choice): \n") - - del node_attr["name"] + file_out.write(f"{indent}{name}(choice): \n") depth_ = depth + 1 # Take care of attributes of choice element, attributes may come in future. @@ -847,7 +844,7 @@ def handle_comment(self, depth, node, file_out): Collect comment element and pass to write_out function """ indent = depth * DEPTH_SIZE - text = self.convert_to_yaml_comment(indent, node.text) + text = self.convert_to_yaml_comment(depth, node.text) self.write_out(indent, text, file_out) @@ -872,8 +869,12 @@ def xmlparse(self, output_yml, xml_tree, depth, verbose): tree = xml_tree["tree"] node = xml_tree["node"] if verbose: - print(f"Node tag: {remove_namespace_from_tag(node.tag)}\n") - print(f"Attributes: {node.attrib}\n") + if isinstance(node.tag, Callable): + print(f"Node tag: {node.tag}\n") + print(f"Node text: {node.text}\n") + else: + print(f"Node tag: {remove_namespace_from_tag(node.tag)}\n") + print(f"Attributes: {node.attrib}\n") with open(output_yml, "a", encoding="utf-8") as file_out: tag = remove_namespace_from_tag(node.tag) if tag == "definition": @@ -882,11 +883,11 @@ def xmlparse(self, output_yml, xml_tree, depth, verbose): # Taking care of root level doc and symbols remove_cmnt_n = None last_comment = "" - for child in list(node): + for child in node: tag_tmp = remove_namespace_from_tag(child.tag) if tag_tmp == CMNT_TAG and self.include_comment: last_comment = self.convert_to_yaml_comment( - depth * DEPTH_SIZE, child.text + depth, child.text ) remove_cmnt_n = child if tag_tmp == "doc": diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py b/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py index fc06712398..4406c02474 100644 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py @@ -24,8 +24,8 @@ import pathlib import textwrap import datetime -import xml.etree.ElementTree as ET -from xml.dom import minidom +import lxml.etree as ET +from urllib.parse import unquote import warnings import yaml @@ -40,12 +40,12 @@ YAML_FIELD_ATTRIBUTES, YAML_GROUP_ATTRIBUTES, YAML_LINK_ATTRIBUTES) -from .nyaml2nxdl_helper import remove_namespace_from_tag +from .nyaml2nxdl_helper import (remove_namespace_from_tag, + is_dom_comment) # pylint: disable=too-many-lines, global-statement, invalid-name DOM_COMMENT = ( - f"\n" f"# NeXus - Neutron and X-ray Common Data Format\n" f"# \n" f"# Copyright (C) 2014-{datetime.date.today().year} NeXus International Advisory Committee (NIAC)\n" @@ -80,14 +80,6 @@ def check_for_dom_comment_in_yaml(): """Check the yaml file has dom comment or dom comment needed to be hard coded.""" - # some signature keywords to distingush dom comments from other comments. - signature_keyword_list = [ - "NeXus", - "GNU Lesser General Public", - "Free Software Foundation", - "Copyright (C)", - "WITHOUT ANY WARRANTY", - ] # Check for dom comments in first five comments dom_comment = "" @@ -98,13 +90,9 @@ def check_for_dom_comment_in_yaml(): text = cmnt_list[0] else: continue - dom_comment = text - dom_comment_ind = ind - for keyword in signature_keyword_list: - if keyword not in text: - dom_comment = "" - break - if dom_comment: + if is_dom_comment(text): + dom_comment = text + dom_comment_ind = ind break # deactivate the root dom_comment, So that the corresponding comment would not be @@ -239,7 +227,7 @@ def format_nxdl_doc(string): else: text_lines = string.split("\n") text_lines = clean_empty_lines(text_lines) - formatted_doc += "\n" + "\n".join(text_lines) + formatted_doc = "\n" + "\n".join(text_lines) if not formatted_doc.endswith("\n"): formatted_doc += "\n" return formatted_doc @@ -870,10 +858,8 @@ def xml_handle_comment( (e.g. __line__data) and line_loc_no (e.g. 30). It does following tasks: - 1. Returns list of comments texts, if comment is intended for definition element. - other return always []. - 2. Rearrange comment elements of xml_ele and xml_ele where comment comes first. - 3. Append comment element when no xml_ele found as general comments. + 1. Rearrange comment elements of xml_ele and xml_ele where comment comes first. + 2. Append comment element when no xml_ele found as general comments. """ line_info = (line_annotation, int(line_loc_no)) @@ -887,6 +873,8 @@ def xml_handle_comment( if xml_ele is not None: obj.remove(xml_ele) for string in cmnt_text: + # Format comment string to preserve text nxdl to yaml and vice versa + string = format_nxdl_doc(string) obj.append(ET.Comment(string)) obj.append(xml_ele) elif not is_def_cmnt and xml_ele is None: @@ -962,32 +950,45 @@ def recursive_build(obj, dct, verbose): ) -def pretty_print_xml(xml_root, output_xml, def_comments=None): +def extend_doc_type(doc_type, new_component, comment=False): + """Extend doc type for etree.tostring function + + Extend doc type to build DOM and process instruction comments. """ + start_sym = '' + if comment: + start_sym = '' + return doc_type + '\n' + start_sym + new_component + end_sym + + +def pretty_print_xml(xml_root, output_xml, def_comments=None): + """Print in nxdl.xml file. + Print better human-readable indented and formatted xml file using built-in libraries and preceding XML processing instruction """ - dom = minidom.parseString(ET.tostring(xml_root, encoding="utf-8", method="xml")) - proc_instruction = dom.createProcessingInstruction( - "xml-stylesheet", 'type="text/xsl" href="nxdlformat.xsl"' - ) - dom_comment = dom.createComment(DOM_COMMENT) - root = dom.firstChild - dom.insertBefore(proc_instruction, root) - dom.insertBefore(dom_comment, root) + # Handle DOM as doc_type + doc_type = '' + if DOM_COMMENT: + doc_type = extend_doc_type(doc_type, DOM_COMMENT, comment=True) if def_comments: for string in def_comments: - def_comt_ele = dom.createComment(string) - dom.insertBefore(def_comt_ele, root) + doc_type = extend_doc_type(doc_type, string, comment=True) + tmp_xml = "tmp.xml" - xml_string = dom.toprettyxml(indent=1 * DEPTH_SIZE, newl="\n", encoding="UTF-8") + xml_string = ET.tostring(xml_root, pretty_print=True, + encoding="UTF-8", xml_declaration=True, + doctype=doc_type) with open(tmp_xml, "wb") as file_tmp: file_tmp.write(xml_string) flag = False with open(tmp_xml, "r", encoding="utf-8") as file_out: with open(output_xml, "w", encoding="utf-8") as file_out_mod: for i in file_out.readlines(): + i = unquote(i.encode()) if "" not in i and "" not in i and flag is False: file_out_mod.write(i) elif "" in i and "" in i: @@ -1029,7 +1030,10 @@ def nyaml2nxdl(input_file: str, out_file, verbose: bool): "application/base contains the following root-level entries:\n" ) print(str(yml_appdef.keys())) - xml_root = ET.Element("definition", {}) + # etree does not allow to set namespace-map after root creation + # So, mimic a nsmap and fill it later as dict has hash property + nsmap = {None: "http://definition.nexusformat.org/nxdl/3.1",} + xml_root = ET.Element("definition", attrib={}, nsmap=nsmap) assert ( "category" in yml_appdef.keys() ), "Required root-level keyword category is missing!" @@ -1087,12 +1091,13 @@ def nyaml2nxdl(input_file: str, out_file, verbose: bool): xml_root.set("type", "group") # Taking care of namespaces namespaces = { - "xmlns": "http://definition.nexusformat.org/nxdl/3.1", - "xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance", - "xsi:schemaLocation": "http://definition.nexusformat.org/nxdl/3.1 ../nxdl.xsd", - } - for key, ns in namespaces.items(): - xml_root.attrib[key] = ns + "xsi": "http://www.w3.org/2001/XMLSchema-instance", + } + # Fill nsmap variable here + nsmap.update(namespaces) + xml_root.attrib["{http://www.w3.org/2001/XMLSchema-instance}schemaLocation"] = \ + "http://definition.nexusformat.org/nxdl/3.1 ../nxdl.xsd".replace(' ', "%20") + # Taking care of Symbols elements if "symbols" in yml_appdef.keys(): xml_handle_symbols(yml_appdef, xml_root, "symbols", yml_appdef["symbols"]) diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py b/dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py index 5aeae54279..6245034762 100644 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py @@ -30,6 +30,7 @@ from yaml.loader import Loader from yaml.nodes import ScalarNode from yaml.resolver import BaseResolver +from typing import Callable # Yaml library does not except the keys (escape char "\t" and yaml separator ":") ESCAPE_CHAR_DICT_IN_YAML = {"\t": " "} @@ -100,8 +101,10 @@ def remove_namespace_from_tag(tag): """Helper function to remove the namespace from an XML tag.""" - - return tag.split("}")[-1] + if isinstance(tag, Callable) and tag.__name__ == 'Comment': + return '!--' + else: + return tag.split("}")[-1] class LineLoader(Loader): # pylint: disable=too-many-ancestors @@ -292,3 +295,24 @@ def separate_hash_yaml_and_nxdl(yaml_file, sep_yaml, sep_xml): yml_f_ob.write(last_line) return sha_hash + + +def is_dom_comment(text): + """Analyze a comment, whether it is a dom comment or not. + + Return true if dom comment. + """ + + # some signature keywords to distingush dom comments from other comments. + signature_keyword_list = [ + "NeXus", + "GNU Lesser General Public", + "Free Software Foundation", + "Copyright (C)", + "WITHOUT ANY WARRANTY", + ] + for keyword in signature_keyword_list: + if keyword not in text: + return False + + return True diff --git a/dev_tools/utils/nxdl_utils.py b/dev_tools/utils/nxdl_utils.py index 9d3f55c2d7..6f39563af2 100644 --- a/dev_tools/utils/nxdl_utils.py +++ b/dev_tools/utils/nxdl_utils.py @@ -3,8 +3,11 @@ """ import os + +from pathlib import Path import textwrap -import xml.etree.ElementTree as ET +import lxml.etree as ET +from lxml.etree import ParseError as xmlER from functools import lru_cache from glob import glob @@ -15,24 +18,17 @@ class NxdlAttributeNotFoundError(Exception): def get_app_defs_names(): """Returns all the AppDef names without their extension: .nxdl.xml""" - app_def_path_glob = os.path.join( - get_nexus_definitions_path(), - "applications", - "*.nxdl*" - ) - contrib_def_path_glob = os.path.join( - get_nexus_definitions_path(), - "contributed_definitions", - "*.nxdl*" - ) - + app_def_path_glob = get_nexus_definitions_path() / "applications" / "*.nxdl*" + + contrib_def_path_glob = Path(get_nexus_definitions_path()) / "contributed_definitions" / "*.nxdl*" + files = sorted(glob(app_def_path_glob)) for nexus_file in sorted(contrib_def_path_glob): root = get_xml_root(nexus_file) if root.attrib["category"] == "application": files.append(nexus_file) - return [os.path.basename(file).split(".")[0] for file in files] + ["NXroot"] + return [Path(file).stem for file in files] + ["NXroot"] @lru_cache(maxsize=None) @@ -48,9 +44,9 @@ def get_nexus_definitions_path(): try: # either given by sys env return os.environ["NEXUS_DEF_PATH"] except KeyError: # or it should be available locally under the dir 'definitions' - local_dir = os.path.abspath(os.path.dirname(__file__)) + local_dir = Path(__file__).resolve().parent for _ in range(2): - local_dir = os.path.dirname(local_dir) + local_dir = local_dir.parent return local_dir @@ -129,30 +125,24 @@ def get_nx_classes(): """Read base classes from the NeXus definition folder. Check each file in base_classes, applications, contributed_definitions. If its category attribute is 'base', then it is added to the list.""" - base_classes = sorted( - glob(os.path.join(get_nexus_definitions_path(), "base_classes", "*.nxdl.xml")) - ) - applications = sorted( - glob(os.path.join(get_nexus_definitions_path(), "applications", "*.nxdl.xml")) - ) - contributed = sorted( - glob( - os.path.join( - get_nexus_definitions_path(), "contributed_definitions", "*.nxdl.xml" - ) - ) - ) + nexus_definition_path = get_nexus_definitions_path() + base_classes = sorted(nexus_definition_path.glob("base_classes/*.nxdl.xml")) + applications = sorted(nexus_definition_path.glob("applications/*.nxdl.xml")) + contributed = sorted(nexus_definition_path.glob("contributed_definitions/*.nxdl.xml")) nx_class = [] for nexus_file in base_classes + applications + contributed: - root = get_xml_root(nexus_file) + try: + root = get_xml_root(nexus_file) + except xmlER as e: + raise ValueError(f"Getting an issue while parsing file {nexus_file}") from e if root.attrib["category"] == "base": - nx_class.append(os.path.basename(nexus_file).split(".")[0]) + nx_class.append(nexus_file.stem) return sorted(nx_class) def get_nx_units(): """Read unit kinds from the NeXus definition/nxdlTypes.xsd file""" - filepath = f"{get_nexus_definitions_path()}{os.sep}nxdlTypes.xsd" + filepath = get_nexus_definitions_path() / "nxdlTypes.xsd" root = get_xml_root(filepath) units_and_type_list = [] for child in root: @@ -172,7 +162,8 @@ def get_nx_units(): def get_nx_attribute_type(): """Read attribute types from the NeXus definition/nxdlTypes.xsd file""" - filepath = get_nexus_definitions_path() + "/nxdlTypes.xsd" + filepath = get_nexus_definitions_path() / "nxdlTypes.xsd" + root = get_xml_root(filepath) units_and_type_list = [] for child in root: @@ -320,15 +311,13 @@ def find_definition_file(bc_name): """find the nxdl file corresponding to the name. Note that it first checks in contributed and goes beyond only if no contributed found """ + nexus_def_path = get_nexus_definitions_path() bc_filename = None for nxdl_folder in ["contributed_definitions", "base_classes", "applications"]: - if os.path.exists( - f"{get_nexus_definitions_path()}{os.sep}" - f"{nxdl_folder}{os.sep}{bc_name}.nxdl.xml" - ): + nxdl_file = nexus_def_path / nxdl_folder/ f"{bc_name}.nxdl.xml" + if nxdl_file.exists(): bc_filename = ( - f"{get_nexus_definitions_path()}{os.sep}" - f"{nxdl_folder}{os.sep}{bc_name}.nxdl.xml" + nexus_def_path / nxdl_folder / f"{bc_name}.nxdl.xml" ) break return bc_filename From cf9b9f06280e7d55975dbcc622f67f11d340f0ba Mon Sep 17 00:00:00 2001 From: Rubel Date: Wed, 4 Oct 2023 16:40:22 +0200 Subject: [PATCH 80/93] updating PR comments. --- dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py | 4 ++-- dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py b/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py index 191eb58138..56a596a99a 100755 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py @@ -247,8 +247,8 @@ def store_root_level_comments(self, holder, comment): def handle_definition(self, node): """Handle definition group and its attributes. - NOTE: Here we try to store the order of the xml element attributes. So that we get - exactly the same file in nxdl from yaml. + NOTE: Here we try to store the order of the xml element attributes, so that we get + the same order in nxdl from yaml. """ keyword = "" # tmp_word for reseving the location diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py b/dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py index 6245034762..205599df83 100644 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py @@ -168,7 +168,7 @@ def get_node_parent_info(tree, node): index = index of grand child node of tree """ - # map from grand child to parent which is child tree + # map from grand child to parent which is child of tree parent_map = {c: p for p in tree.iter() for c in p} parent = parent_map[node] return parent, list(parent).index(node) From 86df4ed2bbf40bf289786f386d67e2e56f4e309c Mon Sep 17 00:00:00 2001 From: Rubel Date: Wed, 4 Oct 2023 16:59:31 +0200 Subject: [PATCH 81/93] Implementing black on nyal2nxdl converter files. --- dev_tools/nyaml2nxdl/nyaml2nxdl.py | 9 +- .../nyaml2nxdl/nyaml2nxdl_backward_tools.py | 88 ++++++----- .../nyaml2nxdl/nyaml2nxdl_forward_tools.py | 145 +++++++++++------- dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py | 81 +++++----- dev_tools/utils/nxdl_utils.py | 29 ++-- 5 files changed, 188 insertions(+), 164 deletions(-) diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl.py b/dev_tools/nyaml2nxdl/nyaml2nxdl.py index 2c092bee88..8ee97c380a 100755 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl.py @@ -71,10 +71,10 @@ def split_name_and_extension(file_path): return file raw name and extension """ path = Path(file_path) - ext = ''.join(path.suffixes) + ext = "".join(path.suffixes) full_path_stem = file_path[0 : file_path.index(ext)] return full_path_stem, ext[1:] - + @click.command() @click.option( @@ -84,7 +84,6 @@ def split_name_and_extension(file_path): help="The path to the XML or YAML input data file to read and create \ a YAML or XML file from, respectively.", ) - @click.option( "--check-consistency", is_flag=True, @@ -115,7 +114,7 @@ def launch_tool(input_file, verbose, do_not_store_nxdl, check_consistency): """ Main function that distinguishes the input file format and launches the tools. """ - + if Path(input_file).is_file(): raw_name, ext = split_name_and_extension(input_file) else: @@ -123,7 +122,7 @@ def launch_tool(input_file, verbose, do_not_store_nxdl, check_consistency): if ext == "yaml": xml_out_file = raw_name + _nxdl generate_nxdl_or_retrieve_nxdl(input_file, xml_out_file, verbose) - + # For consistency running if check_consistency: yaml_out_file = raw_name + "_consistency." + ext diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py b/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py index 56a596a99a..3582b1fee6 100755 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py @@ -29,12 +29,13 @@ from .nyaml2nxdl_helper import clean_empty_lines from .nyaml2nxdl_helper import get_node_parent_info from .nyaml2nxdl_helper import get_yaml_escape_char_dict -from .nyaml2nxdl_helper import (remove_namespace_from_tag, - is_dom_comment) -from .nyaml2nxdl_helper import (NXDL_FIELD_ATTRIBUTES, - NXDL_GROUP_ATTRIBUTES, - NXDL_ATTRIBUTES_ATTRIBUTES, - NXDL_LINK_ATTRIBUTES) +from .nyaml2nxdl_helper import remove_namespace_from_tag, is_dom_comment +from .nyaml2nxdl_helper import ( + NXDL_FIELD_ATTRIBUTES, + NXDL_GROUP_ATTRIBUTES, + NXDL_ATTRIBUTES_ATTRIBUTES, + NXDL_LINK_ATTRIBUTES, +) DEPTH_SIZE = " " @@ -48,7 +49,7 @@ def separate_pi_comments(input_file): """Separate PI comments from ProcessesInstruction (PI). - ProcessesInstruction refers xml element process and version + ProcessesInstruction refers xml element process and version Separate the comments that comes immediately after XML process instruction part, i.e. copyright comment part. """ @@ -59,7 +60,6 @@ def separate_pi_comments(input_file): lines = file.readlines() def_tag = " 0: comment.append(line) - elif def_tag in line: + elif def_tag in line: break return comments_list @@ -82,10 +82,9 @@ def separate_pi_comments(input_file): class _CommentedTreeBuilder(ET.TreeBuilder): def start(self, tag, attrs): super().start(tag=tag, attrs=attrs) - + def Comment(self, text): - """Defining comment builder in TreeBuilder - """ + """Defining comment builder in TreeBuilder""" self.start(CMNT_TAG, {}) self.data(text) self.end(CMNT_TAG_END) @@ -99,14 +98,13 @@ def parse(filepath): """ comments = separate_pi_comments(filepath) ctb = _CommentedTreeBuilder() - xp_parser = ET.XMLParser(target=ctb, encoding='UTF-8') + xp_parser = ET.XMLParser(target=ctb, encoding="UTF-8") root = ET.parse(filepath, xp_parser) return comments, root def handle_mapping_char(text, depth=-1, skip_n_line_on_top=False): - """Check for escape character and replace by alternative character. - """ + """Check for escape character and replace by alternative character.""" escape_char = get_yaml_escape_char_dict() for esc_key, val in escape_char.items(): @@ -160,17 +158,18 @@ def __init__( # 'symbol_comments': [comments]} self.root_level_comment: Dict[str, str] = {} - self.optionality_keys = ("minOccurs", - "maxOccurs", - "optional", - "recommended", - "required") + self.optionality_keys = ( + "minOccurs", + "maxOccurs", + "optional", + "recommended", + "required", + ) # "Take care of general attributes. Note other choices may be allowed in the future" self.choice_allowed_attr = () def print_yml(self, input_file, output_yml, verbose): - """Parse an XML file provided as input and print a YML file. - """ + """Parse an XML file provided as input and print a YML file.""" output_file_path = Path(output_yml) if output_file_path.is_file(): output_file_path.unlink() @@ -196,9 +195,7 @@ def handle_symbols(self, depth, node): for child in list(node): tag = remove_namespace_from_tag(child.tag) if tag == CMNT_TAG and self.include_comment: - last_comment = self.convert_to_yaml_comment( - depth, child.text - ) + last_comment = self.convert_to_yaml_comment(depth, child.text) if tag == "doc": symbol_cmnt_list.append(last_comment) # The line below is for handling length of 'symbol_comments' and @@ -279,7 +276,7 @@ def handle_root_level_doc(self, node): text = node.text text = self.handle_not_root_level_doc(depth=0, text=text) self.root_level_doc = text - + def clean_and_organise_text(self, text, depth): """Reconstruct text from doc and comment. @@ -295,13 +292,12 @@ def clean_and_organise_text(self, text, depth): text = clean_empty_lines(text.split("\n")) text_tmp = [] yaml_indent_n = len((depth + 1) * DEPTH_SIZE) - # Find indentation in the first line of text having alphabet first_line_indent_n = 0 for line in text: # Consider only the lines that has at least one non-space character - # and skip starting lines of a text block are empty + # and skip starting lines of a text block are empty if len(line.lstrip()) != 0: first_line_indent_n = len(line) - len(line.lstrip()) break @@ -333,7 +329,7 @@ def clean_and_organise_text(self, text, depth): elif text: text = "\n" + (depth + 1) * DEPTH_SIZE + text.strip() - + return text # pylint: disable=too-many-branches @@ -365,8 +361,7 @@ def write_out(self, indent, text, file_out): file_out.write(line_string) def print_root_level_doc(self, file_out): - """Print at the root level of YML file the general documentation field found in XML file - """ + """Print at the root level of YML file the general documentation field found in XML file""" indent = 0 * DEPTH_SIZE text = self.root_level_comment.get("root_doc") @@ -554,19 +549,25 @@ def check_for_unwanted_attributes(self, node, allowed_attributes_li=None, tag=No if node_tag == "field": for key in node.attrib.keys(): if key not in NXDL_FIELD_ATTRIBUTES: - raise ValueError(f"Field has an unwanted attribute {key}." - f"NeXus field allows attributes from {NXDL_FIELD_ATTRIBUTES}") + raise ValueError( + f"Field has an unwanted attribute {key}." + f"NeXus field allows attributes from {NXDL_FIELD_ATTRIBUTES}" + ) elif node_tag == "group": for key in node.attrib.keys(): if key not in NXDL_GROUP_ATTRIBUTES: - raise ValueError(f"Attribute has an unwanted attribute {key}." - f"NeXus attribute allows attributes from {NXDL_GROUP_ATTRIBUTES}") + raise ValueError( + f"Attribute has an unwanted attribute {key}." + f"NeXus attribute allows attributes from {NXDL_GROUP_ATTRIBUTES}" + ) elif node_tag == tag: for key in node.attrib.keys(): if key not in allowed_attributes_li: - raise ValueError(f"{tag.capitalized()} has an unwanted attribute {key}." - f"NeXus {tag.capitalized()} allows attributes from {allowed_attributes_li}") - + raise ValueError( + f"{tag.capitalized()} has an unwanted attribute {key}." + f"NeXus {tag.capitalized()} allows attributes from {allowed_attributes_li}" + ) + # pylint: disable=too-many-branches, too-many-locals def handle_dimension(self, depth, node, file_out): """Handle the dimension field. @@ -664,7 +665,7 @@ def handle_dimension(self, depth, node, file_out): f"{indent}{key}: " f"{handle_mapping_char(value, depth + 3, False)}\n" ) - + def handle_enumeration(self, depth, node, file_out): """ Handle the enumeration field parsed from the xml file. @@ -726,10 +727,10 @@ def handle_enumeration(self, depth, node, file_out): remove_nodes.append(item_child) for ch_node in remove_nodes: node.remove(ch_node) - + indent = depth * DEPTH_SIZE tag = remove_namespace_from_tag(node.tag) - enum_list = ', '.join(enum_list) + enum_list = ", ".join(enum_list) file_out.write(f"{indent}{tag}: [{enum_list}]\n") def handle_attributes(self, depth, node, file_out): @@ -847,7 +848,6 @@ def handle_comment(self, depth, node, file_out): text = self.convert_to_yaml_comment(depth, node.text) self.write_out(indent, text, file_out) - def recursion_in_xml_tree(self, depth, xml_tree, output_yml, verbose): """ Descend lower level in xml tree. If we are in the symbols branch, the recursive @@ -886,9 +886,7 @@ def xmlparse(self, output_yml, xml_tree, depth, verbose): for child in node: tag_tmp = remove_namespace_from_tag(child.tag) if tag_tmp == CMNT_TAG and self.include_comment: - last_comment = self.convert_to_yaml_comment( - depth, child.text - ) + last_comment = self.convert_to_yaml_comment(depth, child.text) remove_cmnt_n = child if tag_tmp == "doc": self.store_root_level_comments("root_doc", last_comment) diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py b/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py index 4406c02474..afde294926 100644 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py @@ -36,12 +36,13 @@ from .nyaml2nxdl_helper import clean_empty_lines from .nyaml2nxdl_helper import get_yaml_escape_char_reverter_dict from .nyaml2nxdl_helper import nx_name_type_resolving -from .nyaml2nxdl_helper import (YAML_ATTRIBUTES_ATTRIBUTES, - YAML_FIELD_ATTRIBUTES, - YAML_GROUP_ATTRIBUTES, - YAML_LINK_ATTRIBUTES) -from .nyaml2nxdl_helper import (remove_namespace_from_tag, - is_dom_comment) +from .nyaml2nxdl_helper import ( + YAML_ATTRIBUTES_ATTRIBUTES, + YAML_FIELD_ATTRIBUTES, + YAML_GROUP_ATTRIBUTES, + YAML_LINK_ATTRIBUTES, +) +from .nyaml2nxdl_helper import remove_namespace_from_tag, is_dom_comment # pylint: disable=too-many-lines, global-statement, invalid-name @@ -344,9 +345,9 @@ def xml_handle_dimensions(dct, obj, keyword, value: dict): possible_dimension_attrs = ["rank"] # nxdl attributes line_number = f"__line__{keyword}" line_loc = dct[line_number] - assert "dim" in value.keys(), ( - f"Line {line_loc}: No dim as child of dimension has been found." - ) + assert ( + "dim" in value.keys() + ), f"Line {line_loc}: No dim as child of dimension has been found." xml_handle_comment(obj, line_number, line_loc) dims = ET.SubElement(obj, "dimensions") # Consider all the children under dimension is dim element and @@ -423,9 +424,9 @@ def xml_handle_dim_from_dimension_dict( if attr == "dim": # dim consists of [index, value] list llist_ind_value = vvalue - assert isinstance(llist_ind_value, list), ( - f"Line {value[line_number]}: dim argument not a list !" - ) + assert isinstance( + llist_ind_value, list + ), f"Line {value[line_number]}: dim argument not a list !" xml_handle_comment(dims_obj, line_number, line_loc) if isinstance(rank, int) and rank > 0: assert rank == len(llist_ind_value), ( @@ -471,8 +472,10 @@ def xml_handle_dim_from_dimension_dict( pass else: if kkkey in deprecated_dim_attrs: - dep_text = (f"Attrbute {kkkey} is deprecated. " - f"Check attributes after line {cmnt_loc}") + dep_text = ( + f"Attrbute {kkkey} is deprecated. " + f"Check attributes after line {cmnt_loc}" + ) warnings.warn(dep_text, DeprecationWarning) for i, dim in enumerate(dim_list): # all atribute of dims comes as list @@ -656,9 +659,7 @@ def check_keyword_variable(verbose, dct, keyword, value): """ keyword_name, keyword_type = nx_name_type_resolving(keyword) if verbose: - print( - f"{keyword_name}({keyword_type}): value type is {type(value)}\n" - ) + print(f"{keyword_name}({keyword_type}): value type is {type(value)}\n") if keyword_name == "" and keyword_type == "": line_number = f"__line__{keyword}" raise ValueError(f"Line {dct[line_number]}: found an improper yaml key !") @@ -707,7 +708,9 @@ def xml_handle_attributes(dct, obj, keyword, value, verbose): continue line_number = f"__line__{attr}" line_loc = value[line_number] - if attr in ["doc", *YAML_ATTRIBUTES_ATTRIBUTES] and not isinstance(attr_val, dict): + if attr in ["doc", *YAML_ATTRIBUTES_ATTRIBUTES] and not isinstance( + attr_val, dict + ): if attr == "unit": elemt_obj.set(f"{attr}s", str(value[attr])) rm_key_list.append(attr) @@ -733,7 +736,9 @@ def xml_handle_attributes(dct, obj, keyword, value, verbose): for key in rm_key_list: del value[key] # Check cor skipped attribute - check_for_skipped_attributes("Attribute", value, YAML_ATTRIBUTES_ATTRIBUTES, verbose) + check_for_skipped_attributes( + "Attribute", value, YAML_ATTRIBUTES_ATTRIBUTES, verbose + ) if value: recursive_build(elemt_obj, value, verbose) @@ -768,9 +773,10 @@ def validate_field_attribute_and_value(v_attr, vval, allowed_attribute, value): ) -def xml_handle_fields_or_group(dct, obj, keyword, value, ele_type, allowed_attr, verbose=False): - """Handle a field or group in yaml file. - """ +def xml_handle_fields_or_group( + dct, obj, keyword, value, ele_type, allowed_attr, verbose=False +): + """Handle a field or group in yaml file.""" line_annot = f"__line__{keyword}" line_loc = dct[line_annot] xml_handle_comment(obj, line_annot, line_loc) @@ -782,13 +788,17 @@ def xml_handle_fields_or_group(dct, obj, keyword, value, ele_type, allowed_attr, r_bracket = keyword.index(")") keyword_name, keyword_type = nx_name_type_resolving(keyword) - if ele_type == 'field' and not keyword_name: - raise ValueError(f"No name for NeXus {ele_type} has been found." - f"Check around line:{line_loc}") + if ele_type == "field" and not keyword_name: + raise ValueError( + f"No name for NeXus {ele_type} has been found." + f"Check around line:{line_loc}" + ) elif not keyword_type and not keyword_name: - raise ValueError(f"No name or type for NeXus {ele_type} has been found." - f"Check around line: {line_loc}") - + raise ValueError( + f"No name or type for NeXus {ele_type} has been found." + f"Check around line: {line_loc}" + ) + elemt_obj = ET.SubElement(obj, ele_type) # type come first @@ -861,12 +871,12 @@ def xml_handle_comment( 1. Rearrange comment elements of xml_ele and xml_ele where comment comes first. 2. Append comment element when no xml_ele found as general comments. """ - + line_info = (line_annotation, int(line_loc_no)) if line_info in COMMENT_BLOCKS: # noqa: F821 cmnt = COMMENT_BLOCKS.get_comment_by_line_info(line_info) # noqa: F821 cmnt_text = cmnt.get_comment_text_list() - + # Check comment for definition element and return if is_def_cmnt: return cmnt_text @@ -880,8 +890,8 @@ def xml_handle_comment( elif not is_def_cmnt and xml_ele is None: for string in cmnt_text: obj.append(ET.Comment(string)) - - # The searched comment is not related with definition element + + # The searched comment is not related with definition element return [] @@ -889,12 +899,12 @@ def recursive_build(obj, dct, verbose): """Walk through nested dictionary. Parameters: ----------- - obj : ET.Element + obj : ET.Element Obj is the current node of the XML tree where we want to append to. dct : dict - dct is the nested python dictionary which represents the content of a child and + dct is the nested python dictionary which represents the content of a child and its successors. - + Note: NXDL fields may contain attributes but trigger no recursion so attributes are leafs. """ for keyword, value in iter(dct.items()): @@ -905,9 +915,7 @@ def recursive_build(obj, dct, verbose): keyword_name, keyword_type = nx_name_type_resolving(keyword) check_keyword_variable(verbose, dct, keyword, value) if verbose: - print( - f"keyword_name:{keyword_name} keyword_type {keyword_type}\n" - ) + print(f"keyword_name:{keyword_name} keyword_type {keyword_type}\n") if keyword[-6:] == "(link)": xml_handle_link(dct, obj, keyword, value, verbose) @@ -921,10 +929,17 @@ def recursive_build(obj, dct, verbose): elif (keyword_type in NX_CLSS) or ( keyword_type not in [*NX_TYPE_KEYS, "", *NX_NEW_DEFINED_CLASSES] ): - elem_type = 'group' + elem_type = "group" # we can be sure we need to instantiate a new group - xml_handle_fields_or_group(dct, obj, keyword, value, elem_type, - YAML_GROUP_ATTRIBUTES, verbose=False) + xml_handle_fields_or_group( + dct, + obj, + keyword, + value, + elem_type, + YAML_GROUP_ATTRIBUTES, + verbose=False, + ) elif keyword_name[0:2] == NX_ATTR_IDNT: # check if obj qualifies xml_handle_attributes(dct, obj, keyword, value, verbose) @@ -940,9 +955,16 @@ def recursive_build(obj, dct, verbose): xml_handle_exists(dct, obj, keyword, value) # Handles fileds e.g. AXISNAME elif keyword_name != "" and "__line__" not in keyword_name: - elem_type = 'field' - xml_handle_fields_or_group(dct, obj, keyword, value, elem_type, - YAML_FIELD_ATTRIBUTES, verbose=False) + elem_type = "field" + xml_handle_fields_or_group( + dct, + obj, + keyword, + value, + elem_type, + YAML_FIELD_ATTRIBUTES, + verbose=False, + ) else: raise ValueError( f"An unknown type of element {keyword} has been found which is " @@ -955,12 +977,12 @@ def extend_doc_type(doc_type, new_component, comment=False): Extend doc type to build DOM and process instruction comments. """ - start_sym = '' + start_sym = "" if comment: - start_sym = '' - return doc_type + '\n' + start_sym + new_component + end_sym + start_sym = "" + return doc_type + "\n" + start_sym + new_component + end_sym def pretty_print_xml(xml_root, output_xml, def_comments=None): @@ -979,9 +1001,13 @@ def pretty_print_xml(xml_root, output_xml, def_comments=None): doc_type = extend_doc_type(doc_type, string, comment=True) tmp_xml = "tmp.xml" - xml_string = ET.tostring(xml_root, pretty_print=True, - encoding="UTF-8", xml_declaration=True, - doctype=doc_type) + xml_string = ET.tostring( + xml_root, + pretty_print=True, + encoding="UTF-8", + xml_declaration=True, + doctype=doc_type, + ) with open(tmp_xml, "wb") as file_tmp: file_tmp.write(xml_string) flag = False @@ -1026,13 +1052,13 @@ def nyaml2nxdl(input_file: str, out_file, verbose: bool): def_cmnt_text = [] if verbose: print(f"input-file: {input_file}\n") - print( - "application/base contains the following root-level entries:\n" - ) + print("application/base contains the following root-level entries:\n") print(str(yml_appdef.keys())) # etree does not allow to set namespace-map after root creation # So, mimic a nsmap and fill it later as dict has hash property - nsmap = {None: "http://definition.nexusformat.org/nxdl/3.1",} + nsmap = { + None: "http://definition.nexusformat.org/nxdl/3.1", + } xml_root = ET.Element("definition", attrib={}, nsmap=nsmap) assert ( "category" in yml_appdef.keys() @@ -1092,11 +1118,12 @@ def nyaml2nxdl(input_file: str, out_file, verbose: bool): # Taking care of namespaces namespaces = { "xsi": "http://www.w3.org/2001/XMLSchema-instance", - } + } # Fill nsmap variable here nsmap.update(namespaces) - xml_root.attrib["{http://www.w3.org/2001/XMLSchema-instance}schemaLocation"] = \ - "http://definition.nexusformat.org/nxdl/3.1 ../nxdl.xsd".replace(' ', "%20") + xml_root.attrib[ + "{http://www.w3.org/2001/XMLSchema-instance}schemaLocation" + ] = "http://definition.nexusformat.org/nxdl/3.1 ../nxdl.xsd".replace(" ", "%20") # Taking care of Symbols elements if "symbols" in yml_appdef.keys(): diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py b/dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py index 205599df83..0388833b19 100644 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py @@ -34,17 +34,18 @@ # Yaml library does not except the keys (escape char "\t" and yaml separator ":") ESCAPE_CHAR_DICT_IN_YAML = {"\t": " "} -ESCAPE_CHAR_DICT_IN_XML = {val : key for key, val in ESCAPE_CHAR_DICT_IN_YAML.items()} +ESCAPE_CHAR_DICT_IN_XML = {val: key for key, val in ESCAPE_CHAR_DICT_IN_YAML.items()} # Set up attributes for nxdl version NXDL_GROUP_ATTRIBUTES = ( - "optional", - "recommended", - "name", - "type", - "maxOccurs", - "minOccurs", - "deprecated") + "optional", + "recommended", + "name", + "type", + "maxOccurs", + "minOccurs", + "deprecated", +) NXDL_FIELD_ATTRIBUTES = ( "optional", "recommended", @@ -74,35 +75,27 @@ "deprecated", ) -NXDL_LINK_ATTRIBUTES = ("name", - "target", - "napimount") +NXDL_LINK_ATTRIBUTES = ("name", "target", "napimount") # Set up attributes for yaml version -YAML_GROUP_ATTRIBUTES = ( - *NXDL_GROUP_ATTRIBUTES, - "exists" - ) - -YAML_FIELD_ATTRIBUTES = ( - *NXDL_FIELD_ATTRIBUTES[0:-1], - "unit", - "exists" -) +YAML_GROUP_ATTRIBUTES = (*NXDL_GROUP_ATTRIBUTES, "exists") + +YAML_FIELD_ATTRIBUTES = (*NXDL_FIELD_ATTRIBUTES[0:-1], "unit", "exists") YAML_ATTRIBUTES_ATTRIBUTES = ( - *NXDL_ATTRIBUTES_ATTRIBUTES, - "minOccurs", - "maxOccurs", - "exists", + *NXDL_ATTRIBUTES_ATTRIBUTES, + "minOccurs", + "maxOccurs", + "exists", ) YAML_LINK_ATTRIBUTES = NXDL_LINK_ATTRIBUTES + def remove_namespace_from_tag(tag): """Helper function to remove the namespace from an XML tag.""" - if isinstance(tag, Callable) and tag.__name__ == 'Comment': - return '!--' + if isinstance(tag, Callable) and tag.__name__ == "Comment": + return "!--" else: return tag.split("}")[-1] @@ -136,7 +129,7 @@ def construct_mapping(self, node, deep=False): ) node_pair_lst_for_appending.append((shadow_key_node, shadow_value_node)) - node.value = node.value + node_pair_lst_for_appending + node.value = node.value + node_pair_lst_for_appending return Constructor.construct_mapping(self, node, deep=deep) @@ -183,7 +176,7 @@ def clean_empty_lines(line_list): ends_non_empty_line = None # Find the index of first non-empty line for ind, line in enumerate(line_list): - if len(line.strip()) > 1 : + if len(line.strip()) > 1: start_non_empty_line = ind break @@ -193,9 +186,9 @@ def clean_empty_lines(line_list): ends_non_empty_line = -ind break - if ends_non_empty_line == 0 : + if ends_non_empty_line == 0: ends_non_empty_line = None - return line_list[start_non_empty_line : ends_non_empty_line] + return line_list[start_non_empty_line:ends_non_empty_line] def nx_name_type_resolving(tmp): @@ -211,9 +204,13 @@ def nx_name_type_resolving(tmp): index_start = tmp.index("(") index_end = tmp.index(")", index_start + 1) if index_start > index_end: - raise ValueError(f"Check name and type combination {tmp} which can not be resolved.") + raise ValueError( + f"Check name and type combination {tmp} which can not be resolved." + ) if index_end - index_start == 1: - raise ValueError(f"Check name(type) combination {tmp}, properly not defined.") + raise ValueError( + f"Check name(type) combination {tmp}, properly not defined." + ) typ = tmp[index_start + 1 : index_end] nam = tmp.replace("(" + typ + ")", "") return nam, typ @@ -273,7 +270,7 @@ def separate_hash_yaml_and_nxdl(yaml_file, sep_yaml, sep_xml): sep_xml, "w", encoding="utf-8" ) as xml_f_ob: write_on_yaml = True - + last_line = lines[0] for line in lines[1:]: # Write in file when ensured that the next line is not with '++ SHA HASH ++' @@ -299,20 +296,20 @@ def separate_hash_yaml_and_nxdl(yaml_file, sep_yaml, sep_xml): def is_dom_comment(text): """Analyze a comment, whether it is a dom comment or not. - + Return true if dom comment. """ - + # some signature keywords to distingush dom comments from other comments. signature_keyword_list = [ - "NeXus", - "GNU Lesser General Public", - "Free Software Foundation", - "Copyright (C)", - "WITHOUT ANY WARRANTY", + "NeXus", + "GNU Lesser General Public", + "Free Software Foundation", + "Copyright (C)", + "WITHOUT ANY WARRANTY", ] for keyword in signature_keyword_list: if keyword not in text: return False - + return True diff --git a/dev_tools/utils/nxdl_utils.py b/dev_tools/utils/nxdl_utils.py index 6f39563af2..561b1595ee 100644 --- a/dev_tools/utils/nxdl_utils.py +++ b/dev_tools/utils/nxdl_utils.py @@ -19,9 +19,11 @@ class NxdlAttributeNotFoundError(Exception): def get_app_defs_names(): """Returns all the AppDef names without their extension: .nxdl.xml""" app_def_path_glob = get_nexus_definitions_path() / "applications" / "*.nxdl*" - - contrib_def_path_glob = Path(get_nexus_definitions_path()) / "contributed_definitions" / "*.nxdl*" - + + contrib_def_path_glob = ( + Path(get_nexus_definitions_path()) / "contributed_definitions" / "*.nxdl*" + ) + files = sorted(glob(app_def_path_glob)) for nexus_file in sorted(contrib_def_path_glob): root = get_xml_root(nexus_file) @@ -128,7 +130,9 @@ def get_nx_classes(): nexus_definition_path = get_nexus_definitions_path() base_classes = sorted(nexus_definition_path.glob("base_classes/*.nxdl.xml")) applications = sorted(nexus_definition_path.glob("applications/*.nxdl.xml")) - contributed = sorted(nexus_definition_path.glob("contributed_definitions/*.nxdl.xml")) + contributed = sorted( + nexus_definition_path.glob("contributed_definitions/*.nxdl.xml") + ) nx_class = [] for nexus_file in base_classes + applications + contributed: try: @@ -156,14 +160,14 @@ def get_nx_units(): nx_units.append(line) elif line == "primitiveType": flag = False - + return nx_units def get_nx_attribute_type(): """Read attribute types from the NeXus definition/nxdlTypes.xsd file""" filepath = get_nexus_definitions_path() / "nxdlTypes.xsd" - + root = get_xml_root(filepath) units_and_type_list = [] for child in root: @@ -314,11 +318,9 @@ def find_definition_file(bc_name): nexus_def_path = get_nexus_definitions_path() bc_filename = None for nxdl_folder in ["contributed_definitions", "base_classes", "applications"]: - nxdl_file = nexus_def_path / nxdl_folder/ f"{bc_name}.nxdl.xml" + nxdl_file = nexus_def_path / nxdl_folder / f"{bc_name}.nxdl.xml" if nxdl_file.exists(): - bc_filename = ( - nexus_def_path / nxdl_folder / f"{bc_name}.nxdl.xml" - ) + bc_filename = nexus_def_path / nxdl_folder / f"{bc_name}.nxdl.xml" break return bc_filename @@ -619,8 +621,7 @@ def add_base_classes(elist, nx_name=None, elem: ET.Element = None): def set_nxdlpath(child, nxdl_elem, tag_name=None): - """Setting up child nxdlbase, nxdlpath and nxdlbase_class from nxdl_element. - """ + """Setting up child nxdlbase, nxdlpath and nxdlbase_class from nxdl_element.""" if nxdl_elem.get("nxdlbase"): child.set("nxdlbase", nxdl_elem.get("nxdlbase")) child.set("nxdlbase_class", nxdl_elem.get("nxdlbase_class")) @@ -628,7 +629,9 @@ def set_nxdlpath(child, nxdl_elem, tag_name=None): if tag_name: child.set("nxdlpath", nxdl_elem.get("nxdlpath") + "/" + tag_name) else: - child.set("nxdlpath", nxdl_elem.get("nxdlpath") + "/" + get_node_name(child)) + child.set( + "nxdlpath", nxdl_elem.get("nxdlpath") + "/" + get_node_name(child) + ) return child From 55ee9ad5046889bc6b1f3521cbed8b580df1d1ff Mon Sep 17 00:00:00 2001 From: Rubel Date: Thu, 5 Oct 2023 16:07:36 +0200 Subject: [PATCH 82/93] Fixing test. --- dev_tools/tests/test_nyaml2nxdl.py | 6 ++++-- dev_tools/utils/nxdl_utils.py | 19 +++++++++++++++---- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/dev_tools/tests/test_nyaml2nxdl.py b/dev_tools/tests/test_nyaml2nxdl.py index 6d41d1085e..5423b2aea5 100755 --- a/dev_tools/tests/test_nyaml2nxdl.py +++ b/dev_tools/tests/test_nyaml2nxdl.py @@ -10,10 +10,12 @@ def test_conversion(): root = find_definition_file("NXentry") result = CliRunner().invoke(conv.launch_tool, ["--input-file", root]) assert result.exit_code == 0 - yaml = root[:-9] + "_parsed.yaml" + # Replace suffixes + yaml = root.with_suffix('').with_suffix('.yaml') + yaml = yaml.with_stem(yaml.stem + "_parsed") result = CliRunner().invoke(conv.launch_tool, ["--input-file", yaml]) assert result.exit_code == 0 - new_root = yaml[:-4] + "nxdl.xml" + new_root = yaml.with_suffix(".nxdl.xml") with open(root, encoding="utf-8", mode="r") as tmp_f: root_content = tmp_f.readlines() with open(new_root, encoding="utf-8", mode="r") as tmp_f: diff --git a/dev_tools/utils/nxdl_utils.py b/dev_tools/utils/nxdl_utils.py index 561b1595ee..31c1b08a2c 100644 --- a/dev_tools/utils/nxdl_utils.py +++ b/dev_tools/utils/nxdl_utils.py @@ -8,10 +8,12 @@ import textwrap import lxml.etree as ET from lxml.etree import ParseError as xmlER +from ..nyaml2nxdl.nyaml2nxdl_helper import remove_namespace_from_tag from functools import lru_cache from glob import glob + class NxdlAttributeNotFoundError(Exception): """An exception to throw when an Nxdl attribute is not found.""" @@ -90,7 +92,7 @@ def get_hdf_info_parent(hdf_info): def get_nx_class(nxdl_elem): """Get the nexus class for a NXDL node""" if "category" in nxdl_elem.attrib: - return None + return "" return nxdl_elem.attrib.get("type", "NX_CHAR") @@ -250,7 +252,7 @@ def belongs_to_capital(params): def get_local_name_from_xml(element): """Helper function to extract the element tag without the namespace.""" - return element.tag.rsplit("}")[-1] + return remove_namespace_from_tag(element.tag) def get_own_nxdl_child_reserved_elements(child, name, nxdl_elem): @@ -605,7 +607,16 @@ def add_base_classes(elist, nx_name=None, elem: ET.Element = None): nxdl_file_path = find_definition_file(nx_name) if nxdl_file_path is None: nxdl_file_path = f"{nx_name}.nxdl.xml" - elem = ET.parse(nxdl_file_path).getroot() + + try: + elem = ET.parse(os.path.abspath(nxdl_file_path)).getroot() + # elem = ET.parse(nxdl_file_path).getroot() + except OSError: + with open(nxdl_file_path, 'r') as f: + elem = ET.parse(f).getroot() + + if not isinstance(nxdl_file_path, str): + nxdl_file_path = str(nxdl_file_path) elem.set("nxdlbase", nxdl_file_path) else: elem.set("nxdlbase", "") @@ -756,7 +767,7 @@ def walk_elist(elist, html_name): @lru_cache(maxsize=None) -def get_inherited_nodes( +def get_inherited_nodes( ### Make it compatible for pathlib nxdl_path: str = None, # pylint: disable=too-many-arguments,too-many-locals nx_name: str = None, elem: ET.Element = None, From cf7f0683fe64ed5c4866699c24695153e5ffec6b Mon Sep 17 00:00:00 2001 From: Rubel Date: Thu, 5 Oct 2023 19:19:12 +0200 Subject: [PATCH 83/93] Fixing build errors. --- dev_tools/utils/nxdl_utils.py | 39 +++++++++++++++++------------------ 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/dev_tools/utils/nxdl_utils.py b/dev_tools/utils/nxdl_utils.py index 31c1b08a2c..9fdc4b9e78 100644 --- a/dev_tools/utils/nxdl_utils.py +++ b/dev_tools/utils/nxdl_utils.py @@ -13,17 +13,29 @@ from glob import glob - class NxdlAttributeNotFoundError(Exception): """An exception to throw when an Nxdl attribute is not found.""" +def get_nexus_definitions_path(): + """Check NEXUS_DEF_PATH variable. + If it is empty, this function is filling it""" + try: # either given by sys env + return os.environ["NEXUS_DEF_PATH"] + except KeyError: # or it should be available locally under the dir 'definitions' + local_dir = Path(__file__).resolve().parent + for _ in range(2): + local_dir = local_dir.parent + return local_dir + +nexus_def_path = get_nexus_definitions_path() + def get_app_defs_names(): """Returns all the AppDef names without their extension: .nxdl.xml""" - app_def_path_glob = get_nexus_definitions_path() / "applications" / "*.nxdl*" + app_def_path_glob = nexus_def_path / "applications" / "*.nxdl*" contrib_def_path_glob = ( - Path(get_nexus_definitions_path()) / "contributed_definitions" / "*.nxdl*" + Path(nexus_def_path) / "contributed_definitions" / "*.nxdl*" ) files = sorted(glob(app_def_path_glob)) @@ -42,18 +54,6 @@ def get_xml_root(file_path): return ET.parse(file_path).getroot() -def get_nexus_definitions_path(): - """Check NEXUS_DEF_PATH variable. - If it is empty, this function is filling it""" - try: # either given by sys env - return os.environ["NEXUS_DEF_PATH"] - except KeyError: # or it should be available locally under the dir 'definitions' - local_dir = Path(__file__).resolve().parent - for _ in range(2): - local_dir = local_dir.parent - return local_dir - - def get_hdf_root(hdf_node): """Get the root HDF5 node""" node = hdf_node @@ -129,7 +129,7 @@ def get_nx_classes(): """Read base classes from the NeXus definition folder. Check each file in base_classes, applications, contributed_definitions. If its category attribute is 'base', then it is added to the list.""" - nexus_definition_path = get_nexus_definitions_path() + nexus_definition_path = nexus_def_path base_classes = sorted(nexus_definition_path.glob("base_classes/*.nxdl.xml")) applications = sorted(nexus_definition_path.glob("applications/*.nxdl.xml")) contributed = sorted( @@ -148,7 +148,7 @@ def get_nx_classes(): def get_nx_units(): """Read unit kinds from the NeXus definition/nxdlTypes.xsd file""" - filepath = get_nexus_definitions_path() / "nxdlTypes.xsd" + filepath = nexus_def_path / "nxdlTypes.xsd" root = get_xml_root(filepath) units_and_type_list = [] for child in root: @@ -168,7 +168,7 @@ def get_nx_units(): def get_nx_attribute_type(): """Read attribute types from the NeXus definition/nxdlTypes.xsd file""" - filepath = get_nexus_definitions_path() / "nxdlTypes.xsd" + filepath = nexus_def_path / "nxdlTypes.xsd" root = get_xml_root(filepath) units_and_type_list = [] @@ -317,7 +317,6 @@ def find_definition_file(bc_name): """find the nxdl file corresponding to the name. Note that it first checks in contributed and goes beyond only if no contributed found """ - nexus_def_path = get_nexus_definitions_path() bc_filename = None for nxdl_folder in ["contributed_definitions", "base_classes", "applications"]: nxdl_file = nexus_def_path / nxdl_folder / f"{bc_name}.nxdl.xml" @@ -767,7 +766,7 @@ def walk_elist(elist, html_name): @lru_cache(maxsize=None) -def get_inherited_nodes( ### Make it compatible for pathlib +def get_inherited_nodes( nxdl_path: str = None, # pylint: disable=too-many-arguments,too-many-locals nx_name: str = None, elem: ET.Element = None, From ae0bd30c19813480df91e18543595a88f723c910 Mon Sep 17 00:00:00 2001 From: domna Date: Fri, 20 Oct 2023 18:10:30 +0200 Subject: [PATCH 84/93] Remove pyproject.toml and adapt makefile --- Makefile | 8 ++++---- pyproject.toml | 42 ------------------------------------------ 2 files changed, 4 insertions(+), 46 deletions(-) delete mode 100644 pyproject.toml diff --git a/Makefile b/Makefile index 8e92bbbf54..2b3ff995a9 100644 --- a/Makefile +++ b/Makefile @@ -97,22 +97,22 @@ all :: @echo "PDF built: `ls -lAFgh $(BUILD_DIR)/manual/build/latex/nexus.pdf`" $(BASE_CLASS_DIR)/%.nxdl.xml : $(BASE_CLASS_DIR)/$(NYAML_SUBDIR)/%.yaml - nyaml2nxdl --input-file $< + $(PYTHON) -m dev_tools.nyaml2nxdl.nyaml2nxdl --input-file $< mv $(BASE_CLASS_DIR)/$(NYAML_SUBDIR)/$*.nxdl.xml $@ $(CONTRIB_DIR)/%.nxdl.xml : $(CONTRIB_DIR)/$(NYAML_SUBDIR)/%.yaml - nyaml2nxdl --input-file $< + $(PYTHON) -m dev_tools.nyaml2nxdl.nyaml2nxdl --input-file $< mv $(CONTRIB_DIR)/$(NYAML_SUBDIR)/$*.nxdl.xml $@ $(APPDEF_DIR)/%.nxdl.xml : $(APPDEF_DIR)/$(NYAML_SUBDIR)/%.yaml - nyaml2nxdl --input-file $< + $(PYTHON) -m dev_tools.nyaml2nxdl.nyaml2nxdl --input-file $< mv $(APPDEF_DIR)/$(NYAML_SUBDIR)/$*.nxdl.xml $@ NXDLS := $(foreach dir,$(NXDL_DIRS),$(wildcard $(dir)/*.nxdl.xml)) nyaml : $(DIRS) for file in $(NXDLS); do\ mkdir -p "$${file%/*}/nyaml";\ - nyaml2nxdl --input-file $${file};\ + $(PYTHON) -m dev_tools.nyaml2nxdl.nyaml2nxdl --input-file $${file};\ FNAME=$${file##*/};\ mv -- "$${file%.nxdl.xml}_parsed.yaml" "$${file%/*}/nyaml/$${FNAME%.nxdl.xml}.yaml";\ done diff --git a/pyproject.toml b/pyproject.toml deleted file mode 100644 index b433177e67..0000000000 --- a/pyproject.toml +++ /dev/null @@ -1,42 +0,0 @@ -[build-system] -requires = ["setuptools>=64.0.1", "setuptools-scm[toml]>=6.2"] -build-backend = "setuptools.build_meta" - -[project] -name = "nexusdefinitions" -dynamic = ["version"] -authors = [ - { name = "NIAC" } -] -description = "Nexus definitions" -readme = "README.md" -license = { file = "LGPL.txt" } -requires-python = "" -classifiers = [ - "Operating System :: OS Independent" -] -dependencies = [ - "lxml", - "pyyaml", - "click>=7.1.2", - "sphinx>=5", - "sphinx-tabs", - "sphinx-toolbox", - "pytest", - "black>=22.3", - "flake8>=4", - "isort>=5.10", -] - -[project.urls] -"Homepage" = "https://nexusformat.org" - -[project.scripts] -nyaml2nxdl = "dev_tools.nyaml2nxdl.nyaml2nxdl:launch_tool" - -[tool.setuptools_scm] -version_scheme = "guess-next-dev" -local_scheme = "node-and-date" - -[tool.setuptools] -packages = ["dev_tools"] From afabe9d151e9dab211a42a81081550c15086a8b3 Mon Sep 17 00:00:00 2001 From: domna Date: Wed, 25 Oct 2023 07:28:49 +0200 Subject: [PATCH 85/93] Remove $(DIRS) from nyaml command --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 2b3ff995a9..624931d6ff 100644 --- a/Makefile +++ b/Makefile @@ -109,7 +109,7 @@ $(APPDEF_DIR)/%.nxdl.xml : $(APPDEF_DIR)/$(NYAML_SUBDIR)/%.yaml mv $(APPDEF_DIR)/$(NYAML_SUBDIR)/$*.nxdl.xml $@ NXDLS := $(foreach dir,$(NXDL_DIRS),$(wildcard $(dir)/*.nxdl.xml)) -nyaml : $(DIRS) +nyaml : for file in $(NXDLS); do\ mkdir -p "$${file%/*}/nyaml";\ $(PYTHON) -m dev_tools.nyaml2nxdl.nyaml2nxdl --input-file $${file};\ From 31e836db43be7e3cabb9d08d1fe1e60df4a8569a Mon Sep 17 00:00:00 2001 From: domna Date: Wed, 25 Oct 2023 07:36:30 +0200 Subject: [PATCH 86/93] Updates a warning for using make nxdl --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 0fd9beb45b..68d3af845c 100755 --- a/README.md +++ b/README.md @@ -37,6 +37,9 @@ After editing the definitions in nyaml format in the nyaml subdirectories, the f make nxdl +> [!WARNING] +> Please be aware that your nyaml files might be out of sync with the nxdl files when you update your repo. So it's best practice to stash your changes to the nyaml files and regenerate the files with `make nyaml` before adding any changes. + ### HTML pages with contributor information To build the html pages that contains contributor information on the sidebar set a github access token to an environment variable called GH_TOKEN. From 89afa8e02ae4f874110c79f4084c55959be5224e Mon Sep 17 00:00:00 2001 From: domna Date: Wed, 25 Oct 2023 08:06:06 +0200 Subject: [PATCH 87/93] Fix codestyle --- dev_tools/docs/nxdl.py | 8 +++--- dev_tools/nyaml2nxdl/comment_collector.py | 26 ++++++++----------- dev_tools/nyaml2nxdl/nyaml2nxdl.py | 2 +- .../nyaml2nxdl/nyaml2nxdl_backward_tools.py | 20 +++++++------- .../nyaml2nxdl/nyaml2nxdl_forward_tools.py | 20 +++++++------- dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py | 8 +++--- dev_tools/tests/test_nyaml2nxdl.py | 2 +- dev_tools/utils/nxdl_utils.py | 19 +++++++------- 8 files changed, 49 insertions(+), 56 deletions(-) diff --git a/dev_tools/docs/nxdl.py b/dev_tools/docs/nxdl.py index 26c53cb208..c9e3ebed1c 100755 --- a/dev_tools/docs/nxdl.py +++ b/dev_tools/docs/nxdl.py @@ -668,8 +668,8 @@ def _print(self, *args, end="\n"): def get_first_parent_ref(self, path, tag): spliter = path.find("/", 1) - nx_name = path[1 : spliter] - path = path[spliter :] + nx_name = path[1:spliter] + path = path[spliter:] try: parents = pynxtools_nxlib.get_inherited_nodes(path, nx_name)[2] @@ -680,9 +680,7 @@ def get_first_parent_ref(self, path, tag): parent_path = parent_display_name = parent.attrib["nxdlpath"] parent_path_segments = parent_path[1:].split("/") nxdl_attr = parent.attrib["nxdlbase"] - parent_def_name = nxdl_attr[nxdl_attr.rfind("/") : - nxdl_attr.rfind(".nxdl") - ] + parent_def_name = nxdl_attr[nxdl_attr.rfind("/") : nxdl_attr.rfind(".nxdl")] # Case where the first parent is a base_class if parent_path_segments[0] == "": diff --git a/dev_tools/nyaml2nxdl/comment_collector.py b/dev_tools/nyaml2nxdl/comment_collector.py index 7b63ececfa..6ded2f3720 100644 --- a/dev_tools/nyaml2nxdl/comment_collector.py +++ b/dev_tools/nyaml2nxdl/comment_collector.py @@ -19,8 +19,8 @@ # """ -Collect comments in a list by CommentCollector class and each comment is an instance of Comment -class. Each `Comment` instance(sometimes refered as 'comment block') consists of text and line +Collect comments in a list by CommentCollector class and each comment is an instance of Comment +class. Each `Comment` instance(sometimes refered as 'comment block') consists of text and line info or neighbours info where the comment must be placed. The class Comment is an abstract class for general functions or method to be implemented @@ -67,9 +67,7 @@ def __init__(self, input_file: str = None, loaded_obj: Union[object, Dict] = Non loader = LineLoader(plain_text_yaml) self.comment.__yaml_dict__ = loader.get_single_data() else: - raise ValueError( - "Input file must be a 'yaml' or 'nxdl.xml' type." - ) + raise ValueError("Input file must be a 'yaml' or 'nxdl.xml' type.") elif self.file and loaded_obj: if self.file.split(".")[-1] == "yaml" and isinstance(loaded_obj, dict): self.comment = YAMLComment @@ -83,8 +81,8 @@ def __init__(self, input_file: str = None, loaded_obj: Union[object, Dict] = Non raise ValueError("Incorrect inputs for CommentCollector") def extract_all_comment_blocks(self): - """Collect all comments. - + """Collect all comments. + Note here that comment means (comment text + element or line info intended for comment. """ @@ -101,9 +99,7 @@ def extract_all_comment_blocks(self): # If the last comment comes without post nxdl fields, groups and attributes if "++ SHA HASH ++" in line: # Handle with stored nxdl.xml file that is not part of yaml - single_comment.process_each_line( - "post_comment", (line_num + 1) - ) + single_comment.process_each_line("post_comment", (line_num + 1)) self._comment_chain.append(single_comment) break if line_num < end_line_num: @@ -450,8 +446,7 @@ def append_comment(self, text: str) -> None: # pylint: disable=arguments-differ def store_element(self, line_key, line_number): - """Store comment contents and information of comment location. - """ + """Store comment contents and information of comment location.""" self._elemt = {} self._elemt[line_key] = int(line_number) self._is_elemt_found = False @@ -483,10 +478,11 @@ def replace_escape_char(self, text): return text def get_element_location(self): - """Return yaml line '__line__' info and and line numner - """ + """Return yaml line '__line__' info and and line numner""" if len(self._elemt) == 0: - raise ValueError(f"Comment element should be one or more but got {self._elemt}") + raise ValueError( + f"Comment element should be one or more but got {self._elemt}" + ) return next(self._elemt.items()) def collect_yaml_line_info(self, yaml_dict, line_info_dict): diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl.py b/dev_tools/nyaml2nxdl/nyaml2nxdl.py index 8ee97c380a..4031d6bbdf 100755 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl.py @@ -3,7 +3,7 @@ Main file of nyaml2nxdl tool. To write a definition for a instrument, experiment and/or process in nxdl.xml file from a YAML -file which details a hierarchy of data/metadata elements. It also allows both way +file which details a hierarchy of data/metadata elements. It also allows both wa conversion beteen YAML and nxdl.xml files that follows rules of NeXus ontology or data format. """ # -*- coding: utf-8 -*- diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py b/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py index 3582b1fee6..caf79819e7 100755 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py @@ -21,22 +21,22 @@ # limitations under the License. # -from typing import Dict, Callable -from typing import List from pathlib import Path +from typing import Callable +from typing import Dict +from typing import List + import lxml.etree as ET +from .nyaml2nxdl_helper import NXDL_ATTRIBUTES_ATTRIBUTES +from .nyaml2nxdl_helper import NXDL_FIELD_ATTRIBUTES +from .nyaml2nxdl_helper import NXDL_GROUP_ATTRIBUTES +from .nyaml2nxdl_helper import NXDL_LINK_ATTRIBUTES from .nyaml2nxdl_helper import clean_empty_lines from .nyaml2nxdl_helper import get_node_parent_info from .nyaml2nxdl_helper import get_yaml_escape_char_dict -from .nyaml2nxdl_helper import remove_namespace_from_tag, is_dom_comment -from .nyaml2nxdl_helper import ( - NXDL_FIELD_ATTRIBUTES, - NXDL_GROUP_ATTRIBUTES, - NXDL_ATTRIBUTES_ATTRIBUTES, - NXDL_LINK_ATTRIBUTES, -) - +from .nyaml2nxdl_helper import is_dom_comment +from .nyaml2nxdl_helper import remove_namespace_from_tag DEPTH_SIZE = " " CMNT_TAG = "!--" diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py b/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py index afde294926..600c1978a3 100644 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl_forward_tools.py @@ -21,29 +21,27 @@ # limitations under the License. # +import datetime import pathlib import textwrap -import datetime -import lxml.etree as ET -from urllib.parse import unquote import warnings +from urllib.parse import unquote +import lxml.etree as ET import yaml from ..utils import nxdl_utils as pynxtools_nxlib from .comment_collector import CommentCollector +from .nyaml2nxdl_helper import YAML_ATTRIBUTES_ATTRIBUTES +from .nyaml2nxdl_helper import YAML_FIELD_ATTRIBUTES +from .nyaml2nxdl_helper import YAML_GROUP_ATTRIBUTES +from .nyaml2nxdl_helper import YAML_LINK_ATTRIBUTES from .nyaml2nxdl_helper import LineLoader from .nyaml2nxdl_helper import clean_empty_lines from .nyaml2nxdl_helper import get_yaml_escape_char_reverter_dict +from .nyaml2nxdl_helper import is_dom_comment from .nyaml2nxdl_helper import nx_name_type_resolving -from .nyaml2nxdl_helper import ( - YAML_ATTRIBUTES_ATTRIBUTES, - YAML_FIELD_ATTRIBUTES, - YAML_GROUP_ATTRIBUTES, - YAML_LINK_ATTRIBUTES, -) -from .nyaml2nxdl_helper import remove_namespace_from_tag, is_dom_comment - +from .nyaml2nxdl_helper import remove_namespace_from_tag # pylint: disable=too-many-lines, global-statement, invalid-name DOM_COMMENT = ( diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py b/dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py index 0388833b19..7587df1de1 100644 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl_helper.py @@ -24,13 +24,13 @@ # import hashlib +from typing import Callable from yaml.composer import Composer from yaml.constructor import Constructor from yaml.loader import Loader from yaml.nodes import ScalarNode from yaml.resolver import BaseResolver -from typing import Callable # Yaml library does not except the keys (escape char "\t" and yaml separator ":") ESCAPE_CHAR_DICT_IN_YAML = {"\t": " "} @@ -254,9 +254,9 @@ def separate_hash_yaml_and_nxdl(yaml_file, sep_yaml, sep_xml): """Separate yaml, SHA hash and nxdl parts. Separate the provided yaml file into yaml, nxdl and hash if yaml was extended with - nxdl at the end of yaml as - - + nxdl at the end of yaml as + + '\n# ++++++++++++++++++++++++++++++++++ SHA HASH \ ++++++++++++++++++++++++++++++++++\n' # ' diff --git a/dev_tools/tests/test_nyaml2nxdl.py b/dev_tools/tests/test_nyaml2nxdl.py index 5423b2aea5..14d0127219 100755 --- a/dev_tools/tests/test_nyaml2nxdl.py +++ b/dev_tools/tests/test_nyaml2nxdl.py @@ -11,7 +11,7 @@ def test_conversion(): result = CliRunner().invoke(conv.launch_tool, ["--input-file", root]) assert result.exit_code == 0 # Replace suffixes - yaml = root.with_suffix('').with_suffix('.yaml') + yaml = root.with_suffix("").with_suffix(".yaml") yaml = yaml.with_stem(yaml.stem + "_parsed") result = CliRunner().invoke(conv.launch_tool, ["--input-file", yaml]) assert result.exit_code == 0 diff --git a/dev_tools/utils/nxdl_utils.py b/dev_tools/utils/nxdl_utils.py index 9fdc4b9e78..5c6d3db458 100644 --- a/dev_tools/utils/nxdl_utils.py +++ b/dev_tools/utils/nxdl_utils.py @@ -3,14 +3,15 @@ """ import os - -from pathlib import Path import textwrap +from functools import lru_cache +from glob import glob +from pathlib import Path + import lxml.etree as ET from lxml.etree import ParseError as xmlER + from ..nyaml2nxdl.nyaml2nxdl_helper import remove_namespace_from_tag -from functools import lru_cache -from glob import glob class NxdlAttributeNotFoundError(Exception): @@ -28,15 +29,15 @@ def get_nexus_definitions_path(): local_dir = local_dir.parent return local_dir + nexus_def_path = get_nexus_definitions_path() + def get_app_defs_names(): """Returns all the AppDef names without their extension: .nxdl.xml""" app_def_path_glob = nexus_def_path / "applications" / "*.nxdl*" - contrib_def_path_glob = ( - Path(nexus_def_path) / "contributed_definitions" / "*.nxdl*" - ) + contrib_def_path_glob = Path(nexus_def_path) / "contributed_definitions" / "*.nxdl*" files = sorted(glob(app_def_path_glob)) for nexus_file in sorted(contrib_def_path_glob): @@ -611,7 +612,7 @@ def add_base_classes(elist, nx_name=None, elem: ET.Element = None): elem = ET.parse(os.path.abspath(nxdl_file_path)).getroot() # elem = ET.parse(nxdl_file_path).getroot() except OSError: - with open(nxdl_file_path, 'r') as f: + with open(nxdl_file_path, "r") as f: elem = ET.parse(f).getroot() if not isinstance(nxdl_file_path, str): @@ -766,7 +767,7 @@ def walk_elist(elist, html_name): @lru_cache(maxsize=None) -def get_inherited_nodes( +def get_inherited_nodes( nxdl_path: str = None, # pylint: disable=too-many-arguments,too-many-locals nx_name: str = None, elem: ET.Element = None, From 56f51910c3120f7c8b99e5bf2220b69ebb5b20c2 Mon Sep 17 00:00:00 2001 From: domna Date: Wed, 25 Oct 2023 13:50:27 +0200 Subject: [PATCH 88/93] Build nxdl names from existing variables --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 624931d6ff..dc2893023a 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,6 @@ BUILD_DIR = "build" BASE_CLASS_DIR := base_classes CONTRIB_DIR := contributed_definitions APPDEF_DIR := applications -NXDL_DIRS := $(BASE_CLASS_DIR) $(CONTRIB_DIR) $(APPDEF_DIR) NYAML_SUBDIR := nyaml NYAML_APPENDIX := _parsed @@ -108,7 +107,7 @@ $(APPDEF_DIR)/%.nxdl.xml : $(APPDEF_DIR)/$(NYAML_SUBDIR)/%.yaml $(PYTHON) -m dev_tools.nyaml2nxdl.nyaml2nxdl --input-file $< mv $(APPDEF_DIR)/$(NYAML_SUBDIR)/$*.nxdl.xml $@ -NXDLS := $(foreach dir,$(NXDL_DIRS),$(wildcard $(dir)/*.nxdl.xml)) +NXDLS := $(NXDL_APPDEF) + $(NXDL_CONTRIB) + $(NXDL_BC) nyaml : for file in $(NXDLS); do\ mkdir -p "$${file%/*}/nyaml";\ From e616851a1cbd510f29f29c1d200bcd385c96be10 Mon Sep 17 00:00:00 2001 From: Rubel Date: Sat, 28 Oct 2023 10:47:25 +0200 Subject: [PATCH 89/93] Fixing indentation miss alignment in comments commeing from nxdl to nyaml. --- dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py b/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py index caf79819e7..f6f543713d 100755 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl_backward_tools.py @@ -378,6 +378,7 @@ def convert_to_yaml_comment(self, depth, text): """ # To store indentation text from comment lines = self.clean_and_organise_text(text, depth).split("\n") + indent = DEPTH_SIZE * depth mod_lines = [] for line in lines: line = line.strip() @@ -386,7 +387,7 @@ def convert_to_yaml_comment(self, depth, text): line = "# " + line mod_lines.append(line) # The starting '\n' to keep multiple comments separate - return "\n" + "\n".join(mod_lines) + return "\n" + indent + "\n".join(mod_lines) def print_root_level_info(self, depth, file_out): """ From ebc61c2c1105e2819957b794ac0297539c6a17f4 Mon Sep 17 00:00:00 2001 From: Rubel Date: Fri, 3 Nov 2023 11:09:32 +0100 Subject: [PATCH 90/93] Fixing test for python backward compatibility. --- dev_tools/tests/test_nyaml2nxdl.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/dev_tools/tests/test_nyaml2nxdl.py b/dev_tools/tests/test_nyaml2nxdl.py index 14d0127219..1e68305e3a 100755 --- a/dev_tools/tests/test_nyaml2nxdl.py +++ b/dev_tools/tests/test_nyaml2nxdl.py @@ -1,5 +1,3 @@ -import os - from click.testing import CliRunner from ..nyaml2nxdl import nyaml2nxdl as conv @@ -8,18 +6,18 @@ def test_conversion(): root = find_definition_file("NXentry") - result = CliRunner().invoke(conv.launch_tool, ["--input-file", root]) + result = CliRunner().invoke(conv.launch_tool, ["--input-file", str(root)]) assert result.exit_code == 0 # Replace suffixes - yaml = root.with_suffix("").with_suffix(".yaml") - yaml = yaml.with_stem(yaml.stem + "_parsed") - result = CliRunner().invoke(conv.launch_tool, ["--input-file", yaml]) + yaml = root.with_suffix("").with_suffix(".yaml") # replace .nxdl.xml + yaml = yaml.with_name(yaml.stem + "_parsed.yaml") # extend file name with _parsed + result = CliRunner().invoke(conv.launch_tool, ["--input-file", str(yaml)]) assert result.exit_code == 0 - new_root = yaml.with_suffix(".nxdl.xml") + new_root = yaml.with_suffix(".nxdl.xml") # replace yaml with open(root, encoding="utf-8", mode="r") as tmp_f: root_content = tmp_f.readlines() with open(new_root, encoding="utf-8", mode="r") as tmp_f: new_root_content = tmp_f.readlines() assert root_content == new_root_content - os.remove(yaml) - os.remove(new_root) + yaml.unlink() + new_root.unlink() From 8b64f95fef9dca2739d361265ed63be28a6d326d Mon Sep 17 00:00:00 2001 From: sanbrock <45483558+sanbrock@users.noreply.github.com> Date: Wed, 8 Nov 2023 19:19:30 +0100 Subject: [PATCH 91/93] Update README.md Language is cleaned as requested by prjamin in the PR. --- dev_tools/nyaml2nxdl/README.md | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/dev_tools/nyaml2nxdl/README.md b/dev_tools/nyaml2nxdl/README.md index 4b316fc5d0..e0ff6b5ab2 100644 --- a/dev_tools/nyaml2nxdl/README.md +++ b/dev_tools/nyaml2nxdl/README.md @@ -2,21 +2,17 @@ **NOTE: Please use python3.8 or above to run this converter** -**Tools purpose**: Offer a simple YAML-based schema and a XML-based schema to describe NeXus instances. These can be NeXus application definitions or classes -such as base or contributed classes. Users either create NeXus instances by writing a YAML file or a XML file which details a hierarchy of data/metadata elements. -The forward (YAML -> NXDL.XML) and backward (NXDL.XML -> YAML) conversions are implemented. +**Tools purpose**: Offer a simple way to edit NeXus definitions in YAML format. These can be NeXus application definitions or base classes. +Both forward (YAML -> NXDL.XML) and backward (NXDL.XML -> YAML) conversions are implemented. **How the tool works**: - nyaml2nxdl.py -1. Reads the user-specified NeXus instance, either in YML or XML format. -2. If input is in YAML, creates an instantiated NXDL schema XML tree by walking the dictionary nest. - If input is in XML, creates a YML file walking the dictionary nest. -3. Write the tree into a YAML file or a properly formatted NXDL XML schema file to disk. -4. Optionally, if --append argument is given, - the XML or YAML input file is interpreted as an extension of a base class and the entries contained in it - are appended below a standard NeXus base class. - You need to specify both your input file (with YAML or XML extension) and NeXus class (with no extension). - Both .yaml and .nxdl.xml file of the extended class are printed. +1. Reads the user-specified NeXus definition, either in YAML or XML format. +2. If input is in YAML, it creates the definition in NXDL's XML format. + If input is an NXDL definition in XML format, it creates a corresponding YAML file. +3. Optionally, if --append argument is given, + the XML or YAML input file is interpreted as an extension of the existing definition, + so instead of creating a new file from scratch, the original file is rather extended. ```console user@box:~$ python nyaml2nxdl.py From 124de3081c0c06c0743512cbb54a0edb2bd90a7a Mon Sep 17 00:00:00 2001 From: sanbrock <45483558+sanbrock@users.noreply.github.com> Date: Thu, 9 Nov 2023 12:04:52 +0100 Subject: [PATCH 92/93] Update README.md cleaner definition for optionality as suggested by @prjamin in the PR. --- dev_tools/nyaml2nxdl/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev_tools/nyaml2nxdl/README.md b/dev_tools/nyaml2nxdl/README.md index e0ff6b5ab2..0709fac566 100644 --- a/dev_tools/nyaml2nxdl/README.md +++ b/dev_tools/nyaml2nxdl/README.md @@ -37,7 +37,7 @@ Options: * Named NeXus groups, which are instances of NeXus classes especially base or contributed classes. Creating (NXbeam) is a simple example of a request to define a group named according to NeXus default rules. mybeam1(NXbeam) or mybeam2(NXbeam) are examples how to create multiple named instances at the same hierarchy level. * Members of groups so-called fields or attributes. A simple example of a member is voltage. Here the datatype is implied automatically as the default NeXus NX_CHAR type. By contrast, voltage(NX_FLOAT) can be used to instantiate a member of class which should be of NeXus type NX_FLOAT. * And attributes of either groups or fields. The mark '\@' have to precede the name of attributes. -* Optionality: For all fields, groups and attributes in `application definitions` are `required` by default, except anything (`recommended` or `optional`) mentioned. +* Optionality: In application definitions, all fields, groups, and attributes are required unless marked recommended or optional. **Special keywords**: Several keywords can be used as childs of groups, fields, and attributes to specify the members of these. Groups, fields and attributes are nodes of the XML tree. * **doc**: A human-readable description/docstring From 4ea35ef6f4988dc8d5dea563c85b50398dc2165a Mon Sep 17 00:00:00 2001 From: sanbrock <45483558+sanbrock@users.noreply.github.com> Date: Tue, 21 Nov 2023 10:26:29 +0100 Subject: [PATCH 93/93] typos fixed as suggested --- dev_tools/nyaml2nxdl/nyaml2nxdl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev_tools/nyaml2nxdl/nyaml2nxdl.py b/dev_tools/nyaml2nxdl/nyaml2nxdl.py index 4031d6bbdf..f5b62436c6 100755 --- a/dev_tools/nyaml2nxdl/nyaml2nxdl.py +++ b/dev_tools/nyaml2nxdl/nyaml2nxdl.py @@ -2,8 +2,8 @@ """ Main file of nyaml2nxdl tool. -To write a definition for a instrument, experiment and/or process in nxdl.xml file from a YAML -file which details a hierarchy of data/metadata elements. It also allows both wa +To write a definition for an instrument, experiment and/or process in nxdl.xml file from a YAML +file which details a hierarchy of data/metadata elements. It also allows back and forth conversion beteen YAML and nxdl.xml files that follows rules of NeXus ontology or data format. """ # -*- coding: utf-8 -*-