Nyaml2nxdl migration

.gitignore

@@ -1,6 +1,5 @@
 # Hidden files
 .*
-!.github
 
 # Python byte / compiled / optimized
 *.py[cod]
@@ -10,6 +9,22 @@ __pycache__/
 build/
 makelog.txt
 
-# Unknown
-/python/
-__github_creds__.txt
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST

MANIFEST.in

@@ -0,0 +1,4 @@
+recursive-include applications/ *.nxdl.xml
+recursive-include contributed_definitions/ *.nxdl.xml
+recursive-include base_classes/ *.nxdl.xml
+include ./ *.xsd

Makefile

@@ -6,8 +6,18 @@
 PYTHON = python3
 SPHINX = sphinx-build
 BUILD_DIR = "build"
+BASE_CLASS_DIR := base_classes
+CONTRIB_DIR := contributed_definitions
+APPDEF_DIR := applications
+NXDL_DIRS := $(BASE_CLASS_DIR) $(CONTRIB_DIR) $(APPDEF_DIR)
+NYAML_SUBDIR := nyaml
+NYAML_APPENDIX := _parsed
 
-.PHONY: help install style autoformat test clean prepare html pdf impatient-guide all local
+NXDL_BC := $(wildcard $(BASE_CLASS_DIR)/*.nxdl.xml)
+NXDL_CONTRIB := $(wildcard $(CONTRIB_DIR)/*.nxdl.xml)
+NXDL_APPDEF := $(wildcard $(APPDEF_DIR)/*.nxdl.xml)
+
+.PHONY: help install style autoformat test clean prepare html pdf impatient-guide all local nyaml nxdl
 
 help ::
 	@echo ""
@@ -49,6 +59,9 @@ test ::
 
 clean ::
 	$(RM) -rf $(BUILD_DIR)
+	$(RM) -rf $(BASE_CLASS_DIR)/$(NYAML_SUBDIR)
+	$(RM) -rf $(APPDEF_DIR)/$(NYAML_SUBDIR)
+	$(RM) -rf $(CONTRIB_DIR)/$(NYAML_SUBDIR)
 
 prepare ::
 	$(PYTHON) -m dev_tools manual --prepare --build-root $(BUILD_DIR)
@@ -83,6 +96,28 @@ all ::
 	@echo "HTML built: `ls -lAFgh $(BUILD_DIR)/manual/build/html/index.html`"
 	@echo "PDF built: `ls -lAFgh $(BUILD_DIR)/manual/build/latex/nexus.pdf`"
 
+$(BASE_CLASS_DIR)/%.nxdl.xml : $(BASE_CLASS_DIR)/$(NYAML_SUBDIR)/%.yaml
+	nyaml2nxdl --input-file $<
+	mv $(BASE_CLASS_DIR)/$(NYAML_SUBDIR)/$*.nxdl.xml $@
+
+$(CONTRIB_DIR)/%.nxdl.xml : $(CONTRIB_DIR)/$(NYAML_SUBDIR)/%.yaml
+	nyaml2nxdl --input-file $<
+	mv $(CONTRIB_DIR)/$(NYAML_SUBDIR)/$*.nxdl.xml $@
+
+$(APPDEF_DIR)/%.nxdl.xml : $(APPDEF_DIR)/$(NYAML_SUBDIR)/%.yaml
+	nyaml2nxdl --input-file $<
+	mv $(APPDEF_DIR)/$(NYAML_SUBDIR)/$*.nxdl.xml $@
+
+NXDLS := $(foreach dir,$(NXDL_DIRS),$(wildcard $(dir)/*.nxdl.xml))
+nyaml : $(DIRS)
+	for file in $(NXDLS); do\
+		mkdir -p "$${file%/*}/nyaml";\
+		nyaml2nxdl --input-file $${file};\
+		FNAME=$${file##*/};\
+		mv -- "$${file%.nxdl.xml}_parsed.yaml" "$${file%/*}/nyaml/$${FNAME%.nxdl.xml}.yaml";\
+	done
+
+nxdl: $(NXDL_APPDEF) $(NXDL_CONTRIB) $(NXDL_BC)
 
 # NeXus - Neutron and X-ray Common Data Format
 #

README.md

@@ -7,6 +7,10 @@
 
 ## NeXus definition developers
 
+Note if this package was not properly installed using pip, it has to be first done by  
+
+    pip install -e  .
+
 After making a change to the NeXus class definitions there are two important checks
 to be made before commiting the change:
 
@@ -25,6 +29,14 @@ Open the HTML manual in a web brower for visual verification
 
     firefox build/manual/build/html/index.html
 
+Convenient editing of definitions is available in nyaml format. For this, definitions need to be converted first from xml to yaml format by the command
+
+    make nyaml
+
+After editing the definitions in nyaml format in the nyaml subdirectories, the following command can be used to update the definitions in nxdl.xml format:
+
+    make nxdl
+
 ### HTML pages with contributor information
 
 To build the html pages that contains contributor information on the sidebar set a github access token to an environment variable called GH_TOKEN.
@@ -57,4 +69,4 @@ package/                       | directory for packaging this content
 utils/                         | various tools used in the definitions tree
 www/                           | launch (home) page of NeXus WWW site
 xslt/                          | various XML stylesheet transformations
-dev_tools/                     | developer tools for testing and building
+dev_tools/                     | developer tools for testing and building

dev_tools/docs/nxdl.py

@@ -12,6 +12,7 @@
 from ..globals.errors import NXDLParseError
 from ..globals.nxdl import NXDL_NAMESPACE
 from ..globals.urls import REPO_URL
+from ..utils import nxdl_utils as pynxtools_nxlib
 from ..utils.github import get_file_contributors_via_api
 from ..utils.types import PathLike
 from .anchor_list import AnchorRegistry
@@ -129,7 +130,7 @@ def _parse_nxdl_file(self, nxdl_file: Path):
         # print official description of this class
         self._print("")
         self._print("**Description**:\n")
-        self._print_doc(self._INDENTATION_UNIT, ns, root, required=True)
+        self._print_doc_enum("", ns, root, required=True)
 
         # print symbol list
         node_list = root.xpath("nx:symbols", namespaces=ns)
@@ -139,7 +140,7 @@ def _parse_nxdl_file(self, nxdl_file: Path):
         elif len(node_list) > 1:
             raise Exception(f"Invalid symbol table in {nxclass_name}")
         else:
-            self._print_doc(self._INDENTATION_UNIT, ns, node_list[0])
+            self._print_doc_enum("", ns, node_list[0])
             for node in node_list[0].xpath("nx:symbol", namespaces=ns):
                 doc = self._get_doc_line(ns, node)
                 self._print(f"  **{node.get('name')}**", end="")
@@ -518,6 +519,32 @@ def _print_doc(self, indent, ns, node, required=False):
                     self._print(f"{indent}{line}")
                 self._print()
 
+    def long_doc(self, ns, node):
+        length = 0
+        line = "documentation"
+        fnd = False
+        blocks = self._get_doc_blocks(ns, node)
+        for block in blocks:
+            lines = block.splitlines()
+            length += len(lines)
+            for single_line in lines:
+                if len(single_line) > 2 and single_line[0] != "." and not fnd:
+                    fnd = True
+                    line = single_line
+        return (length, line, blocks)
+
+    def _print_doc_enum(self, indent, ns, node, required=False):
+        collapse_indent = indent
+        node_list = node.xpath("nx:enumeration", namespaces=ns)
+        (doclen, line, blocks) = self.long_doc(ns, node)
+        self._print_doc(
+            collapse_indent + self._INDENTATION_UNIT, ns, node, required=required
+        )
+        if len(node_list) == 1:
+            self._print_enumeration(
+                collapse_indent + self._INDENTATION_UNIT, ns, node_list[0]
+            )
+
     def _print_attribute(self, ns, kind, node, optional, indent, parent_path):
         name = node.get("name")
         index_name = name
@@ -526,12 +553,9 @@ def _print_attribute(self, ns, kind, node, optional, indent, parent_path):
         )
         self._print(f"{indent}.. index:: {index_name} ({kind} attribute)\n")
         self._print(
-            f"{indent}**@{name}**: {optional}{self._format_type(node)}{self._format_units(node)}\n"
+            f"{indent}**@{name}**: {optional}{self._format_type(node)}{self._format_units(node)} {self.get_first_parent_ref(f'{parent_path}/{name}', 'attribute')}\n"
         )
-        self._print_doc(indent + self._INDENTATION_UNIT, ns, node)
-        node_list = node.xpath("nx:enumeration", namespaces=ns)
-        if len(node_list) == 1:
-            self._print_enumeration(indent + self._INDENTATION_UNIT, ns, node_list[0])
+        self._print_doc_enum(indent, ns, node)
 
     def _print_if_deprecated(self, ns, node, indent):
         deprecated = node.get("deprecated", None)
@@ -569,17 +593,12 @@ def _print_full_tree(self, ns, parent, name, indent, parent_path):
                 f"{self._format_type(node)}"
                 f"{dims}"
                 f"{self._format_units(node)}"
+                f" {self.get_first_parent_ref(f'{parent_path}/{name}', 'field')}"
                 "\n"
             )
 
             self._print_if_deprecated(ns, node, indent + self._INDENTATION_UNIT)
-            self._print_doc(indent + self._INDENTATION_UNIT, ns, node)
-
-            node_list = node.xpath("nx:enumeration", namespaces=ns)
-            if len(node_list) == 1:
-                self._print_enumeration(
-                    indent + self._INDENTATION_UNIT, ns, node_list[0]
-                )
+            self._print_doc_enum(indent, ns, node)
 
             for subnode in node.xpath("nx:attribute", namespaces=ns):
                 optional = self._get_required_or_optional_text(subnode)
@@ -605,10 +624,12 @@ def _print_full_tree(self, ns, parent, name, indent, parent_path):
             # target = hTarget.replace(".. _", "").replace(":\n", "")
             # TODO: https://github.com/nexusformat/definitions/issues/1057
             self._print(f"{indent}{hTarget}")
-            self._print(f"{indent}**{name}**: {optional_text}{typ}\n")
+            self._print(
+                f"{indent}**{name}**: {optional_text}{typ} {self.get_first_parent_ref(f'{parent_path}/{name}', 'group')}\n"
+            )
 
             self._print_if_deprecated(ns, node, indent + self._INDENTATION_UNIT)
-            self._print_doc(indent + self._INDENTATION_UNIT, ns, node)
+            self._print_doc_enum(indent, ns, node)
 
             for subnode in node.xpath("nx:attribute", namespaces=ns):
                 optional = self._get_required_or_optional_text(subnode)
@@ -639,8 +660,49 @@ def _print_full_tree(self, ns, parent, name, indent, parent_path):
                 f"(suggested target: ``{node.get('target')}``)"
                 "\n"
             )
-            self._print_doc(indent + self._INDENTATION_UNIT, ns, node)
+            self._print_doc_enum(indent, ns, node)
 
     def _print(self, *args, end="\n"):
         # TODO: change instances of \t to proper indentation
         self._rst_lines.append(" ".join(args) + end)
+
+    def get_first_parent_ref(self, path, tag):
+        spliter = path.find("/", 1)
+        nx_name = path[1 : spliter]
+        path = path[spliter :]
+
+        try:
+            parents = pynxtools_nxlib.get_inherited_nodes(path, nx_name)[2]
+        except FileNotFoundError:
+            return ""
+        if len(parents) > 1:
+            parent = parents[1]
+            parent_path = parent_display_name = parent.attrib["nxdlpath"]
+            parent_path_segments = parent_path[1:].split("/")
+            nxdl_attr = parent.attrib["nxdlbase"]
+            parent_def_name = nxdl_attr[nxdl_attr.rfind("/") :
+                                        nxdl_attr.rfind(".nxdl")
+            ]
+
+            # Case where the first parent is a base_class
+            if parent_path_segments[0] == "":
+                return ""
+
+            # special treatment for NXnote@type
+            if (
+                tag == "attribute"
+                and parent_def_name == "/NXnote"
+                and parent_path == "/type"
+            ):
+                return ""
+
+            if tag == "attribute":
+                pos_of_right_slash = parent_path.rfind("/")
+                parent_path = (
+                    parent_path[:pos_of_right_slash]
+                    + "@"
+                    + parent_path[pos_of_right_slash + 1 :]
+                )
+            parent_display_name = f"{parent_def_name[1:]}{parent_path}"
+            return f":ref:`⤆ </{parent_display_name}-{tag}>`"
+        return ""

dev_tools/nyaml2nxdl/README.md

@@ -0,0 +1,68 @@
+# YAML to NXDL converter and NXDL to YAML converter
+
+**NOTE: Please use python3.8 or above to run this converter**
+
+**Tools purpose**: Offer a simple YAML-based schema and a XML-based schema to describe NeXus instances. These can be NeXus application definitions or classes
+such as base or contributed classes. Users either create NeXus instances by writing a YAML file or a XML file which details a hierarchy of data/metadata elements.
+The forward (YAML -> NXDL.XML) and backward (NXDL.XML -> YAML) conversions are implemented.
+
+**How the tool works**:
+- nyaml2nxdl.py
+1. Reads the user-specified NeXus instance, either in YML or XML format.
+2. If input is in YAML, creates an instantiated NXDL schema XML tree by walking the dictionary nest.
+   If input is in XML, creates a YML file walking the dictionary nest.
+3. Write the tree into a YAML file or a properly formatted NXDL XML schema file to disk.
+4. Optionally, if --append argument is given,
+   the XML or YAML input file is interpreted as an extension of a base class and the entries contained in it
+   are appended below a standard NeXus base class.
+   You need to specify both your input file (with YAML or XML extension) and NeXus class (with no extension).
+   Both .yaml and .nxdl.xml file of the extended class are printed.
+
+```console
+user@box:~$ python nyaml2nxdl.py
+
+Usage: python nyaml2nxdl.py [OPTIONS]
+
+Options:
+   --input-file TEXT     The path to the input data file to read.
+   --append TEXT         Parse xml NeXus file and append to specified base class,
+                         write the base class name with no extension.
+   --check-consistency   Check consistency by generating another version of the input file.
+                         E.g. for input file: NXexample.nxdl.xml the output file
+                         NXexample_consistency.nxdl.xml.
+   --verbose             Additional std output info is printed to help debugging.
+   --help                Show this message and exit.
+
+```
+
+## Documentation
+
+**Rule set**: From transcoding YAML files we need to follow several rules.
+* Named NeXus groups, which are instances of NeXus classes especially base or contributed classes. Creating (NXbeam) is a simple example of a request to define a group named according to NeXus default rules. mybeam1(NXbeam) or mybeam2(NXbeam) are examples how to create multiple named instances at the same hierarchy level.
+* Members of groups so-called fields or attributes. A simple example of a member is voltage. Here the datatype is implied automatically as the default NeXus NX_CHAR type.  By contrast, voltage(NX_FLOAT) can be used to instantiate a member of class which should be of NeXus type NX_FLOAT.
+* And attributes of either groups or fields. The mark '\@' have to precede the name of attributes.
+* Optionality: For all fields, groups and attributes in `application definitions` are `required` by default, except anything (`recommended` or `optional`) mentioned.
+
+**Special keywords**: Several keywords can be used as childs of groups, fields, and attributes to specify the members of these. Groups, fields and attributes are nodes of the XML tree.
+* **doc**: A human-readable description/docstring
+* **exists** Options are recommended, required, [min, 1, max, `infty`] numbers like here 1 can be replaced by any `uint` (unsigned integer), or `infty` to indicate no restriction on how frequently the entry can occur inside the NXDL schema at the same hierarchy level.
+* **link** Define links between nodes.
+* **units** A statement introducing NeXus-compliant NXDL units arguments, like NX_VOLTAGE
+* **dimensions** Details which dimensional arrays to expect
+* **enumeration** Python list of strings which are considered as recommended entries to choose from.
+* **dim_parameters** `dim` which is a child of `dimension` and the `dim` might have several attributes `ref`,
+`incr` including `index` and `value`. So while writing `yaml` file schema definition please following structure:
+```
+dimensions:
+   rank: integer value
+   dim: [[ind_1, val_1], [ind_2, val_2], ...]
+   dim_parameters:
+      ref: [ref_value_1, ref_value_2, ...]
+      incr: [incr_value_1, incr_value_2, ...]
+```
+Keep in mind that length of all the lists must have the **same size**.
+**Important Note**: The attributes `ref`, `incr`, `index` are deprecated. 
+
+## Next steps
+
+The NOMAD team is currently working to establish a one-to-one mapping between NeXus definitions and the NOMAD MetaInfo(scientific data model in nomad). As soon as this is in place the YAML files will be annotated with further metadata so that they can serve two purposes. On the one hand they can serve as an instance for a schema to create a GUI representation of a NOMAD Oasis ELN schema. On the other hand the YAML to NXDL converter will skip all those pieces of information which are irrelevant from a NeXus perspective.

dev_tools/nyaml2nxdl/__init__.py

@@ -0,0 +1,21 @@
+"""
+# Load paths
+"""
+# -*- coding: utf-8 -*-
+#
+# Copyright The NOMAD Authors.
+#
+# This file is part of NOMAD. See https://nomad-lab.eu for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#