diff --git a/.gitignore b/.gitignore index b6e4761..b1619d4 100644 --- a/.gitignore +++ b/.gitignore @@ -109,6 +109,7 @@ venv/ ENV/ env.bak/ venv.bak/ +.pyenv # Spyder project settings .spyderproject diff --git a/README.md b/README.md index 432da45..85648c8 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ $ pip install -e . ``` You can then run the program with: ``` -$ metainfo-yaml2py +$ metainfo-yaml2py ``` ## Example @@ -73,7 +73,6 @@ class Activity: description='The starting date and time of the activity.\n', a_eln={ "component": "DateTimeEditQuantity"}) - end_time = Quantity( type=Datetime, description='The ending date and time of the activity.\n', @@ -87,4 +86,22 @@ class Entity: m_package.__init_metainfo__() +``` + +## Command Line Interface +``` +$ metainfo-yaml2py --help +usage: metainfo-yaml2py [-h] [-o OUTPUT_DIR] [-n] [-p] yaml_path + +positional arguments: + yaml_path The path to the YAML schema that should be converted to Python + classes. + +optional arguments: + -h, --help show this help message and exit + -o OUTPUT_DIR, --output_dir OUTPUT_DIR + The path to the output directory of the conversion. Defaults to + the current directory. + -n, --normalizers Add empty normalizers to all class definitions. + -p, --plugin Create all the necessary files for a nomad plugin. ``` \ No newline at end of file diff --git a/example/__init__.py b/example/__init__.py index e7322ab..b30807e 100644 --- a/example/__init__.py +++ b/example/__init__.py @@ -16,20 +16,29 @@ # limitations under the License. # -from nomad.metainfo import Datetime, Package, Quantity, Section +from nomad.metainfo import ( + Package, + Quantity, + Datetime, + Section, +) +from nomad.datamodel.data import ( + ArchiveSection, +) m_package = Package(name='Example') -class Activity: - '''A base class for any activity in relation to an entity.''' +class Activity(ArchiveSection): + ''' + A base class for any activity in relation to an entity. + ''' m_def = Section() start_time = Quantity( type=Datetime, description='The starting date and time of the activity.\n', a_eln={ "component": "DateTimeEditQuantity"}) - end_time = Quantity( type=Datetime, description='The ending date and time of the activity.\n', @@ -37,8 +46,10 @@ class Activity: "component": "DateTimeEditQuantity"}) -class Entity: - '''A base class for any entity which can be related to an activity.''' +class Entity(ArchiveSection): + ''' + A base class for any entity which can be related to an activity. + ''' m_def = Section() diff --git a/example/example.schema.archive.yaml b/example/example.schema.archive.yaml index ed2e84d..5ca055c 100644 --- a/example/example.schema.archive.yaml +++ b/example/example.schema.archive.yaml @@ -1,5 +1,5 @@ definitions: - name: Example + name: Example Schema sections: Activity: description: | diff --git a/example/example_schema_plugin/LICENSE b/example/example_schema_plugin/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/example/example_schema_plugin/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/example/example_schema_plugin/README.md b/example/example_schema_plugin/README.md new file mode 100644 index 0000000..db08ca4 --- /dev/null +++ b/example/example_schema_plugin/README.md @@ -0,0 +1,67 @@ +# NOMAD's schema example plugin + +## Getting started + +### Fork the project + +Go to the github project page https://github.com/nomad-coe/nomad-schema-plugin-example, hit +fork (and leave a star, thanks!). Maybe you want to rename the project while forking! + +### Clone your fork + +Follow the github instructions. The URL and directory depends on your user name or organization and the +project name you choose. But, it should look somewhat like this: + +``` +git clone git@github.com:markus1978/my-nomad-schema.git +cd my-nomad-schema +``` + +### Install the dependencies + +You should create a virtual environment. You will need the `nomad-lab` package (and `pytest`). +You need at least Python 3.9. + +```sh +python3 -m venv .pyenv +source .pyenv/bin/activate +pip install -r requirements.txt --index-url https://gitlab.mpcdf.mpg.de/api/v4/projects/2187/packages/pypi/simple +``` + +**Note!** +Until we have an official pypi NOMAD release with the plugins functionality. Make +sure to include NOMAD's internal package registry (e.g. via `--index-url`). Follow the instructions +in `requirements.txt`. + +### Run the tests + +Make sure the current directory is in your path: + +```sh +export PYTHONPATH=. +``` + +You can run automated tests with `pytest`: + +```sh +pytest -svx tests +``` + +You can parse an example archive that uses the schema with `nomad` +(installed via `nomad-lab` Python package): + +```sh +nomad parse tests/data/test.archive.yaml --show-archive +``` + +## Developing your schema + +You can now start to develop you schema. Here are a few things that you might want to change: + +- The metadata in `nomad_plugin.yaml`. +- The name of the Python package `nomadschemaexample`. If you want to define multiple plugins, you can nest packages. +- The name of the example section `ExampleSection`. You will also want to define more than one section. +- When you change module and class names, make sure to update the `nomad_plugin.yaml` accordingly. + +To learn more about plugins, how to add them to an Oasis, how to publish them, read our +documentation on plugins: https://nomad-lab/prod/v1/staging/docs/plugins.html diff --git a/example/example_schema_plugin/nomad.yaml b/example/example_schema_plugin/nomad.yaml new file mode 100644 index 0000000..5d3b566 --- /dev/null +++ b/example/example_schema_plugin/nomad.yaml @@ -0,0 +1,9 @@ +normalize: + normalizers: + include: + - MetainfoNormalizer +plugins: + include: schemas/example + options: + schemas/example: + python_package: example_schema diff --git a/example/example_schema_plugin/pyproject.toml b/example/example_schema_plugin/pyproject.toml new file mode 100644 index 0000000..dcf9007 --- /dev/null +++ b/example/example_schema_plugin/pyproject.toml @@ -0,0 +1,18 @@ +[build-system] +requires = [ "setuptools>=61.0.0",] +build-backend = "setuptools.build_meta" + +[project] +name = "example_schema" +version = "0.0.1" +description = "A plugin for NOMAD" +readme = "README.md" +requires-python = ">=3.9" +classifiers = [ "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: 3.9",] +dependencies = [ "nomad-lab>=1.2.0-pre", "pytest", "typing-extensions==4.4.0",] + +[project.license] +file = "LICENSE" + +[tool.setuptools.packages.find] +where = [ "src",] diff --git a/example/example_schema_plugin/setup.py b/example/example_schema_plugin/setup.py new file mode 100644 index 0000000..7f1a176 --- /dev/null +++ b/example/example_schema_plugin/setup.py @@ -0,0 +1,4 @@ +from setuptools import setup + +if __name__ == "__main__": + setup() diff --git a/example/example_schema_plugin/src/example_schema/__init__.py b/example/example_schema_plugin/src/example_schema/__init__.py new file mode 100644 index 0000000..b0744dd --- /dev/null +++ b/example/example_schema_plugin/src/example_schema/__init__.py @@ -0,0 +1 @@ +from .schema import * \ No newline at end of file diff --git a/example/example_schema_plugin/src/example_schema/nomad_plugin.yaml b/example/example_schema_plugin/src/example_schema/nomad_plugin.yaml new file mode 100644 index 0000000..760b39c --- /dev/null +++ b/example/example_schema_plugin/src/example_schema/nomad_plugin.yaml @@ -0,0 +1,3 @@ +description: This is a plugin schema generated from a yaml schema. +name: Example Schema +plugin_type: schema diff --git a/example/example_schema_plugin/src/example_schema/schema.py b/example/example_schema_plugin/src/example_schema/schema.py new file mode 100644 index 0000000..b9d1103 --- /dev/null +++ b/example/example_schema_plugin/src/example_schema/schema.py @@ -0,0 +1,81 @@ +# +# Copyright The NOMAD Authors. +# +# This file is part of NOMAD. See https://nomad-lab.eu for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from structlog.stdlib import ( + BoundLogger, +) +from nomad.metainfo import ( + Package, + Quantity, + Datetime, + Section, +) +from nomad.datamodel.data import ( + ArchiveSection, +) + +m_package = Package(name='Example Schema') + + +class Activity(ArchiveSection): + ''' + A base class for any activity in relation to an entity. + ''' + m_def = Section() + start_time = Quantity( + type=Datetime, + description='The starting date and time of the activity.\n', + a_eln={ + "component": "DateTimeEditQuantity"}) + end_time = Quantity( + type=Datetime, + description='The ending date and time of the activity.\n', + a_eln={ + "component": "DateTimeEditQuantity"}) + + def normalize(self, archive, logger: BoundLogger) -> None: + ''' + The normalizer for the `Activity` class. + + Args: + archive (EntryArchive): The archive containing the section that is being + normalized. + logger (BoundLogger): A structlog logger. + ''' + super(Activity, self).normalize(archive, logger) + + +class Entity(ArchiveSection): + ''' + A base class for any entity which can be related to an activity. + ''' + m_def = Section() + + def normalize(self, archive, logger: BoundLogger) -> None: + ''' + The normalizer for the `Entity` class. + + Args: + archive (EntryArchive): The archive containing the section that is being + normalized. + logger (BoundLogger): A structlog logger. + ''' + super(Entity, self).normalize(archive, logger) + + +m_package.__init_metainfo__() diff --git a/example/example_schema_plugin/tests/data/test_activity.archive.yaml b/example/example_schema_plugin/tests/data/test_activity.archive.yaml new file mode 100644 index 0000000..1236157 --- /dev/null +++ b/example/example_schema_plugin/tests/data/test_activity.archive.yaml @@ -0,0 +1,2 @@ +data: + m_def: example_schema.Activity diff --git a/example/example_schema_plugin/tests/data/test_entity.archive.yaml b/example/example_schema_plugin/tests/data/test_entity.archive.yaml new file mode 100644 index 0000000..1e215a3 --- /dev/null +++ b/example/example_schema_plugin/tests/data/test_entity.archive.yaml @@ -0,0 +1,2 @@ +data: + m_def: example_schema.Entity diff --git a/example/example_schema_plugin/tests/test_schema.py b/example/example_schema_plugin/tests/test_schema.py new file mode 100644 index 0000000..302b3bb --- /dev/null +++ b/example/example_schema_plugin/tests/test_schema.py @@ -0,0 +1,10 @@ +import os.path +import glob + +from nomad.client import parse, normalize_all + +def test_schema(): + test_files = glob.glob(os.path.join(os.path.dirname(__file__), 'data', '*.archive.yaml')) + for test_file in test_files: + entry_archive = parse(test_file)[0] + normalize_all(entry_archive) diff --git a/pyproject.toml b/pyproject.toml index 2b90214..0cbdbab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,41 @@ [build-system] -requires = ["setuptools>=42.0", "wheel"] -build-backend = "setuptools.build_meta" \ No newline at end of file +requires = ["setuptools>=61.0.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "metainfoyaml2py" +version = "0.0.1" +description = "A program for converting NOMAD metainfo YAML schemas into Python class definitions" +readme = "README.md" +requires-python = ">=3.9" +license = {file = "LICENSE"} +keywords = ["nomad"] +authors = [ + {name = "Hampus Näsström", email = "hampus.naesstroem@physik.hu-berlin.de"}, +] +maintainers = [ + {name = "Hampus Näsström", email = "hampus.naesstroem@physik.hu-berlin.de"} +] +classifiers = [ + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.9", +] + +dependencies = [ + "autopep8>=1.7.0", + "autoflake>=1.6.1", + "PyYAML>=6.0", + "toml>=0.10.2", +] +[project.optional-dependencies] +dev = [ + "nomad-lab>=1.2.0-pre", + "structlog", +] + +[project.scripts] +metainfo-yaml2py = "metainfoyaml2py.metainfoyaml2py:main" + +[tool.setuptools.packages.find] +where = ["src"] \ No newline at end of file diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index f31bbac..0000000 --- a/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -autopep8>=1.7.0 -autoflake>=1.6.1 -PyYAML>=6.0 \ No newline at end of file diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index d2ed041..0000000 --- a/setup.cfg +++ /dev/null @@ -1,28 +0,0 @@ -[metadata] -name = metainfoyaml2py -version = 0.0.1 -description = A program for converting NOMAD metainfo YAML schemas into Python class definitions -author = Hampus Näsström -license = Apache-2.0 license -license_file = LICENSE -platforms = unix, linux, osx, cygwin, win32 -classifiers = - Programming Language :: Python :: 3 - Programming Language :: Python :: 3 :: Only - Programming Language :: Python :: 3.8 - -[options] -packages = - metainfoyaml2py -install_requires = - autopep8>=1.7.0 - autoflake>=1.6.1 - PyYAML>=6.0 -python_requires = >=3.6 -package_dir = - =src -zip_safe = no - -[options.entry_points] -console_scripts = - metainfo-yaml2py= metainfoyaml2py.metainfoyaml2py:main \ No newline at end of file diff --git a/src/metainfoyaml2py/metainfoyaml2py.py b/src/metainfoyaml2py/metainfoyaml2py.py index c28a3d3..0b932e6 100644 --- a/src/metainfoyaml2py/metainfoyaml2py.py +++ b/src/metainfoyaml2py/metainfoyaml2py.py @@ -2,12 +2,15 @@ metainfoyaml2py module ''' -import sys +import argparse import os +import shutil import json +from typing import Any, Iterable import warnings import re +import toml import yaml import autopep8 import autoflake @@ -18,10 +21,11 @@ def _to_camel_case(input_string: str) -> str: - '''Help function for converting sub section names to CamelCase. + ''' + Help function for converting sub section names to CamelCase. Args: - input_string (str): The input string with space, - or _ for separation. + input_string (str): The input string with space, -, _, or CamelCase for separation. Returns: str: The input converted to CamelCase. @@ -36,8 +40,52 @@ def _to_camel_case(input_string: str) -> str: return ''.join(word[:1].upper() + word[1:] for word in words) +def _to_snake_case(input_string: str) -> str: + ''' + Help function for converting package name to snake_case. + + Args: + input_string (str): The input string with space, -, _, or CamelCase for separation. + + Returns: + str: The input converted to snake_case. + ''' + matches = re.finditer( + r'.+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)', + input_string, + ) + words = [] + for match in matches: + words += match.group(0).replace("-", " ").replace("_", " ").split() + return '_'.join(word.lower() for word in words) + + +def set_nested(mapping: dict, nested_key: list, value: Any) -> None: + ''' + Helper function for setting a nested value in a dict. + + Args: + mapping (dict): The nested dictionary. + nested_key (list): A list of the nested keys from outer to inner. + value (Any): The value to set. + + Returns: + _type_: The nested dictionary with the set value. + ''' + _nested_dict = mapping.copy() + _keys = nested_key.copy() + key = _keys.pop(0) + if len(_keys) == 0: + _nested_dict[key] = value + return _nested_dict + else: + _nested_dict[key] = set_nested(_nested_dict.get(key, {}), _keys, value) + return _nested_dict + + def read_yaml(path: str) -> dict: - '''Help function for reading YAML file into dict using pyyaml. + ''' + Help function for reading YAML file into dict using pyyaml. Args: path (str): The path to the YAML file including the `.yaml` extension. @@ -47,7 +95,34 @@ def read_yaml(path: str) -> dict: ''' with open(path, 'r', encoding="utf8") as file: return yaml.safe_load(file) + + +def update_mapping_file(path: str, nested_keys: Iterable[list], values: Iterable) -> None: + ''' + Help function for updating a nested key value in a yaml or toml file. + Args: + path (str): The path to the file. + nested_keys (Iterable[list]): An iterable of lists with the nested keys as items. + values (Iterable): An iterable of the values corresponding to the keys. + + Raises: + ValueError: For unsupported file endings. + ''' + with open(path, 'r', encoding='utf-8') as fh: + if path.endswith('.yaml'): + mapping = yaml.safe_load(fh) + elif path.endswith('.toml'): + mapping = toml.load(fh) + else: + raise ValueError(f'Unsupported file ending for: {path}') + for nested_key, value in zip(nested_keys, values): + mapping = set_nested(mapping=mapping, nested_key=nested_key, value=value) + with open(path, 'w', encoding='utf-8') as fh: + if path.endswith('.yaml'): + yaml.dump(mapping, fh) + elif path.endswith('.toml'): + toml.dump(mapping, fh) def parse_annotation(section_dict: dict) -> str: ''' @@ -67,7 +142,8 @@ def parse_annotation(section_dict: dict) -> str: def parse_quantity(quantity_name: str, quantity_dict: dict) -> str: - '''Parse the content of metainfo quantity into Python instance. + ''' + Parse the content of metainfo quantity into Python instance. Args: quantity_name (str): The name of the quantity. @@ -109,7 +185,8 @@ def parse_quantity(quantity_name: str, quantity_dict: dict) -> str: def parse_section(section_name: str, section_dict: dict) -> str: - '''Parse the content of a metainfo section into a Python class. + ''' + Parse the content of a metainfo section into a Python class. Args: section_name (str): The name of the section. @@ -158,6 +235,7 @@ def parse_section(section_name: str, section_dict: dict) -> str: base_sections.append(base_class) else: warnings.warn(f"Unable to inherit from referenced base section: {base_section}.") + base_sections.append('ArchiveSection') base_classes = "" if len(base_sections) > 0: base_classes = f"({','.join(base_sections)})" @@ -166,7 +244,7 @@ def parse_section(section_name: str, section_dict: dict) -> str: 'description', 'Class autogenerated from yaml schema.') if description[-1] == '\n': description = description[:-1] - code += f"class {section_name}{base_classes}:\n '''{description}'''\n pass\n" + code += f"class {section_name}{base_classes}:\n '''\n {description}\n '''\n" # Pop quantities quantities = section_dict.pop('quantities', {}) code += " m_def = Section(" + parse_annotation(section_dict)[2:] @@ -185,13 +263,58 @@ def parse_section(section_name: str, section_dict: dict) -> str: return code -def yaml2py(yaml_path: str, output_dir: str = '') -> None: - '''Function for parsing a NOMAD metainfo YAML schema into a python file of class definitions. +def create_plugin(location: str, package_name: str) -> str: + ''' + Function for creating a nomad plugin package at a given location. + + Args: + location (str): The location where the nomad plugin folder will be created. + package_name (str): The name of the package. + + Returns: + str: The location with filename where the schema should be placed. + ''' + snake_package_name = _to_snake_case(package_name) + plugin_loc = os.path.join(location, snake_package_name + '_plugin') + shutil.copytree( + src=os.path.join(resource_path, 'standard_plugin_content'), + dst=plugin_loc, + ) + os.rename( + src=os.path.join(plugin_loc, 'src', 'plugin_name'), + dst=os.path.join(plugin_loc, 'src', snake_package_name), + ) + update_mapping_file( + path=os.path.join(plugin_loc, 'src', snake_package_name, 'nomad_plugin.yaml'), + nested_keys=(['name'],), + values=(package_name,) + ) + update_mapping_file( + path=os.path.join(plugin_loc, 'pyproject.toml'), + nested_keys=(['project','name'],), + values=(snake_package_name,) + ) + update_mapping_file( + path=os.path.join(plugin_loc, 'nomad.yaml'), + nested_keys=(['plugins','options','schemas/example','python_package'],), + values=(snake_package_name,) + ) + return os.path.join(plugin_loc, 'src', snake_package_name, 'schema.py') + + +def yaml2py(yaml_path: str, output_dir: str = '', normalizers: bool = False, + plugin: bool = False) -> None: + ''' + Function for parsing a NOMAD metainfo YAML schema into a python file of class definitions. Args: yaml_path (str): The path to the YAML file including the `.yaml` extension output_dir (str, optional): The output directory where the `__init__.py` file is saved. Defaults to ''. + normalizers (bool, optional): Whether to add empty normalizers or not. + Defaults to False. + plugin (bool, optional): Whether or not to create the files needed for a NOMAD plugin. + Defaults to False. Raises: ValueError: If the YAML file is not a valid NOMAD metainfo schema. @@ -204,14 +327,18 @@ def yaml2py(yaml_path: str, output_dir: str = '') -> None: # Get the standard contents from the 'standard_file_content.yaml' file content = read_yaml(os.path.join( resource_path, 'standard_file_content.yaml')) + # Get the package name, defaults to YAML file name (without + # .schema.archive.yaml) + file_name = os.path.basename(yaml_path).split("/")[-1] + package_name = yaml_dict.get('name', file_name.split('.')[0]) + if plugin: + output_file = create_plugin(output_dir, package_name) + else: + output_file = os.path.join(output_dir, '__init__.py') # Create output file with context manager - with open(os.path.join(output_dir, '__init__.py'), 'w', encoding="utf8") as file: + with open(output_file, 'w', encoding="utf8") as file: # Write the file content to string variable `code` code = content['imports'] + '\n' - # Get the package name, defaults to YAML file name (without - # .schema.archive.yaml) - file_name = os.path.basename(yaml_path).split("/")[-1] - package_name = yaml_dict.get('name', file_name.split('.')[0]) code += content['package_name'] % package_name + '\n' sections = yaml_dict.get('sections', {}) for section in sections: @@ -220,6 +347,30 @@ def yaml2py(yaml_path: str, output_dir: str = '') -> None: section_name=section, section_dict=section_dict, ) + '\n' + if normalizers: + code += ( + ' ' + + content['normalizer'].replace('\n','\n ') % (section, section) + + '\n' + ) + if plugin: + test_loc = os.path.join( + output_dir, + _to_snake_case(package_name) + '_plugin', + 'tests' + ) + test_file = os.path.join( + test_loc,'data',f'test_{_to_snake_case(section)}.archive.yaml' + ) + with open(test_file, 'w') as fh: + yaml.dump( + { + 'data': { + 'm_def': f'{_to_snake_case(package_name)}.{section}' + } + }, + fh + ) code += content['footer'] + '\n' code = content['header'] + '\n' + code code = code.replace('true', 'True') @@ -235,12 +386,40 @@ def yaml2py(yaml_path: str, output_dir: str = '') -> None: def main() -> None: - '''Main function for running the metainfo YAML to Python class definition parser. ''' - if len(sys.argv) < 2 or len(sys.argv) > 3: - sys.exit( - "Please provide path to YAML file and optionally path to output directory.") - yaml2py(*sys.argv[1:3]) + Main function for running the metainfo YAML to Python class definition parser. + ''' + parser = argparse.ArgumentParser() + parser.add_argument( + 'yaml_path', + help='The path to the YAML schema that should be converted to Python classes.', + ) + parser.add_argument( + '-o', + '--output_dir', + default='', + help=('The path to the output directory of the conversion. ' + 'Defaults to the current directory.'), + ) + parser.add_argument( + '-n', + '--normalizers', + action='store_true', + help='Add empty normalizers to all class definitions.', + ) + parser.add_argument( + '-p', + '--plugin', + action='store_true', + help='Create all the necessary files for a nomad plugin.', + ) + args = parser.parse_args() + yaml2py( + yaml_path=args.yaml_path, + output_dir=args.output_dir, + normalizers=args.normalizers, + plugin=args.plugin, + ) if __name__ == "__main__": diff --git a/src/metainfoyaml2py/resources/standard_file_content.yaml b/src/metainfoyaml2py/resources/standard_file_content.yaml index 8dcac81..6ad3d79 100644 --- a/src/metainfoyaml2py/resources/standard_file_content.yaml +++ b/src/metainfoyaml2py/resources/standard_file_content.yaml @@ -18,9 +18,35 @@ header: | # imports: | import numpy as np - from nomad.metainfo import MSection, Package, Quantity, SubSection, MEnum, Reference, Datetime, Section - from nomad.datamodel.data import EntryData, ArchiveSection + from structlog.stdlib import ( + BoundLogger, + ) + from nomad.metainfo import ( + MSection, + Package, + Quantity, + SubSection, + MEnum, + Reference, + Datetime, + Section, + ) + from nomad.datamodel.data import ( + EntryData, + ArchiveSection, + ) package_name: | m_package = Package(name='%s') footer: | m_package.__init_metainfo__() +normalizer: | + def normalize(self, archive, logger: BoundLogger) -> None: + ''' + The normalizer for the `%s` class. + + Args: + archive (EntryArchive): The archive containing the section that is being + normalized. + logger (BoundLogger): A structlog logger. + ''' + super(%s, self).normalize(archive, logger) diff --git a/src/metainfoyaml2py/resources/standard_plugin_content/LICENSE b/src/metainfoyaml2py/resources/standard_plugin_content/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/src/metainfoyaml2py/resources/standard_plugin_content/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/src/metainfoyaml2py/resources/standard_plugin_content/README.md b/src/metainfoyaml2py/resources/standard_plugin_content/README.md new file mode 100644 index 0000000..db08ca4 --- /dev/null +++ b/src/metainfoyaml2py/resources/standard_plugin_content/README.md @@ -0,0 +1,67 @@ +# NOMAD's schema example plugin + +## Getting started + +### Fork the project + +Go to the github project page https://github.com/nomad-coe/nomad-schema-plugin-example, hit +fork (and leave a star, thanks!). Maybe you want to rename the project while forking! + +### Clone your fork + +Follow the github instructions. The URL and directory depends on your user name or organization and the +project name you choose. But, it should look somewhat like this: + +``` +git clone git@github.com:markus1978/my-nomad-schema.git +cd my-nomad-schema +``` + +### Install the dependencies + +You should create a virtual environment. You will need the `nomad-lab` package (and `pytest`). +You need at least Python 3.9. + +```sh +python3 -m venv .pyenv +source .pyenv/bin/activate +pip install -r requirements.txt --index-url https://gitlab.mpcdf.mpg.de/api/v4/projects/2187/packages/pypi/simple +``` + +**Note!** +Until we have an official pypi NOMAD release with the plugins functionality. Make +sure to include NOMAD's internal package registry (e.g. via `--index-url`). Follow the instructions +in `requirements.txt`. + +### Run the tests + +Make sure the current directory is in your path: + +```sh +export PYTHONPATH=. +``` + +You can run automated tests with `pytest`: + +```sh +pytest -svx tests +``` + +You can parse an example archive that uses the schema with `nomad` +(installed via `nomad-lab` Python package): + +```sh +nomad parse tests/data/test.archive.yaml --show-archive +``` + +## Developing your schema + +You can now start to develop you schema. Here are a few things that you might want to change: + +- The metadata in `nomad_plugin.yaml`. +- The name of the Python package `nomadschemaexample`. If you want to define multiple plugins, you can nest packages. +- The name of the example section `ExampleSection`. You will also want to define more than one section. +- When you change module and class names, make sure to update the `nomad_plugin.yaml` accordingly. + +To learn more about plugins, how to add them to an Oasis, how to publish them, read our +documentation on plugins: https://nomad-lab/prod/v1/staging/docs/plugins.html diff --git a/src/metainfoyaml2py/resources/standard_plugin_content/nomad.yaml b/src/metainfoyaml2py/resources/standard_plugin_content/nomad.yaml new file mode 100644 index 0000000..ce0a4e2 --- /dev/null +++ b/src/metainfoyaml2py/resources/standard_plugin_content/nomad.yaml @@ -0,0 +1,12 @@ +normalize: + normalizers: + include: + - MetainfoNormalizer +plugins: + # We only include our schema here. Without the explicit include, all plugins will be + # loaded. Many build in plugins require more dependencies. Install nomad-lab[parsing] + # to make all default plugins work. + include: 'schemas/example' + options: + schemas/example: + python_package: nomadschemaexample \ No newline at end of file diff --git a/src/metainfoyaml2py/resources/standard_plugin_content/pyproject.toml b/src/metainfoyaml2py/resources/standard_plugin_content/pyproject.toml new file mode 100644 index 0000000..8c7ab2a --- /dev/null +++ b/src/metainfoyaml2py/resources/standard_plugin_content/pyproject.toml @@ -0,0 +1,25 @@ +[build-system] +requires = ["setuptools>=61.0.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "plugin_name" +version = "0.0.1" +description = "A plugin for NOMAD" +readme = "README.md" +requires-python = ">=3.9" +license = {file = "LICENSE"} +classifiers = [ + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.9", +] + +dependencies = [ + "nomad-lab>=1.2.0-pre", + "pytest", + "typing-extensions==4.4.0", +] + +[tool.setuptools.packages.find] +where = ["src"] \ No newline at end of file diff --git a/src/metainfoyaml2py/resources/standard_plugin_content/setup.py b/src/metainfoyaml2py/resources/standard_plugin_content/setup.py new file mode 100644 index 0000000..7f1a176 --- /dev/null +++ b/src/metainfoyaml2py/resources/standard_plugin_content/setup.py @@ -0,0 +1,4 @@ +from setuptools import setup + +if __name__ == "__main__": + setup() diff --git a/src/metainfoyaml2py/resources/standard_plugin_content/src/plugin_name/__init__.py b/src/metainfoyaml2py/resources/standard_plugin_content/src/plugin_name/__init__.py new file mode 100644 index 0000000..b0744dd --- /dev/null +++ b/src/metainfoyaml2py/resources/standard_plugin_content/src/plugin_name/__init__.py @@ -0,0 +1 @@ +from .schema import * \ No newline at end of file diff --git a/src/metainfoyaml2py/resources/standard_plugin_content/src/plugin_name/nomad_plugin.yaml b/src/metainfoyaml2py/resources/standard_plugin_content/src/plugin_name/nomad_plugin.yaml new file mode 100644 index 0000000..c6eb6f4 --- /dev/null +++ b/src/metainfoyaml2py/resources/standard_plugin_content/src/plugin_name/nomad_plugin.yaml @@ -0,0 +1,4 @@ +plugin_type: schema +name: Plugin name +description: | + This is a plugin schema generated from a yaml schema. \ No newline at end of file diff --git a/src/metainfoyaml2py/resources/standard_plugin_content/tests/test_schema.py b/src/metainfoyaml2py/resources/standard_plugin_content/tests/test_schema.py new file mode 100644 index 0000000..302b3bb --- /dev/null +++ b/src/metainfoyaml2py/resources/standard_plugin_content/tests/test_schema.py @@ -0,0 +1,10 @@ +import os.path +import glob + +from nomad.client import parse, normalize_all + +def test_schema(): + test_files = glob.glob(os.path.join(os.path.dirname(__file__), 'data', '*.archive.yaml')) + for test_file in test_files: + entry_archive = parse(test_file)[0] + normalize_all(entry_archive)