From 071422e74bda60afabdcf0d42dff4b1589ad843d Mon Sep 17 00:00:00 2001 From: Amir Golparvar Date: Mon, 21 Oct 2024 21:24:30 +0200 Subject: [PATCH] updated elabftw parser --- .../example_uploads/__init__.py | 2 +- .../parsers/chemotion/nomad_plugin.yaml | 21 ------ .../parsers/elabftw/nomad_plugin.yaml | 22 ------ .../parsers/elabftw/parser.py | 67 ++++++++++++++++--- .../labfolder/test_labfolder.py | 5 +- tests/schema_packages/openbis/test_openbis.py | 5 +- 6 files changed, 68 insertions(+), 54 deletions(-) delete mode 100644 src/nomad_external_eln_integrations/parsers/chemotion/nomad_plugin.yaml delete mode 100644 src/nomad_external_eln_integrations/parsers/elabftw/nomad_plugin.yaml diff --git a/src/nomad_external_eln_integrations/example_uploads/__init__.py b/src/nomad_external_eln_integrations/example_uploads/__init__.py index fa34ee5..632374c 100644 --- a/src/nomad_external_eln_integrations/example_uploads/__init__.py +++ b/src/nomad_external_eln_integrations/example_uploads/__init__.py @@ -4,6 +4,6 @@ title='ELN File Format Example', category='Examples', description='Thi example contains an ELabFTW Experiment exported into .ELN ' - 'file format.', + 'file format.', path='example_uploads/files', ) diff --git a/src/nomad_external_eln_integrations/parsers/chemotion/nomad_plugin.yaml b/src/nomad_external_eln_integrations/parsers/chemotion/nomad_plugin.yaml deleted file mode 100644 index ac57a9b..0000000 --- a/src/nomad_external_eln_integrations/parsers/chemotion/nomad_plugin.yaml +++ /dev/null @@ -1,21 +0,0 @@ -#code_category: ELN -#code_homepage: https://chemotion.net/ -#code_name: Chemotion -#metadata: -# codeCategory: ELN -# codeLabel: Chemotion -# codeLabelStyle: 'capitals: C.' -# codeName: Chemotion -# codeUrl: https://chemotion.net/ -# parserDirName: nomad_external_eln_integrations/parsers/chemotion -# parserGitUrl: https://gitlab.mpcdf.mpg.de/nomad-lab/nomad-FAIR.git -# preabmle: "" -# status: production -# tableOfFiles: | -# |Input Filename| Description| -# |--- | --- | -# |`*.export.json` | **Mainfile** as part of the .eln chemotion export | -# |`*` | Files in .eln chemotion export that are references by the mainfile and description.txt | -#name: parsers/chemotion -#parser_class_name: nomad_external_eln_integrations.parsers.chemotion.ChemotionParser -#python_package: nomad_external_eln_integrations.parsers.chemotion \ No newline at end of file diff --git a/src/nomad_external_eln_integrations/parsers/elabftw/nomad_plugin.yaml b/src/nomad_external_eln_integrations/parsers/elabftw/nomad_plugin.yaml deleted file mode 100644 index 92eeaf7..0000000 --- a/src/nomad_external_eln_integrations/parsers/elabftw/nomad_plugin.yaml +++ /dev/null @@ -1,22 +0,0 @@ -#code_category: ELN -#code_homepage: https://www.elabftw.net/ -#code_name: eLabFTW -#metadata: -# codeCategory: ELN -# codeLabel: eLabFTW -# codeLabelStyle: 'capitals: L, F, T, W.' -# codeName: elabftw -# codeUrl: https://www.elabftw.net/ -# parserDirName: nomad_external_eln_integrations/parsers/elabftw -# parserGitUrl: https://gitlab.mpcdf.mpg.de/nomad-lab/nomad-FAIR.git -# preabmle: "" -# status: production -# tableOfFiles: | -# |Input Filename| Description| -# |--- | --- | -# |`*.ro-crate-metadata.json` | **Mainfile** as part of the .eln eLabFTW export | -# |`*` | Files in .eln eLabFTW export that are potentially references by the mainfile | -#name: parsers/elabftw -#parser_class_name: nomad_external_eln_integrations.parsers.elabftw.ELabFTWParser -#python_package: nomad_external_eln_integrations.parsers.elabftw -#plugin_source_code_url: https://gitlab.mpcdf.mpg.de/nomad-lab/nomad-FAIR/-/tree/develop/nomad/parsing/elabftw diff --git a/src/nomad_external_eln_integrations/parsers/elabftw/parser.py b/src/nomad_external_eln_integrations/parsers/elabftw/parser.py index b3d620f..10b6bbe 100644 --- a/src/nomad_external_eln_integrations/parsers/elabftw/parser.py +++ b/src/nomad_external_eln_integrations/parsers/elabftw/parser.py @@ -181,6 +181,12 @@ class ELabFTWExperimentData(MSection): a_eln=dict(component='StringEditQuantity'), description="Author's full name", ) + tags = Quantity( + type=str, + shape=['*'], + a_eln=dict(component='StringEditQuantity'), + description='Tags', + ) items_links = SubSection(sub_section=ELabFTWItemLink, repeats=True) experiments_links = SubSection(sub_section=ELabFTWExperimentLink, repeats=True) @@ -374,7 +380,11 @@ def is_mainfile( ) no_of_experiments = len(root_experiment['hasPart']) else: - no_of_experiments = len(data['@graph'][-2]['hasPart']) + no_of_experiments = len( + [item['hasPart'] for item in data['@graph'] if item['@id'] == './'][ + 0 + ] + ) except (KeyError, IndexError, TypeError): return False @@ -486,28 +496,68 @@ def _parse_legacy( title = extracted_title elabftw_experiment.title = title - path_to_export_json = os.path.join(mainfile_raw_path, exp_id, 'export-elabftw.json') try: + export_json_path = [ + item['id'] + for item in raw_experiment['has_part'] + if item['id'].endswith('.json', None) + ][0] + path_to_export_json = os.path.join(mainfile_raw_path, export_json_path) + with open(path_to_export_json) as f: export_data = json.load(f) + export_data = ( + [export_data] if isinstance(export_data, dict) else export_data + ) except FileNotFoundError: - raise ELabFTWParserError("Couldn't find export-elabftw.json file.") + raise ELabFTWParserError( + f"Couldn't locate the {export_json_path} file of the main entry." + f'Either the the exported file is corrupted or the data model is changed.' + ) def clean_nones(value): if isinstance(value, list): - return [ - clean_nones(x) for x in value - ] # ??? what to do if this list has None values + return [clean_nones(x) for x in value if x is not None] if isinstance(value, dict): return {k: clean_nones(v) for k, v in value.items() if v is not None} return value + def convert_keys(target_dict, conversion_mapping): + new_dict = {} + for key, value in target_dict.items(): + new_key = next( + ( + conversion_mapping[conv_key] + for conv_key in conversion_mapping + if conv_key in key + ), + key, + ) + if isinstance(value, dict): + value = convert_keys(value, conversion_mapping) + new_dict[new_key] = ( + {i: value[i] for i in range(len(value))} + if isinstance(value, list) and new_key == 'extra_fields' + else value + ) + return new_dict + + key_mapping = { + 'extra': 'extra_fields', + 'link': 'experiments_links', + 'description': 'body', + 'name': 'title', + } experiment_data = ELabFTWExperimentData() try: - experiment_data.m_update_from_dict(clean_nones(export_data[0])) + cleaned_data = clean_nones(export_data[0]) + cleaned_data = convert_keys(cleaned_data, key_mapping) + experiment_data.m_update_from_dict(cleaned_data) except (IndexError, KeyError, TypeError): - logger.warning("Couldn't read and parse the data from export-elabftw.json file") + logger.warning( + f"Couldn't read and parse the data from {export_json_path.lstrip('./')} file." + ) try: experiment_data.extra_fields = export_data[0]['metadata']['extra_fields'] except Exception: @@ -540,6 +590,7 @@ def _set_child_entry_name(exp_id, child_archive, archive, logger): matched_title = exp_id.split('/') if len(matched_title) > 1: extracted_title = matched_title[1] + extracted_title = re.sub(r'[-_]', ' ', extracted_title) archive.metadata.m_update_from_dict(dict(entry_name='ELabFTW Schema')) child_archive.metadata.m_update_from_dict(dict(entry_name=extracted_title)) else: diff --git a/tests/schema_packages/labfolder/test_labfolder.py b/tests/schema_packages/labfolder/test_labfolder.py index 736e49b..d0e285c 100644 --- a/tests/schema_packages/labfolder/test_labfolder.py +++ b/tests/schema_packages/labfolder/test_labfolder.py @@ -23,7 +23,10 @@ import requests from nomad.datamodel import EntryArchive, EntryMetadata -from nomad_external_eln_integrations.schema_packages.labfolder.schema import LabfolderProject, LabfolderImportError +from nomad_external_eln_integrations.schema_packages.labfolder.schema import ( + LabfolderProject, + LabfolderImportError, +) def test_labfolder_integration(): diff --git a/tests/schema_packages/openbis/test_openbis.py b/tests/schema_packages/openbis/test_openbis.py index bfdb28d..f4055e5 100644 --- a/tests/schema_packages/openbis/test_openbis.py +++ b/tests/schema_packages/openbis/test_openbis.py @@ -25,7 +25,10 @@ from nomad import utils from nomad.datamodel import EntryArchive, EntryMetadata -from nomad_external_eln_integrations.schema_packages.openbis.schema import OpenbisEntry, OpenbisImportError +from nomad_external_eln_integrations.schema_packages.openbis.schema import ( + OpenbisEntry, + OpenbisImportError, +) def mocked_login(url):