diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml new file mode 100644 index 0000000..2da819a --- /dev/null +++ b/.github/workflows/actions.yml @@ -0,0 +1,56 @@ +name: install-and-test-workflow +on: [push] +jobs: + install-and-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.9 + uses: actions/setup-python@v5 + with: + python-version: 3.9 + - name: Install dependencies + run: | + pip install --upgrade pip + pip install '.[dev]' --index-url https://gitlab.mpcdf.mpg.de/api/v4/projects/2187/packages/pypi/simple + pip install coverage coveralls + - name: Test with pytest + run: | + python -m coverage run -m pytest -sv + - name: Submit to coveralls + continue-on-error: true + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + coveralls --service=github + build-and-install: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.9 + uses: actions/setup-python@v5 + with: + python-version: 3.9 + - name: Build the package + run: | + pip install --upgrade pip + pip install build + python -m build --sdist + - name: Install the package + run: | + pip install dist/*.tar.gz --index-url https://gitlab.mpcdf.mpg.de/api/v4/projects/2187/packages/pypi/simple + ruff-linting: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: chartboost/ruff-action@v1 + with: + args: "check ." + # to enable auto-formatting check, uncomment the following lines below + # ruff-formatting: + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v4 + # - uses: chartboost/ruff-action@v1 + # with: + # args: "format . --check" \ No newline at end of file diff --git a/.github/workflows/mkdocs-deploy.yml b/.github/workflows/mkdocs-deploy.yml new file mode 100644 index 0000000..c46ff23 --- /dev/null +++ b/.github/workflows/mkdocs-deploy.yml @@ -0,0 +1,24 @@ +name: Deploy MkDocs Site + +on: + push: + branches: + - main # Triggers deployment on push to the main branch + +permissions: + contents: write + +jobs: + deploy: + runs-on: ubuntu-latest + + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Deploy docs + uses: mhausenblas/mkdocs-deploy-gh-pages@master + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + CONFIG_FILE: mkdocs.yml + REQUIREMENTS: requirements_docs.txt diff --git a/docs/how_to/use_api_functions.md b/docs/how_to/use_api_functions.md index 6194d86..2e3ff3e 100644 --- a/docs/how_to/use_api_functions.md +++ b/docs/how_to/use_api_functions.md @@ -342,7 +342,7 @@ For example: ```python metadata_new = {'upload_name': 'Test Upload', 'comment': 'This is a test upload...'} -edit_upload_metadata(upload_id, url='test', **metadata_new) +edit_upload_metadata(upload_id, url='test', upload_metadata=metadata_new) ``` ??? success "output" @@ -742,7 +742,7 @@ The returned `dataset_id` can then be used to add individual entries (or all ent ```python metadata_new = {'dataset_id': dataset_id} -edit_upload_metadata(upload_id, url='test', **metadata_new) +edit_upload_metadata(upload_id, url='test', upload_metadata=metadata_new) ``` ??? success "output" @@ -828,7 +828,7 @@ Alternatively, you can search for datasets, e.g., by `user_id` or `dataset_name` ```python my_datasets = retrieve_datasets( - user_id=nomad_user_me.user_id, url='test', max_datasets=20 + dataset_params={'user_id': nomad_user_me.user_id, 'max_datasets': 20}, url='test' ) pprint(my_datasets) ``` @@ -851,7 +851,7 @@ To get the list of entries contained within a dataset, use `query_entries()`: ```python -dataset_entries = query_entries(dataset_id=dataset_id, url='test') +dataset_entries = query_entries(query_params={'dataset_id': dataset_id}, url='test') for entry in dataset_entries: pprint(f'entry_id={entry.entry_id}, upload_id={entry.upload_id}') ``` diff --git a/src/nomad_utility_workflows/utils/__init__.py b/src/nomad_utility_workflows/utils/__init__.py index e8ffd7b..f653fd5 100644 --- a/src/nomad_utility_workflows/utils/__init__.py +++ b/src/nomad_utility_workflows/utils/__init__.py @@ -1 +1,2 @@ -from nomad_utility_workflows.utils.workflows import build_nomad_workflow +# from nomad_utility_workflows.utils.workflows import build_nomad_workflow + diff --git a/src/nomad_utility_workflows/utils/core.py b/src/nomad_utility_workflows/utils/core.py index 7d1e0f8..e3096e2 100644 --- a/src/nomad_utility_workflows/utils/core.py +++ b/src/nomad_utility_workflows/utils/core.py @@ -1,6 +1,6 @@ import json import logging -from typing import Any +from typing import Any, TypedDict import requests from cachetools.func import ttl_cache @@ -50,12 +50,32 @@ def get_authentication_token( return response.json().get('access_token') +class RequestOptions(TypedDict, total=False): + """_summary_ + + Args: + TypedDict (_type_): _description_ + total (bool, optional): _description_. Defaults to False. + """ + + section: str + url: str = None + timeout_in_sec: int = TIMEOUT_IN_SEC + headers: dict = None + with_authentication: bool = False + + +default_request_options = { + 'section': None, + 'url': None, + 'timeout_in_sec': TIMEOUT_IN_SEC, + 'headers': None, + 'with_authentication': False, +} + + def get_nomad_request( - section: str, - url: str = None, - timeout_in_sec: int = TIMEOUT_IN_SEC, - headers: dict = None, - with_authentication: bool = False, + request_options: RequestOptions = default_request_options.copy(), return_json: bool = True, accept_field: str = 'application/json', ) -> Any: @@ -76,6 +96,12 @@ def get_nomad_request( Returns: Any: _description_ """ + section = request_options.get('section') + url = request_options.get('url') + timeout_in_sec = request_options.get('timeout_in_sec') + headers = request_options.get('headers') + with_authentication = request_options.get('with_authentication') + url_base = get_nomad_url(url) url = url_base + f"{'/' if section[0] != '/' else ''}{section}" logger.info('Sending get request @ %s', url) @@ -96,6 +122,14 @@ def get_nomad_request( def get_nomad_url_name(url: str) -> str: + """_summary_ + + Args: + url (str): _description_ + + Returns: + str: _description_ + """ try: return url.split('/')[-3] except IndexError: @@ -129,18 +163,42 @@ def get_nomad_url(url: str) -> str: def get_nomad_base_url(url: str) -> str: + """_summary_ + + Args: + url (str): _description_ + + Returns: + str: _description_ + """ return (get_nomad_url(url)).removesuffix('/api/v1') def post_nomad_request( - section: str, - headers: dict = None, + request_options: RequestOptions = default_request_options.copy(), data: Any = None, json_dict: dict = None, - url: str = None, - timeout_in_sec: int = TIMEOUT_IN_SEC, - with_authentication: bool = False, ) -> json: + """_summary_ + + Args: + request_options (RequestOptions, optional): _description_. + Defaults to default_request_options.copy(). + data (Any, optional): _description_. Defaults to None. + json_dict (dict, optional): _description_. Defaults to None. + + Raises: + ValueError: _description_ + + Returns: + json: _description_ + """ + section = request_options.get('section') + url = request_options.get('url') + timeout_in_sec = request_options.get('timeout_in_sec') + headers = request_options.get('headers') + with_authentication = request_options.get('with_authentication') + if headers is None: headers = {} if with_authentication: @@ -165,12 +223,26 @@ def post_nomad_request( def delete_nomad_request( - section: str, - headers: dict = None, - url: str = None, - timeout_in_sec: int = TIMEOUT_IN_SEC, - with_authentication: bool = False, + request_options: RequestOptions = default_request_options.copy(), ) -> json: + """_summary_ + + Args: + request_options (RequestOptions, optional): _description_. + Defaults to default_request_options.copy(). + + Raises: + ValueError: _description_ + + Returns: + json: _description_ + """ + section = request_options.get('section') + url = request_options.get('url') + timeout_in_sec = request_options.get('timeout_in_sec') + headers = request_options.get('headers') + with_authentication = request_options.get('with_authentication') + if headers is None: headers = {} if with_authentication: diff --git a/src/nomad_utility_workflows/utils/datasets.py b/src/nomad_utility_workflows/utils/datasets.py index cfbb2e6..3f336b2 100644 --- a/src/nomad_utility_workflows/utils/datasets.py +++ b/src/nomad_utility_workflows/utils/datasets.py @@ -1,11 +1,12 @@ import datetime as dt import logging -from typing import Optional +from typing import Optional, TypedDict from marshmallow import Schema, pre_load from marshmallow_dataclass import class_schema, dataclass from nomad_utility_workflows.utils.core import ( + RequestOptions, delete_nomad_request, get_nomad_request, get_nomad_url, @@ -25,6 +26,23 @@ def convert_users(self, data, **kwargs): return data +class DatasetParams(TypedDict, total=False): + dataset_id: str + dataset_name: str + user_id: str + page_size: int + max_datasets: int + + +default_dataset_params = { + 'dataset_id': None, + 'dataset_name': None, + 'user_id': None, + 'page_size': 10, + 'max_datasets': 50, +} + + @dataclass(frozen=True) class NomadDataset: dataset_id: str @@ -39,21 +57,15 @@ class NomadDataset: def retrieve_datasets( - dataset_id: str = None, - dataset_name: str = None, - user_id: str = None, - page_size: int = 10, - max_datasets: int = 50, + dataset_params: DatasetParams = default_dataset_params.copy(), url: str = None, ) -> list[NomadDataset]: parameters = [] - if dataset_id: - parameters.append(f'dataset_id={dataset_id}') - if dataset_name: - parameters.append(f'dataset_name={dataset_name}') - if user_id: - parameters.append(f'user_id={user_id}') - parameters.append(f'page_size={page_size}') + max_datasets = dataset_params.pop( + 'max_datasets', default_dataset_params['max_datasets'] + ) + for key, value in dataset_params.items(): + parameters.append(f'{key}={value}') url = get_nomad_url(url) section = '/datasets/' if len(parameters) > 0: @@ -70,7 +82,9 @@ def retrieve_datasets( if page_after_value else section ) - response = get_nomad_request(section, headers=headers, url=url) + response = get_nomad_request( + RequestOptions(section=section, headers=headers, url=url) + ) if len(response['data']) == 0: break datasets.extend([nomad_entry_schema().load(d) for d in response['data']]) @@ -81,7 +95,7 @@ def retrieve_datasets( def get_dataset_by_id(dataset_id: str, url: str = None) -> NomadDataset: - datasets = retrieve_datasets(dataset_id=dataset_id, url=url) + datasets = retrieve_datasets(DatasetParams(dataset_id=dataset_id), url=url) if len(datasets) != 1: raise ValueError(f'Problem retrieving dataset {dataset_id}: {datasets}') return datasets[0] @@ -93,11 +107,13 @@ def create_dataset(dataset_name: str, url: str = None, timeout_in_sec: int = 10) logger.info('creating dataset name %s on %s server', dataset_name, url_name) json_dict = {'dataset_name': dataset_name} response = post_nomad_request( - '/datasets/', - with_authentication=True, + RequestOptions( + section='/datasets/', + with_authentication=True, + url=url, + timeout_in_sec=timeout_in_sec, + ), json_dict=json_dict, - url=url, - timeout_in_sec=timeout_in_sec, ) return response.get('dataset_id') @@ -107,10 +123,12 @@ def delete_dataset(dataset_id: str, url: str = None, timeout_in_sec: int = 10) - url_name = get_nomad_url_name(url) logger.info('deleting dataset %s on %s server', dataset_id, url_name) response = delete_nomad_request( - f'/datasets/{dataset_id}', - with_authentication=True, - url=url, - timeout_in_sec=timeout_in_sec, + RequestOptions( + section=f'/datasets/{dataset_id}', + with_authentication=True, + url=url, + timeout_in_sec=timeout_in_sec, + ) ) if response.get('dataset_id'): logger.info('successfully deleted dataset %s', dataset_id) diff --git a/src/nomad_utility_workflows/utils/entries.py b/src/nomad_utility_workflows/utils/entries.py index 912b0e2..228ce10 100644 --- a/src/nomad_utility_workflows/utils/entries.py +++ b/src/nomad_utility_workflows/utils/entries.py @@ -3,13 +3,14 @@ import logging from collections.abc import ByteString from dataclasses import field -from typing import Any, Optional +from typing import Any, Optional, TypedDict from cachetools.func import ttl_cache from marshmallow import Schema, pre_load from marshmallow_dataclass import class_schema, dataclass from nomad_utility_workflows.utils.core import ( + RequestOptions, get_nomad_base_url, get_nomad_request, get_nomad_url, @@ -53,6 +54,25 @@ def convert_users(self, data, **kwargs): return data +class QueryParams(TypedDict, total=False): + worfklow_name: str + program_name: str + dataset_id: str + origin: str + page_size: int + max_entries: int + + +default_query_params = { + 'worfklow_name': None, + 'program_name': None, + 'dataset_id': None, + 'origin': None, + 'page_size': 10, + 'max_entries': 50, +} + + # ? Should we just make these all optional to avoid the get functions breaking? @dataclass(frozen=True) class NomadEntry: @@ -108,7 +128,10 @@ def nomad_gui_url(self) -> str: raise ValueError( f"missing attributes 'upload_id' or 'entry_id' for entry {self}" ) - return f'{self.base_url}/gui/user/uploads/upload/id/{self.upload_id}/entry/id/{self.entry_id}' + return ( + f'{self.base_url}/gui/user/uploads/upload/id/{self.upload_id}/entry/id/' + f'{self.entry_id}' + ) @property def job_id(self) -> Optional[str]: @@ -142,10 +165,12 @@ def get_entry_by_id( url_name = get_nomad_url_name(url) logger.info('retrieving entry %s on %s server', entry_id, url_name) response = get_nomad_request( - f'/entries/{entry_id}', - with_authentication=with_authentication, - url=url, - timeout_in_sec=timeout_in_sec, + RequestOptions( + section=f'/entries/{entry_id}', + with_authentication=with_authentication, + url=url, + timeout_in_sec=timeout_in_sec, + ) ) nomad_entry_schema = class_schema(NomadEntry, base_schema=NomadEntrySchema) return nomad_entry_schema().load({**response['data'], 'url': url}) @@ -160,12 +185,14 @@ def get_entries_of_upload( ) -> list[NomadEntry]: url = get_nomad_url(url) url_name = get_nomad_url_name(url) - logger.info(f'retrieving entries for upload {upload_id} on {url_name} server') + logger.info('retrieving entries for upload %s on %s server', upload_id, url_name) response = get_nomad_request( - f'/uploads/{upload_id}/entries', - with_authentication=with_authentication, - url=url, - timeout_in_sec=timeout_in_sec, + RequestOptions( + section=f'/uploads/{upload_id}/entries', + with_authentication=with_authentication, + url=url, + timeout_in_sec=timeout_in_sec, + ) ) nomad_entry_schema = class_schema(NomadEntry, base_schema=NomadEntrySchema) return [ @@ -186,38 +213,41 @@ def get_entries_of_my_uploads( ] -# def get_entries_in_database(database_id: str = DEFAULT_DATABASE, use_prod: bool = DEFAULT_USE_PROD) -> list[NomadEntry]: -# return query_entries(dataset_id=database_id, use_prod=use_prod) - - -@ttl_cache(maxsize=128, ttl=180) +# @ttl_cache(maxsize=128, ttl=180) +# ! Had to remove caching because of the use of dict as input +# ! which was required to reduce the number of inputs for ruff def query_entries( - worfklow_name: str = None, - program_name: str = None, - dataset_id: str = None, - origin: str = None, - page_size: int = 10, - max_entries: int = 50, + query_params: QueryParams = default_query_params.copy(), url: str = None, ) -> list[NomadEntry]: json_dict = { 'query': {}, - 'pagination': {'page_size': page_size}, + 'pagination': { + 'page_size': query_params.pop( + 'page_size', default_query_params['page_size'] + ) + }, 'required': {'include': ['entry_id']}, } entries = [] + max_entries = query_params.pop('max_entries', default_query_params['max_entries']) while (max_entries > 0 and len(entries) <= max_entries) or (max_entries < 0): - if dataset_id: - json_dict['query']['datasets'] = {'dataset_id': dataset_id} - if worfklow_name: - json_dict['query']['results.method'] = {'workflow_name': worfklow_name} - if program_name: + if 'dataset_id' in query_params.keys(): + json_dict['query']['datasets'] = {'dataset_id': query_params['dataset_id']} + if 'worfklow_name' in query_params.keys(): json_dict['query']['results.method'] = { - 'simulation': {'program_name': program_name} + 'workflow_name': query_params['worfklow_name'] } - if origin: - json_dict['query']['origin'] = origin - query = post_nomad_request('/entries/query', json_dict=json_dict, url=url) + if 'program_name' in query_params.keys(): + json_dict['query']['results.method'] = { + 'simulation': {'program_name': query_params['program_name']} + } + if 'origin' in query_params.keys(): + json_dict['query']['origin'] = query_params['origin'] + + query = post_nomad_request( + RequestOptions(section='/entries/query', url=url), json_dict=json_dict + ) entries.extend([q['entry_id'] for q in query['data']]) next_page_after_value = query['pagination'].get('next_page_after_value', None) if next_page_after_value: @@ -240,10 +270,12 @@ def download_entry_raw_data_by_id( url_name = get_nomad_url_name(url) logger.info('retrieving raw data of entry ID %s on %s server', entry_id, url_name) response = get_nomad_request( - f'/entries/{entry_id}/raw?compress=true', - with_authentication=with_authentication, - url=url, - timeout_in_sec=timeout_in_sec, + RequestOptions( + section=f'/entries/{entry_id}/raw?compress=true', + with_authentication=with_authentication, + url=url, + timeout_in_sec=timeout_in_sec, + ), return_json=False, accept_field='application/zip', ) @@ -267,10 +299,12 @@ def download_entry_by_id( logger.info('retrieving data of entry ID %s on %s server', entry_id, url_name) response = get_nomad_request( - f'/entries/{entry_id}/archive/download', - with_authentication=with_authentication, - url=url, - timeout_in_sec=timeout_in_sec, + RequestOptions( + section=f'/entries/{entry_id}/archive/download', + with_authentication=with_authentication, + url=url, + timeout_in_sec=timeout_in_sec, + ) ) if zip_file_name is not None: diff --git a/src/nomad_utility_workflows/utils/uploads.py b/src/nomad_utility_workflows/utils/uploads.py index a40f949..7a659a6 100644 --- a/src/nomad_utility_workflows/utils/uploads.py +++ b/src/nomad_utility_workflows/utils/uploads.py @@ -1,12 +1,13 @@ import datetime as dt import logging -from typing import Any, Optional +from typing import Any, Optional, TypedDict from cachetools.func import ttl_cache from marshmallow import Schema, pre_load from marshmallow_dataclass import class_schema, dataclass from nomad_utility_workflows.utils.core import ( + RequestOptions, delete_nomad_request, get_nomad_base_url, get_nomad_request, @@ -31,6 +32,15 @@ def convert_users(self, data, **kwargs): return data +class UploadMetadata(TypedDict, total=False): + upload_name: str + references: list[str] + dataset_id: str + embargo_length: float + coauthors_ids: list[str] + comment: str + + @dataclass(frozen=True) class NomadUpload: upload_id: str @@ -85,10 +95,12 @@ def get_all_my_uploads(url: str = None, timeout_in_sec: int = 10) -> list[NomadU url_name = get_nomad_url_name(url) logger.info('retrieving all uploads on %s server', url_name) response = get_nomad_request( - '/uploads', - url=url, - with_authentication=True, - timeout_in_sec=timeout_in_sec, + RequestOptions( + section='/uploads', + url=url, + with_authentication=True, + timeout_in_sec=timeout_in_sec, + ) ) upload_class_schema = class_schema(NomadUpload, base_schema=NomadUploadSchema) return [upload_class_schema().load({**r, 'url': url}) for r in response['data']] @@ -101,10 +113,12 @@ def get_upload_by_id( url_name = get_nomad_url_name(url) logger.info('retrieving upload %s on %s server', upload_id, url_name) response = get_nomad_request( - f'/uploads/{upload_id}', - url=url, - with_authentication=True, - timeout_in_sec=timeout_in_sec, + RequestOptions( + section=f'/uploads/{upload_id}', + url=url, + with_authentication=True, + timeout_in_sec=timeout_in_sec, + ) ) upload_class_schema = class_schema(NomadUpload, base_schema=NomadUploadSchema) return upload_class_schema().load({**response['data'], 'url': url}) @@ -117,10 +131,12 @@ def delete_upload( url_name = get_nomad_url_name(url) logger.info(f'deleting upload {upload_id} on {url_name} server') response = delete_nomad_request( - f'/uploads/{upload_id}', - with_authentication=True, - url=url, - timeout_in_sec=timeout_in_sec, + RequestOptions( + section=f'/uploads/{upload_id}', + with_authentication=True, + url=url, + timeout_in_sec=timeout_in_sec, + ) ) upload_class_schema = class_schema(NomadUpload, base_schema=NomadUploadSchema) return upload_class_schema().load({**response['data'], 'url': url}) @@ -134,11 +150,13 @@ def upload_files_to_nomad( logger.info('uploading file %s on %s server', filename, url_name) with open(filename, 'rb') as f: response = post_nomad_request( - '/uploads', - with_authentication=True, + RequestOptions( + section='/uploads', + with_authentication=True, + url=url, + timeout_in_sec=timeout_in_sec, + ), data=f, - url=url, - timeout_in_sec=timeout_in_sec, ) upload_id = response.get('upload_id') if upload_id: @@ -153,22 +171,19 @@ def publish_upload(upload_id: str, url: str = None, timeout_in_sec: int = 10) -> url_name = get_nomad_url_name(url) logger.info('publishing upload %s on %s server', upload_id, url_name) response = post_nomad_request( - f'/uploads/{upload_id}/action/publish', - with_authentication=True, - url=url, - timeout_in_sec=timeout_in_sec, + RequestOptions( + section=f'/uploads/{upload_id}/action/publish', + with_authentication=True, + url=url, + timeout_in_sec=timeout_in_sec, + ) ) return response def edit_upload_metadata( upload_id: str, - upload_name: Optional[str] = None, - references: Optional[list[str]] = None, - dataset_id: Optional[str] = None, - embargo_length: Optional[float] = None, - coauthors_ids: Optional[list[str]] = None, - comment: Optional[str] = None, + upload_metadata: UploadMetadata = {}, url: str = None, timeout_in_sec: int = 10, ) -> dict: @@ -176,23 +191,17 @@ def edit_upload_metadata( url_name = get_nomad_url_name(url) logger.info('editing the metadata for upload %s on %s server', upload_id, url_name) metadata = {'metadata': {}} - if upload_name: - metadata['metadata']['upload_name'] = upload_name - if references: - metadata['metadata']['references'] = references - if dataset_id: - metadata['metadata']['datasets'] = dataset_id - if embargo_length: - metadata['metadata']['embargo_length'] = embargo_length - if coauthors_ids: - metadata['metadata']['coauthors'] = coauthors_ids - if comment: - metadata['metadata']['comment'] = comment + if 'dataset_id' in upload_metadata.keys(): + upload_metadata['datasets'] = upload_metadata.pop('dataset_id') + for key, value in upload_metadata.items(): + metadata['metadata'][key] = value response = post_nomad_request( - f'/uploads/{upload_id}/edit', - url=url, - with_authentication=True, + RequestOptions( + section=f'/uploads/{upload_id}/edit', + url=url, + with_authentication=True, + timeout_in_sec=timeout_in_sec, + ), json_dict=metadata, - timeout_in_sec=timeout_in_sec, ) return response diff --git a/src/nomad_utility_workflows/utils/users.py b/src/nomad_utility_workflows/utils/users.py index 03c2541..4ac2c05 100644 --- a/src/nomad_utility_workflows/utils/users.py +++ b/src/nomad_utility_workflows/utils/users.py @@ -7,6 +7,7 @@ from marshmallow_dataclass import class_schema, dataclass from nomad_utility_workflows.utils.core import ( + RequestOptions, get_nomad_request, get_nomad_url, get_nomad_url_name, @@ -42,7 +43,9 @@ def search_users_by_name( url_name = get_nomad_url_name(url) logger.info('retrieving user %s on %s server', user_name, url_name) response = get_nomad_request( - f'/users?prefix={user_name}', timeout_in_sec=timeout_in_sec, url=url + RequestOptions( + section=f'/users?prefix={user_name}', timeout_in_sec=timeout_in_sec, url=url + ) ).get('data', []) return [class_schema(NomadUser)().load(user) for user in response] @@ -55,7 +58,9 @@ def get_user_by_id( url_name = get_nomad_url_name(url) logger.info('retrieving user %s on %s server', user_id, url_name) response = get_nomad_request( - f'/users/{user_id}', timeout_in_sec=timeout_in_sec, url=url + RequestOptions( + section=f'/users/{user_id}', timeout_in_sec=timeout_in_sec, url=url + ) ) user_schema = class_schema(NomadUser) return user_schema().load(response) @@ -67,7 +72,12 @@ def who_am_i(url: str = None, timeout_in_sec: int = 10) -> NomadUser: url_name = get_nomad_url_name(url) logger.info('retrieving self user info on %s server', url_name) response = get_nomad_request( - '/users/me', with_authentication=True, timeout_in_sec=timeout_in_sec, url=url + RequestOptions( + section='/users/me', + with_authentication=True, + timeout_in_sec=timeout_in_sec, + url=url, + ) ) user_schema = class_schema(NomadUser) return user_schema().load(response) diff --git a/src/nomad_utility_workflows/utils/workflows.py b/src/nomad_utility_workflows/utils/workflows.py index 3a7a6df..bb4fc5f 100644 --- a/src/nomad_utility_workflows/utils/workflows.py +++ b/src/nomad_utility_workflows/utils/workflows.py @@ -1,6 +1,6 @@ import logging from collections import OrderedDict -from typing import Any, Dict, List, Literal, Optional, TypedDict, Union +from typing import Any, Literal, Optional, TypedDict, Union import networkx as nx import yaml @@ -48,14 +48,14 @@ class PathInfo(TypedDict, total=False): class NomadSection(BaseModel): name: Optional[str] = Field(None, description='Name of the section') type: Optional[SectionType] = Field(None, description='Type of the section') - path_info: Dict[str, Any] = Field( + path_info: dict[str, Any] = Field( default=default_path_info.copy(), description='Archive path' ) - inputs: List[Dict[str, Any]] = Field( + inputs: list[dict[str, Any]] = Field( [{}], description='section inputs', ) - outputs: List[Dict[str, Any]] = Field([{}], description='section outputs') + outputs: list[dict[str, Any]] = Field([{}], description='section outputs') def __init__(self, **data): super().__init__(**data) @@ -82,9 +82,9 @@ def archive_path(self) -> str: 'supersection_path' ]: # case 1 - supersection path is given archive_path = self.path_info['supersection_path'] - if ( - self.path_info.get('supersection_index') is not None - ): # add supersection index when given, else supersection is assumed to be nonrepeating + if self.path_info.get('supersection_index') is not None: + # add supersection index when given, else supersection is assumed + # to be nonrepeating archive_path += f'/{self.path_info.get("supersection_index")}' elif self.path_info.get('section_type') in [ 'system', @@ -93,12 +93,16 @@ def archive_path(self) -> str: ]: # case 2 - no supersection path, but section type is contained in run run_index = self.path_info.get('supersection_index') run_index = run_index if run_index is not None else -1 - archive_path = f'run/{run_index}' # add run index when given, else use last run section + # add run index when given, else use last run section + archive_path = f'run/{run_index}' elif self.path_info.get('section_type') in ['results']: archive_path = 'workflow2' else: logger.warning( - 'No supersection path provided for %s-%s. Section reference may be incorrect.', + ( + 'No supersection path provided for %s-%s. ' + 'Section reference may be incorrect.' + ), self.type, self.name, ) @@ -110,7 +114,10 @@ def archive_path(self) -> str: archive_path += f'/{self.path_info["section_index"]}' else: logger.warning( - 'No section type provided for %s-%s. Section reference may be incorrect.', + ( + 'No section type provided for %s-%s. ' + 'Section reference may be incorrect.' + ), self.type, self.name, ) @@ -121,7 +128,8 @@ def archive_path(self) -> str: def upload_prefix(self) -> str: if not self.path_info['mainfile_path']: logger.warning( - 'No mainfile path provided for %s-%s. Section reference will be missing.', + 'No mainfile path provided for %s-%s. ' + 'Section reference will be missing.', self.type, self.name, ) @@ -150,8 +158,8 @@ def to_dict(self) -> dict: class NomadTask(BaseModel): name: str m_def: str - inputs: List[NomadSection] = Field(default_factory=list) - outputs: List[NomadSection] = Field(default_factory=list) + inputs: list[NomadSection] = Field(default_factory=list) + outputs: list[NomadSection] = Field(default_factory=list) task_section: Optional[NomadSection] = None # class Config: @@ -220,9 +228,9 @@ def to_yaml(self, destination_filename: str) -> None: class NomadWorkflow(BaseModel): destination_filename: str - node_attributes: Dict[int, Any] = {} + node_attributes: dict[int, Any] = {} workflow_graph: nx.DiGraph = None - task_elements: Dict[str, NomadSection] = Field(default_factory=dict) + task_elements: dict[str, NomadSection] = Field(default_factory=dict) class Config: arbitrary_types_allowed = True @@ -235,13 +243,85 @@ def __init__(self, **data): self.fill_workflow_graph() def register_section( - self, node_key: Union[int, str, tuple], node_attrs: Dict[str, Any] + self, node_key: Union[int, str, tuple], node_attrs: dict[str, Any] ) -> None: section = NomadSection(**node_attrs) self.task_elements[node_key] = section # ! build the tasks section by section + # # TODO Extend the graph to add nodes for the additional default inouts etc + # def nodes_to_graph(self) -> nx.DiGraph: + # if not self.node_attributes: + # logger.error( + # 'No workflow graph or node attributes provided. + # Cannot build workflow.' + # ) + # return None + + # workflow_graph = nx.DiGraph() + # workflow_graph.add_nodes_from(self.node_attributes.keys()) + # nx.set_node_attributes(workflow_graph, self.node_attributes) + + # for node_key, node_attrs in list(workflow_graph.nodes(data=True)): + # # Add any given edges + # for edge in node_attrs.get('in_edge_nodes', []): + # workflow_graph.add_edge(edge, node_key) + # for edge in node_attrs.get('out_edge_nodes', []): + # workflow_graph.add_edge(node_key, edge) + # # Global inputs/outputs + # if node_attrs.get('type', '') == 'input': + # for edge_node in node_attrs.get('out_edge_nodes', []): + # workflow_graph.add_edge(node_key, edge_node) + # elif node_attrs.get('type', '') == 'output': + # for edge_node in node_attrs.get('in_edge_nodes', []): + # workflow_graph.add_edge(edge_node, node_key) + + # # Task inputs/outputs + # inputs = node_attrs.pop('inputs', []) + # for input_ in inputs: + # edge_nodes = input_.get('out_edge_nodes', []) + # if not edge_nodes: + # edge_nodes.append(len(workflow_graph.nodes)) + # workflow_graph.add_node(edge_nodes[0], type='input', **input_) + + # # transfer node inputs to edge ouputs + # for edge_node in edge_nodes: + # workflow_graph.add_edge(edge_node, node_key) + # if not workflow_graph.edges[edge_node, node_key].get( + # 'outputs', []): + # nx.set_edge_attributes( + # workflow_graph, {(edge_node, node_key): {'outputs': []}} + # ) + # workflow_graph.edges[edge_node, node_key]['outputs'].append( + # input_) + + # outputs = node_attrs.pop('outputs', []) + # for output_ in outputs: + # edge_nodes = output_.get('in_edge_node', []) + # if not edge_nodes: + # edge_nodes.append(len(workflow_graph.nodes)) + # workflow_graph.add_node(edge_nodes[0], type='output', **output_) + + # # transfer node outputs to edge inputs + # for edge_node in edge_nodes: + # workflow_graph.add_edge(node_key, edge_node) + # if not workflow_graph.edges[node_key, edge_node].get( + # 'inputs', []): + # nx.set_edge_attributes( + # workflow_graph, {(node_key, edge_node): {'inputs': []}} + # ) + # workflow_graph.edges[node_key, edge_node]['inputs'].append( + # output_) + + # return workflow_graph + + # ! suggested by copilot to fix too many branches ruff # TODO Extend the graph to add nodes for the additional default inouts etc def nodes_to_graph(self) -> nx.DiGraph: + """_summary_ + + Returns: + nx.DiGraph: _description_ + """ if not self.node_attributes: logger.error( 'No workflow graph or node attributes provided. Cannot build workflow.' @@ -253,176 +333,303 @@ def nodes_to_graph(self) -> nx.DiGraph: nx.set_node_attributes(workflow_graph, self.node_attributes) for node_key, node_attrs in list(workflow_graph.nodes(data=True)): - # Add any given edges - for edge in node_attrs.get('in_edge_nodes', []): - workflow_graph.add_edge(edge, node_key) - for edge in node_attrs.get('out_edge_nodes', []): - workflow_graph.add_edge(node_key, edge) - # Global inputs/outputs - if node_attrs.get('type', '') == 'input': - for edge_node in node_attrs.get('out_edge_nodes', []): - workflow_graph.add_edge(node_key, edge_node) - elif node_attrs.get('type', '') == 'output': - for edge_node in node_attrs.get('in_edge_nodes', []): - workflow_graph.add_edge(edge_node, node_key) - - # Task inputs/outputs - inputs = node_attrs.pop('inputs', []) - for input_ in inputs: - edge_nodes = input_.get('out_edge_nodes', []) - if not edge_nodes: - edge_nodes.append(len(workflow_graph.nodes)) - workflow_graph.add_node(edge_nodes[0], type='input', **input_) - - # transfer node inputs to edge ouputs - for edge_node in edge_nodes: - workflow_graph.add_edge(edge_node, node_key) - if not workflow_graph.edges[edge_node, node_key].get('outputs', []): - nx.set_edge_attributes( - workflow_graph, {(edge_node, node_key): {'outputs': []}} - ) - workflow_graph.edges[edge_node, node_key]['outputs'].append(input_) - - outputs = node_attrs.pop('outputs', []) - for output_ in outputs: - edge_nodes = output_.get('in_edge_node', []) - if not edge_nodes: - edge_nodes.append(len(workflow_graph.nodes)) - workflow_graph.add_node(edge_nodes[0], type='output', **output_) - - # transfer node outputs to edge inputs - for edge_node in edge_nodes: - workflow_graph.add_edge(node_key, edge_node) - if not workflow_graph.edges[node_key, edge_node].get('inputs', []): - nx.set_edge_attributes( - workflow_graph, {(node_key, edge_node): {'inputs': []}} - ) - workflow_graph.edges[node_key, edge_node]['inputs'].append(output_) + self._add_edges(workflow_graph, node_key, node_attrs) + self._add_global_inouts(workflow_graph, node_key, node_attrs) + self._add_task_inouts(workflow_graph, node_key, node_attrs) return workflow_graph - # TODO Change the archive building function to loop over nodes and then add the corrsponding inputs/outputs from the edges + def _add_edges(self, workflow_graph, node_key, node_attrs): + for edge in node_attrs.get('in_edge_nodes', []): + workflow_graph.add_edge(edge, node_key) + for edge in node_attrs.get('out_edge_nodes', []): + workflow_graph.add_edge(node_key, edge) + + def _add_global_inouts(self, workflow_graph, node_key, node_attrs): + if node_attrs.get('type', '') == 'input': + for edge_node in node_attrs.get('out_edge_nodes', []): + workflow_graph.add_edge(node_key, edge_node) + elif node_attrs.get('type', '') == 'output': + for edge_node in node_attrs.get('in_edge_nodes', []): + workflow_graph.add_edge(edge_node, node_key) + + def _add_task_inouts(self, workflow_graph, node_key, node_attrs): + inputs = node_attrs.pop('inputs', []) + for input_ in inputs: + edge_nodes = input_.get('out_edge_nodes', []) + if not edge_nodes: + edge_nodes.append(len(workflow_graph.nodes)) + workflow_graph.add_node(edge_nodes[0], type='input', **input_) + + for edge_node in edge_nodes: + workflow_graph.add_edge(edge_node, node_key) + if not workflow_graph.edges[edge_node, node_key].get('outputs', []): + nx.set_edge_attributes( + workflow_graph, {(edge_node, node_key): {'outputs': []}} + ) + workflow_graph.edges[edge_node, node_key]['outputs'].append(input_) + + outputs = node_attrs.pop('outputs', []) + for output_ in outputs: + edge_nodes = output_.get('in_edge_node', []) + if not edge_nodes: + edge_nodes.append(len(workflow_graph.nodes)) + workflow_graph.add_node(edge_nodes[0], type='output', **output_) + + for edge_node in edge_nodes: + workflow_graph.add_edge(node_key, edge_node) + if not workflow_graph.edges[node_key, edge_node].get('inputs', []): + nx.set_edge_attributes( + workflow_graph, {(node_key, edge_node): {'inputs': []}} + ) + workflow_graph.edges[node_key, edge_node]['inputs'].append(output_) + + # # TODO Change the archive building function to loop over nodes and then add the + # # corresponding inputs/outputs from the edges + # def fill_workflow_graph(self) -> None: + # """_summary_""" + + # def get_mainfile_path(node): + # return ( + # self.workflow_graph.nodes[node] + # .get('path_info', '') + # .get('mainfile_path', '') + # ) + + # def check_for_defaults(inout_type, default_section, edge) -> bool: + # inout_type = 'inputs' if inout_type == 'outputs' else 'outputs' + # for input_ in edge.get(inout_type, []): + # if ( + # input_.get('path_info', {}).get('section_type', '') + # == default_section + # ): + # return True + # return False + + # def get_defaults( + # inout_type: Literal['inputs', 'outputs'], node_source, node_dest + # ) -> list: + # defaults = { + # 'inputs': { + # 'section': 'system', + # }, + # 'outputs': { + # 'section': 'calculation', + # }, + # } + # partner_node = node_source + # node_source_type = self.workflow_graph.nodes[node_source].get('type', '') + # if node_source_type == 'input': + # partner_node = node_dest + + # default_section = defaults[inout_type]['section'] + # flag_defaults = False + # if inout_type == 'outputs': + # for _, _, edge2 in self.workflow_graph.out_edges( + # node_source, data=True + # ): + # if check_for_defaults(inout_type, default_section, edge2): + # flag_defaults = True + # break + # elif inout_type == 'inputs': + # for _, _, edge2 in self.workflow_graph.in_edges(node_dest, data=True): + # if check_for_defaults(inout_type, default_section, edge2): + # flag_defaults = True + # break + # if flag_defaults: + # return [] + + # partner_name = self.workflow_graph.nodes[partner_node].get('name', '') + # inouts = [ + # { + # 'name': ( + # f'DEFAULT {inout_type[:-1]} {default_section} ' + # f'from {partner_name}' + # ), + # 'path_info': { + # 'section_type': default_section, + # 'mainfile_path': get_mainfile_path(partner_node), + # }, + # }, + # ] + + # return inouts + + # # resolve mainfile for all edge inouts and add defaults + # for node_source, node_dest, edge in self.workflow_graph.edges(data=True): + # # EDGE INPUTS + # if not edge.get('inputs'): + # nx.set_edge_attributes( + # self.workflow_graph, {(node_source, node_dest): {'inputs': []}} + # ) + # for input_ in edge['inputs']: + # if not input_.get('path_info', {}): + # continue + # if not input_['path_info'].get('mainfile_path', ''): + # # edge inputs always coming from the source node + # input_['path_info']['mainfile_path'] = get_mainfile_path( + # node_source + # ) + + # # EDGE OUTPUTS + # if not edge.get('outputs'): + # nx.set_edge_attributes( + # self.workflow_graph, {(node_source, node_dest): {'outputs': []}} + # ) + # for output_ in edge.get('outputs', []): + # if not output_.get('path_info', {}): + # continue + # if not output_['path_info'].get('mainfile_path', ''): + # node_source_type = self.workflow_graph.nodes[node_source].get( + # 'type', '' + # ) + # # edge output assigned to source unless source is an input node + # if node_source_type == 'input': + # # ! assuming here that the input is coming from the same + # # archive, but will not be assigned anyway if path_info is + # empty + # # for this node + # output_['path_info']['mainfile_path'] = get_mainfile_path( + # node_dest + # ) + # else: + # output_['path_info']['mainfile_path'] = get_mainfile_path( + # node_source + # ) + + # # ADD DEFAULTS + # # ? Here I am added the default to the first edge in case they are + # missing, + # # not positive this covers all cases correctly + # # edge_input is source output + # if self.workflow_graph.nodes[node_source].get('type', '') in [ + # 'task', + # 'workflow', + # ]: + # for outputs_ in get_defaults('outputs', node_source, node_dest): + # edge['inputs'].append(outputs_) + + # # edge_output is dest input + # if self.workflow_graph.nodes[node_dest].get('type', '') in [ + # 'task', + # 'workflow', + # ]: + # for inputs_ in get_defaults('inputs', node_source, node_dest): + # edge['outputs'].append(inputs_) + + # ! Suggested by copilot to replace the above in order to fix too many branches ruff def fill_workflow_graph(self) -> None: - def get_mainfile_path(node): - return ( - self.workflow_graph.nodes[node] - .get('path_info', '') - .get('mainfile_path', '') + """_summary_""" + for node_source, node_dest, edge in self.workflow_graph.edges(data=True): + self._resolve_edge_inputs(node_source, node_dest, edge) + self._resolve_edge_outputs(node_source, node_dest, edge) + self._add_defaults(node_source, node_dest, edge) + + def _resolve_edge_inputs(self, node_source, node_dest, edge) -> None: + if not edge.get('inputs'): + nx.set_edge_attributes( + self.workflow_graph, {(node_source, node_dest): {'inputs': []}} ) + for input_ in edge['inputs']: + if not input_.get('path_info', {}): + continue + if not input_['path_info'].get('mainfile_path', ''): + input_['path_info']['mainfile_path'] = self._get_mainfile_path( + node_source + ) - def check_for_defaults(inout_type, default_section, edge) -> bool: - inout_type = 'inputs' if inout_type == 'outputs' else 'outputs' - for input_ in edge.get(inout_type, []): - if ( - input_.get('path_info', {}).get('section_type', '') - == default_section - ): - return True - return False - - def get_defaults( - inout_type: Literal['inputs', 'outputs'], node_source, node_dest - ) -> List: - defaults = { - 'inputs': { - 'section': 'system', - }, - 'outputs': { - 'section': 'calculation', - }, - } - partner_node = node_source - node_source_type = self.workflow_graph.nodes[node_source].get('type', '') - if node_source_type == 'input': - partner_node = node_dest - - default_section = defaults[inout_type]['section'] - flag_defaults = False - if inout_type == 'outputs': - for _, _, edge2 in self.workflow_graph.out_edges( - node_source, data=True - ): - if check_for_defaults(inout_type, default_section, edge2): - flag_defaults = True - break - elif inout_type == 'inputs': - for _, _, edge2 in self.workflow_graph.in_edges(node_dest, data=True): - if check_for_defaults(inout_type, default_section, edge2): - flag_defaults = True - break - if flag_defaults: - return [] - - partner_name = self.workflow_graph.nodes[partner_node].get('name', '') - inouts = [ - { - 'name': f'DEFAULT {inout_type[:-1]} {default_section} from {partner_name}', - 'path_info': { - 'section_type': default_section, - 'mainfile_path': get_mainfile_path(partner_node), - }, - }, - ] - - return inouts - - # resolve mainfile for all edge inouts and add defaults - for node_source, node_dest, edge in self.workflow_graph.edges(data=True): - # EDGE INPUTS - if not edge.get('inputs'): - nx.set_edge_attributes( - self.workflow_graph, {(node_source, node_dest): {'inputs': []}} + def _resolve_edge_outputs(self, node_source, node_dest, edge) -> None: + if not edge.get('outputs'): + nx.set_edge_attributes( + self.workflow_graph, {(node_source, node_dest): {'outputs': []}} + ) + for output_ in edge.get('outputs', []): + if not output_.get('path_info', {}): + continue + if not output_['path_info'].get('mainfile_path', ''): + node_source_type = self.workflow_graph.nodes[node_source].get( + 'type', '' ) - for input_ in edge['inputs']: - if not input_.get('path_info', {}): - continue - if not input_['path_info'].get('mainfile_path', ''): - # edge inputs always coming from the source node - input_['path_info']['mainfile_path'] = get_mainfile_path( + if node_source_type == 'input': + output_['path_info']['mainfile_path'] = self._get_mainfile_path( + node_dest + ) + else: + output_['path_info']['mainfile_path'] = self._get_mainfile_path( node_source ) - # EDGE OUTPUTS - if not edge.get('outputs'): - nx.set_edge_attributes( - self.workflow_graph, {(node_source, node_dest): {'outputs': []}} - ) - for output_ in edge.get('outputs', []): - if not output_.get('path_info', {}): - continue - if not output_['path_info'].get('mainfile_path', ''): - node_source_type = self.workflow_graph.nodes[node_source].get( - 'type', '' - ) - # edge output assigned to source unless source is an input node - if ( - node_source_type == 'input' - ): # ! assuming here that the input is coming from the same archive, but will not be assigned anyway if path_info is empty for this node - output_['path_info']['mainfile_path'] = get_mainfile_path( - node_dest - ) - else: - output_['path_info']['mainfile_path'] = get_mainfile_path( - node_source - ) - - # ADD DEFAULTS - # ? Here I am added the default to the first edge in case they are missing, not positive this covers all cases correctly - # edge_input is source output - if self.workflow_graph.nodes[node_source].get('type', '') in [ - 'task', - 'workflow', - ]: - for outputs_ in get_defaults('outputs', node_source, node_dest): - edge['inputs'].append(outputs_) - - # edge_output is dest input - if self.workflow_graph.nodes[node_dest].get('type', '') in [ - 'task', - 'workflow', - ]: - for inputs_ in get_defaults('inputs', node_source, node_dest): - edge['outputs'].append(inputs_) + def _add_defaults(self, node_source, node_dest, edge) -> None: + if self.workflow_graph.nodes[node_source].get('type', '') in [ + 'task', + 'workflow', + ]: + for outputs_ in self._get_defaults('outputs', node_source, node_dest): + edge['inputs'].append(outputs_) + if self.workflow_graph.nodes[node_dest].get('type', '') in ['task', 'workflow']: + for inputs_ in self._get_defaults('inputs', node_source, node_dest): + edge['outputs'].append(inputs_) + + def _get_mainfile_path(self, node): + return ( + self.workflow_graph.nodes[node] + .get('path_info', '') + .get('mainfile_path', '') + ) + + def _check_for_defaults(self, inout_type, default_section, edge) -> bool: + inout_type = 'inputs' if inout_type == 'outputs' else 'outputs' + for input_ in edge.get(inout_type, []): + if input_.get('path_info', {}).get('section_type', '') == default_section: + return True + return False + + def _get_defaults( + self, inout_type: Literal['inputs', 'outputs'], node_source, node_dest + ) -> list: + defaults = { + 'inputs': { + 'section': 'system', + }, + 'outputs': { + 'section': 'calculation', + }, + } + partner_node = node_source + node_source_type = self.workflow_graph.nodes[node_source].get('type', '') + if node_source_type == 'input': + partner_node = node_dest + + default_section = defaults[inout_type]['section'] + flag_defaults = False + if inout_type == 'outputs': + for _, _, edge2 in self.workflow_graph.out_edges(node_source, data=True): + if self._check_for_defaults(inout_type, default_section, edge2): + flag_defaults = True + break + elif inout_type == 'inputs': + for _, _, edge2 in self.workflow_graph.in_edges(node_dest, data=True): + if self._check_for_defaults(inout_type, default_section, edge2): + flag_defaults = True + break + if flag_defaults: + return [] + + partner_name = self.workflow_graph.nodes[partner_node].get('name', '') + inouts = [ + { + 'name': ( + f'DEFAULT {inout_type[:-1]} {default_section} ' + f'from {partner_name}' + ), + 'path_info': { + 'section_type': default_section, + 'mainfile_path': self._get_mainfile_path(partner_node), + }, + }, + ] + + return inouts def build_workflow_yaml(self) -> None: # register the sections and build task_elements @@ -471,7 +678,7 @@ def generate_archive(self) -> NomadWorkflowArchive: def build_nomad_workflow( destination_filename: str = './nomad_workflow.archive.yaml', - node_attributes: Dict[int, Any] = {}, + node_attributes: dict[int, Any] = {}, workflow_graph: nx.DiGraph = None, write_to_yaml: bool = False, ) -> nx.DiGraph: @@ -486,13 +693,16 @@ def build_nomad_workflow( return workflow.workflow_graph -# TODO make sure that the post_nomad etc with authentication are passing the correct urls without sections added! +# TODO make sure that the post_nomad etc with authentication are passing the correct +# urls without sections added! # TODO add is_simulation, is_nomad_entry as flags # TODO test this code on a number of already existing examples # TODO create docs with some examples for dict and graph input types -# TODO add to readme/docs that this is not currently using NOMAD, but could be linked later? -# TODO rename utils.py -# TODO should nodes_to_graph() be an external function from the class? So, that the user can call it, but also add attributes from there? +# TODO add to readme/docs that this is not currently using NOMAD, but could be linked +# later? +# TODO should nodes_to_graph() be an external function from the class? So, that the user +# can call it, but also add attributes from there? # TODO add some text to the test notebooks -# TODO change the rest of the functions to pydantic -- not sure if I really want to tackle this now +# TODO change the rest of the functions to pydantic -- not sure if I really want to +# tackle this now diff --git a/tests/utils/how_to_use_api_functions.ipynb b/tests/utils/how_to_use_api_functions.ipynb index b9c3331..b7af314 100644 --- a/tests/utils/how_to_use_api_functions.ipynb +++ b/tests/utils/how_to_use_api_functions.ipynb @@ -193,7 +193,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "metadata": {}, "outputs": [ { @@ -202,7 +202,7 @@ "NomadUser(name='Joseph Rudzinski')" ] }, - "execution_count": 4, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } @@ -221,7 +221,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "metadata": {}, "outputs": [ { @@ -230,7 +230,7 @@ "[NomadUser(name='Joseph Rudzinski'), NomadUser(name='Joseph Rudzinski')]" ] }, - "execution_count": 5, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } @@ -249,7 +249,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -258,7 +258,7 @@ "NomadUser(name='Joseph Rudzinski')" ] }, - "execution_count": 6, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } @@ -318,25 +318,27 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ - "test_upload_fnm = './test.zip' # a dummy upload file containing a single empty json file" + "test_upload_fnm = (\n", + " './test.zip' # a dummy upload file containing a single empty json file\n", + ")" ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 9, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "'RdA_3ZsOTMqbtAhYLivVsw'" + "'e2b5o4KSR5yoS1EV9E9jXQ'" ] }, - "execution_count": 8, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } @@ -357,20 +359,20 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 10, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "NomadUpload(upload_id='RdA_3ZsOTMqbtAhYLivVsw',\n", - " upload_create_time=datetime.datetime(2024, 10, 15, 20, 2, 10, 378000),\n", + "NomadUpload(upload_id='e2b5o4KSR5yoS1EV9E9jXQ',\n", + " upload_create_time=datetime.datetime(2024, 10, 16, 10, 14, 57, 722000),\n", " main_author=NomadUser(name='Joseph Rudzinski'),\n", - " process_running=False,\n", + " process_running=True,\n", " current_process='process_upload',\n", - " process_status='SUCCESS',\n", - " last_status_message='Process process_upload completed successfully',\n", + " process_status='RUNNING',\n", + " last_status_message='Cleanup',\n", " errors=[],\n", " warnings=[],\n", " coauthors=[],\n", @@ -388,7 +390,7 @@ " license='CC BY 4.0',\n", " entries=1,\n", " n_entries=None,\n", - " upload_files_server_path='/nomad/test/fs/staging/R/RdA_3ZsOTMqbtAhYLivVsw',\n", + " upload_files_server_path='/nomad/test/fs/staging/e/e2b5o4KSR5yoS1EV9E9jXQ',\n", " publish_time=None,\n", " references=None,\n", " datasets=None,\n", @@ -396,7 +398,7 @@ " upload_name=None,\n", " comment=None,\n", " url='https://nomad-lab.eu/prod/v1/test/api/v1',\n", - " complete_time=datetime.datetime(2024, 10, 15, 20, 2, 11, 320000))\n" + " complete_time=None)\n" ] } ], @@ -460,22 +462,22 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 11, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "{'upload_id': 'RdA_3ZsOTMqbtAhYLivVsw',\n", + "{'upload_id': 'e2b5o4KSR5yoS1EV9E9jXQ',\n", " 'data': {'process_running': True,\n", " 'current_process': 'edit_upload_metadata',\n", " 'process_status': 'PENDING',\n", " 'last_status_message': 'Pending: edit_upload_metadata',\n", " 'errors': [],\n", " 'warnings': [],\n", - " 'complete_time': '2024-10-15T20:02:11.320000',\n", - " 'upload_id': 'RdA_3ZsOTMqbtAhYLivVsw',\n", - " 'upload_create_time': '2024-10-15T20:02:10.378000',\n", + " 'complete_time': '2024-10-16T10:14:58.322000',\n", + " 'upload_id': 'e2b5o4KSR5yoS1EV9E9jXQ',\n", + " 'upload_create_time': '2024-10-16T10:14:57.722000',\n", " 'main_author': '8f052e1f-1906-41fd-b2eb-690c03407788',\n", " 'coauthors': [],\n", " 'coauthor_groups': [],\n", @@ -491,17 +493,17 @@ " 'embargo_length': 0,\n", " 'license': 'CC BY 4.0',\n", " 'entries': 1,\n", - " 'upload_files_server_path': '/nomad/test/fs/staging/R/RdA_3ZsOTMqbtAhYLivVsw'}}" + " 'upload_files_server_path': '/nomad/test/fs/staging/e/e2b5o4KSR5yoS1EV9E9jXQ'}}" ] }, - "execution_count": 10, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ "metadata_new = {'upload_name': 'Test Upload', 'comment': 'This is a test upload...'}\n", - "edit_upload_metadata(upload_id, url='test', **metadata_new)" + "edit_upload_metadata(upload_id, url='test', upload_metadata=metadata_new)" ] }, { @@ -513,7 +515,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 12, "metadata": {}, "outputs": [ { @@ -591,15 +593,15 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 13, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "'Entries within upload_id=RdA_3ZsOTMqbtAhYLivVsw:'\n", - "'entry_id=Htbl78lHDSNAKbvPjEgEN_6sOcxF'\n" + "'Entries within upload_id=e2b5o4KSR5yoS1EV9E9jXQ:'\n", + "'entry_id=u4qPSfILguvJ9fabpklMTxbUJ7x2'\n" ] } ], @@ -619,16 +621,16 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 14, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "NomadEntry(entry_id='Htbl78lHDSNAKbvPjEgEN_6sOcxF', upload_id='RdA_3ZsOTMqbtAhYLivVsw', references=[], origin='Joseph Rudzinski', n_quantities=0, nomad_version='1.3.7.dev55+ge83de27b3', upload_create_time=datetime.datetime(2024, 10, 15, 20, 2, 10, 378000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), nomad_commit='', processing_errors=[], entry_name='test.archive.json', last_processing_time=datetime.datetime(2024, 10, 15, 20, 2, 10, 752000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), parser_name='parsers/archive', calc_id='Htbl78lHDSNAKbvPjEgEN_6sOcxF', published=False, writers=[NomadUser(name='Joseph Rudzinski')], processed=True, mainfile='test.archive.json', main_author=NomadUser(name='Joseph Rudzinski'), entry_create_time=datetime.datetime(2024, 10, 15, 20, 2, 10, 543000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), with_embargo=False, entry_type=None, license='CC BY 4.0', domain=None, comment='This is a test upload...', upload_name='Test Upload', text_search_contents=[], publish_time=None, entry_references=None, url='https://nomad-lab.eu/prod/v1/test/api/v1')" + "NomadEntry(entry_id='u4qPSfILguvJ9fabpklMTxbUJ7x2', upload_id='e2b5o4KSR5yoS1EV9E9jXQ', references=[], origin='Joseph Rudzinski', n_quantities=0, nomad_version='1.3.7.dev55+ge83de27b3', upload_create_time=datetime.datetime(2024, 10, 16, 10, 14, 57, 722000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), nomad_commit='', processing_errors=[], entry_name='test.archive.json', last_processing_time=datetime.datetime(2024, 10, 16, 10, 14, 58, 23000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), parser_name='parsers/archive', calc_id='u4qPSfILguvJ9fabpklMTxbUJ7x2', published=False, writers=[NomadUser(name='Joseph Rudzinski')], processed=True, mainfile='test.archive.json', main_author=NomadUser(name='Joseph Rudzinski'), entry_create_time=datetime.datetime(2024, 10, 16, 10, 14, 57, 855000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), with_embargo=False, entry_type=None, license='CC BY 4.0', domain=None, comment='This is a test upload...', upload_name='Test Upload', text_search_contents=[], publish_time=None, entry_references=None, url='https://nomad-lab.eu/prod/v1/test/api/v1')" ] }, - "execution_count": 13, + "execution_count": 14, "metadata": {}, "output_type": "execute_result" } @@ -659,9 +661,9 @@ " 'process_worker_id': 'BOiybXorRqW5XImFf0SyoA',\n", " 'parser': 'parsers/archive',\n", " 'logger': 'nomad.processing',\n", - " 'timestamp': '2024-10-15 20:02.10',\n", + " 'timestamp': '2024-10-16 10:14.57',\n", " 'level': 'DEBUG'},\n", - " {'exec_time': '0.001298666000366211',\n", + " {'exec_time': '0.0013990402221679688',\n", " 'input_size': '3',\n", " 'event': 'parser executed',\n", " 'proc': 'Entry',\n", @@ -670,7 +672,7 @@ " 'parser': 'parsers/archive',\n", " 'step': 'parsers/archive',\n", " 'logger': 'nomad.processing',\n", - " 'timestamp': '2024-10-15 20:02.10',\n", + " 'timestamp': '2024-10-16 10:14.58',\n", " 'level': 'INFO'},\n", " {'normalizer': 'MetainfoNormalizer',\n", " 'step': 'MetainfoNormalizer',\n", @@ -680,9 +682,9 @@ " 'process_worker_id': 'BOiybXorRqW5XImFf0SyoA',\n", " 'parser': 'parsers/archive',\n", " 'logger': 'nomad.processing',\n", - " 'timestamp': '2024-10-15 20:02.10',\n", + " 'timestamp': '2024-10-16 10:14.58',\n", " 'level': 'INFO'},\n", - " {'exec_time': '0.0011508464813232422',\n", + " {'exec_time': '0.0006549358367919922',\n", " 'input_size': '3',\n", " 'event': 'normalizer executed',\n", " 'proc': 'Entry',\n", @@ -692,7 +694,7 @@ " 'normalizer': 'MetainfoNormalizer',\n", " 'step': 'MetainfoNormalizer',\n", " 'logger': 'nomad.processing',\n", - " 'timestamp': '2024-10-15 20:02.10',\n", + " 'timestamp': '2024-10-16 10:14.58',\n", " 'level': 'INFO'},\n", " {'normalizer': 'ResultsNormalizer',\n", " 'step': 'ResultsNormalizer',\n", @@ -702,9 +704,9 @@ " 'process_worker_id': 'BOiybXorRqW5XImFf0SyoA',\n", " 'parser': 'parsers/archive',\n", " 'logger': 'nomad.processing',\n", - " 'timestamp': '2024-10-15 20:02.10',\n", + " 'timestamp': '2024-10-16 10:14.58',\n", " 'level': 'INFO'},\n", - " {'exec_time': '0.0011796951293945312',\n", + " {'exec_time': '0.0005123615264892578',\n", " 'input_size': '3',\n", " 'event': 'normalizer executed',\n", " 'proc': 'Entry',\n", @@ -714,32 +716,32 @@ " 'normalizer': 'ResultsNormalizer',\n", " 'step': 'ResultsNormalizer',\n", " 'logger': 'nomad.processing',\n", - " 'timestamp': '2024-10-15 20:02.10',\n", + " 'timestamp': '2024-10-16 10:14.58',\n", " 'level': 'INFO'},\n", - " {'exec_time': '0.002213716506958008',\n", + " {'exec_time': '0.0015552043914794922',\n", " 'event': 'entry metadata saved',\n", " 'proc': 'Entry',\n", " 'process': 'process_entry',\n", " 'process_worker_id': 'BOiybXorRqW5XImFf0SyoA',\n", " 'parser': 'parsers/archive',\n", " 'logger': 'nomad.processing',\n", - " 'timestamp': '2024-10-15 20:02.10',\n", + " 'timestamp': '2024-10-16 10:14.58',\n", " 'level': 'INFO'},\n", - " {'exec_time': '0.07823586463928223',\n", + " {'exec_time': '0.09751391410827637',\n", " 'event': 'entry metadata indexed',\n", " 'proc': 'Entry',\n", " 'process': 'process_entry',\n", " 'process_worker_id': 'BOiybXorRqW5XImFf0SyoA',\n", " 'parser': 'parsers/archive',\n", " 'logger': 'nomad.processing',\n", - " 'timestamp': '2024-10-15 20:02.10',\n", + " 'timestamp': '2024-10-16 10:14.58',\n", " 'level': 'INFO'}],\n", - " 'metadata': {'upload_id': 'RdA_3ZsOTMqbtAhYLivVsw',\n", - " 'upload_create_time': '2024-10-15T20:02:10.378000+00:00',\n", - " 'entry_id': 'Htbl78lHDSNAKbvPjEgEN_6sOcxF',\n", + " 'metadata': {'upload_id': 'e2b5o4KSR5yoS1EV9E9jXQ',\n", + " 'upload_create_time': '2024-10-16T10:14:57.722000+00:00',\n", + " 'entry_id': 'u4qPSfILguvJ9fabpklMTxbUJ7x2',\n", " 'entry_name': 'test.archive.json',\n", " 'entry_hash': 't6Zf68GLfrWxWRAIQu7QAY8LVmlL',\n", - " 'entry_create_time': '2024-10-15T20:02:10.543000+00:00',\n", + " 'entry_create_time': '2024-10-16T10:14:57.855000+00:00',\n", " 'parser_name': 'parsers/archive',\n", " 'mainfile': 'test.archive.json',\n", " 'text_search_contents': [],\n", @@ -749,7 +751,7 @@ " 'embargo_length': 0,\n", " 'license': 'CC BY 4.0',\n", " 'processed': True,\n", - " 'last_processing_time': '2024-10-15T20:02:10.752287+00:00',\n", + " 'last_processing_time': '2024-10-16T10:14:58.023933+00:00',\n", " 'processing_errors': [],\n", " 'nomad_version': '1.3.7.dev55+ge83de27b3',\n", " 'nomad_commit': '',\n", @@ -809,9 +811,9 @@ " 'nomad.datamodel.results.Properties',\n", " 'nomad.datamodel.results.Results'],\n", " 'entry_timestamp': {'token_seed': 't6Zf68GLfrWxWRAIQu7QAY8LVmlL',\n", - " 'token': 'MIIEQwYJKoZIhvcNAQcCoIIENDCCBDACAQMxDTALBglghkgBZQMEAgEwfAYLKoZIhvcNAQkQAQSgbQRrMGkCAQEGDCsGAQQBga0hgiwWATAvMAsGCWCGSAFlAwQCAQQgYnRB2tk2mTRtMamyedr2QFd3bb0lFM56N52xD8rv/OECFFIumicGBkskyLovIs3OBywh3EXbGA8yMDI0MTAxNTIwMDIxMFoxggOcMIIDmAIBATCBnjCBjTELMAkGA1UEBhMCREUxRTBDBgNVBAoMPFZlcmVpbiB6dXIgRm9lcmRlcnVuZyBlaW5lcyBEZXV0c2NoZW4gRm9yc2NodW5nc25ldHplcyBlLiBWLjEQMA4GA1UECwwHREZOLVBLSTElMCMGA1UEAwwcREZOLVZlcmVpbiBHbG9iYWwgSXNzdWluZyBDQQIMKQLVczMPeOL0nrS5MAsGCWCGSAFlAwQCAaCB0TAaBgkqhkiG9w0BCQMxDQYLKoZIhvcNAQkQAQQwHAYJKoZIhvcNAQkFMQ8XDTI0MTAxNTIwMDIxMFowKwYJKoZIhvcNAQk0MR4wHDALBglghkgBZQMEAgGhDQYJKoZIhvcNAQELBQAwLwYJKoZIhvcNAQkEMSIEIMF/vhL+ddm6reCVHQgPz5FLJYZKE3ag+HqzfGTfwTvPMDcGCyqGSIb3DQEJEAIvMSgwJjAkMCIEILYIjb3dCJjTSQeNfCMyp07MhBQMoINZ8CNXJUbPboLkMA0GCSqGSIb3DQEBCwUABIICAJSu4GSAVDGNwuA+Kr5Qhi7rrcQcZpAcA2TOotKVS9b8wDyCE+J7IwobbVDIVURsa0b9QsReNUZHc+U9TmlWGprwY1j1BVy+ccXNg2U2Uf0dMrnOzVWfPNAoMT1iv9tK3kMwq2Gal35yh/Arrp+XYMmLfKFRZpzNcjz0TFokjCFbrPxreLCHgSnPWy3VHncNWghyMg6Nxq0rUwvV7dp7cjt1R9Ky6eHdxQvyMmVuSRbTetg+B43KkrYXoqGNGVEqiaScZWJswM5jFApNquUZnOuRl4/bhABNGWpLXQRxDz2r4Bqvgz4DfQt/8EPg7YvWNWNzfw+oPXgh5dDqKW9DiKoa0U2J1+/YKnBcdJefDsyZHnOcXOAIaX/f8k9Wg90v+c/WrbSjbfYMMhJBGqKYgU6DAeh7p7DZQNeDS4qspVWaTb10zgxmnkCbtqzmlzJvY4pOnjirVHfMFyXxL1rkXG91swhjK6FnL/okQKVT2tGQHC4I15VnEmrhit/ptXJHpWABO6TKJCW4oCgUdxUBPMT3bY7RaT3Fe+XzjjRuau0dVd2bYIbUTlrGFBUjJf+9Zuj145FdqdSjezx7NaIy7zsF1wV/e9feu6vbu3kEu32hZOT7Agw/Ryqrqgx4KuJuelOlwTljeQVHUfblr6yhwNDzdxufsQyAbJAvRUS2tyMk',\n", + " 'token': 'MIIERAYJKoZIhvcNAQcCoIIENTCCBDECAQMxDTALBglghkgBZQMEAgEwfQYLKoZIhvcNAQkQAQSgbgRsMGoCAQEGDCsGAQQBga0hgiwWATAvMAsGCWCGSAFlAwQCAQQgYnRB2tk2mTRtMamyedr2QFd3bb0lFM56N52xD8rv/OECFQDfPonFJHlhxHevtRkQ/Z85hZIntRgPMjAyNDEwMTYxMDE0NTdaMYIDnDCCA5gCAQEwgZ4wgY0xCzAJBgNVBAYTAkRFMUUwQwYDVQQKDDxWZXJlaW4genVyIEZvZXJkZXJ1bmcgZWluZXMgRGV1dHNjaGVuIEZvcnNjaHVuZ3NuZXR6ZXMgZS4gVi4xEDAOBgNVBAsMB0RGTi1QS0kxJTAjBgNVBAMMHERGTi1WZXJlaW4gR2xvYmFsIElzc3VpbmcgQ0ECDCkC1XMzD3ji9J60uTALBglghkgBZQMEAgGggdEwGgYJKoZIhvcNAQkDMQ0GCyqGSIb3DQEJEAEEMBwGCSqGSIb3DQEJBTEPFw0yNDEwMTYxMDE0NTdaMCsGCSqGSIb3DQEJNDEeMBwwCwYJYIZIAWUDBAIBoQ0GCSqGSIb3DQEBCwUAMC8GCSqGSIb3DQEJBDEiBCA6s2JcwOqwu7132Qqq//qLCZ/RvK+vbG2KEmFaDWTOezA3BgsqhkiG9w0BCRACLzEoMCYwJDAiBCC2CI293QiY00kHjXwjMqdOzIQUDKCDWfAjVyVGz26C5DANBgkqhkiG9w0BAQsFAASCAgBq9WFjtr/M3uah2LM9SuT7Mg/clX91fkKHk+/a+Y4gZGzOGdeE4P3AlHKDsNmOqghgvX7EVXgMadc+Hyf72YGNaqrca3BFYivuI4NqbjX7+wC9vq0SZIn+5d3TBCCZisZmAtZHTj7+KZpD071qGLMmsaCi+qOTQExgB0G7DRYeGibq3slZXKTr2wY8frQDB6M36y69nPgovO4azvy+553w2EQQkAGfemSxz78S9A2jcV9frKxOIGXUb5R/FVYGfJfCM/C+d35iUGisi9ZWHl0PvSe2Zzwv+4UQRr4ljEGrnfDkdKrV1ZluxerBNrzxcYU+DpWUPs8NZdfQibSkmc9aMwvznrqB4GL3iW+8RrQrGSgiIm84HKlYp6TNCWldlGlV3Xfgo+6AO3hZv5v7tiM7GWLL6/fFi4IRPaIGCldL9D0Yun6U95vaiy+jRT1yzZx4pPlgM3JbSp4z/BkmTURBhHySUsr91LkB8OAiS5LKEZye7qJTuBdI9Ny7TNB050c5+mmfv/TKmbExIZoYvChXQKCUMCCLLSFmxN4eoVvwULziEaJd+d6e94QoEJjc3Queb/15zT0yP6X/alokIVhi3AhkHUwkFWCJlg/d9UsdiXZYe3JfzAHxwslS204ohwSQ6NhSvKIUN1ybb19ChR+QqA1JEyv33DKGAr5BPpN72A==',\n", " 'tsa_server': 'http://zeitstempel.dfn.de',\n", - " 'timestamp': '2024-10-15T20:02:10+00:00'},\n", + " 'timestamp': '2024-10-16T10:14:57+00:00'},\n", " 'section_defs': [{'definition_qualified_name': 'nomad.datamodel.data.ArchiveSection',\n", " 'definition_id': '7047cbff9980abff17cce4b1b6b0d1c783505b7f',\n", " 'used_directly': True},\n", @@ -841,7 +843,10 @@ ], "source": [ "test = download_entry_by_id(\n", - " entry.entry_id, url='test', zip_file_name='./raw_entry_data.zip', with_authentication=True\n", + " entry.entry_id,\n", + " url='test',\n", + " zip_file_name='./raw_entry_data.zip',\n", + " with_authentication=True,\n", ")\n", "test" ] @@ -865,17 +870,17 @@ { "data": { "text/plain": [ - "{'upload_id': 'RdA_3ZsOTMqbtAhYLivVsw',\n", + "{'upload_id': 'e2b5o4KSR5yoS1EV9E9jXQ',\n", " 'data': {'process_running': True,\n", " 'current_process': 'publish_upload',\n", " 'process_status': 'PENDING',\n", " 'last_status_message': 'Pending: publish_upload',\n", " 'errors': [],\n", " 'warnings': [],\n", - " 'complete_time': '2024-10-15T20:03:51.605000',\n", - " 'upload_id': 'RdA_3ZsOTMqbtAhYLivVsw',\n", + " 'complete_time': '2024-10-16T10:14:59.363000',\n", + " 'upload_id': 'e2b5o4KSR5yoS1EV9E9jXQ',\n", " 'upload_name': 'Test Upload',\n", - " 'upload_create_time': '2024-10-15T20:02:10.378000',\n", + " 'upload_create_time': '2024-10-16T10:14:57.722000',\n", " 'main_author': '8f052e1f-1906-41fd-b2eb-690c03407788',\n", " 'coauthors': [],\n", " 'coauthor_groups': [],\n", @@ -891,7 +896,7 @@ " 'embargo_length': 0,\n", " 'license': 'CC BY 4.0',\n", " 'entries': 1,\n", - " 'upload_files_server_path': '/nomad/test/fs/staging/R/RdA_3ZsOTMqbtAhYLivVsw'}}" + " 'upload_files_server_path': '/nomad/test/fs/staging/e/e2b5o4KSR5yoS1EV9E9jXQ'}}" ] }, "execution_count": 16, @@ -917,16 +922,16 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 17, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "'NCKd75f9R9S8rnkd-GBZlg'" + "'EfhadCxpRaGpw50rWzK22w'" ] }, - "execution_count": 18, + "execution_count": 17, "metadata": {}, "output_type": "execute_result" } @@ -945,23 +950,23 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 18, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "{'upload_id': 'RdA_3ZsOTMqbtAhYLivVsw',\n", + "{'upload_id': 'e2b5o4KSR5yoS1EV9E9jXQ',\n", " 'data': {'process_running': True,\n", " 'current_process': 'edit_upload_metadata',\n", " 'process_status': 'PENDING',\n", " 'last_status_message': 'Pending: edit_upload_metadata',\n", " 'errors': [],\n", " 'warnings': [],\n", - " 'complete_time': '2024-10-15T20:09:28.769000',\n", - " 'upload_id': 'RdA_3ZsOTMqbtAhYLivVsw',\n", + " 'complete_time': '2024-10-16T10:15:04.112000',\n", + " 'upload_id': 'e2b5o4KSR5yoS1EV9E9jXQ',\n", " 'upload_name': 'Test Upload',\n", - " 'upload_create_time': '2024-10-15T20:02:10.378000',\n", + " 'upload_create_time': '2024-10-16T10:14:57.722000',\n", " 'main_author': '8f052e1f-1906-41fd-b2eb-690c03407788',\n", " 'coauthors': [],\n", " 'coauthor_groups': [],\n", @@ -973,26 +978,26 @@ " 'viewer_groups': [],\n", " 'published': True,\n", " 'published_to': [],\n", - " 'publish_time': '2024-10-15T20:09:28.757000',\n", + " 'publish_time': '2024-10-16T10:15:04.099000',\n", " 'with_embargo': False,\n", " 'embargo_length': 0,\n", " 'license': 'CC BY 4.0',\n", " 'entries': 1}}" ] }, - "execution_count": 19, + "execution_count": 18, "metadata": {}, "output_type": "execute_result" } ], "source": [ "metadata_new = {'dataset_id': dataset_id}\n", - "edit_upload_metadata(upload_id, url='test', **metadata_new)" + "edit_upload_metadata(upload_id, url='test', upload_metadata=metadata_new)" ] }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 19, "metadata": {}, "outputs": [ { @@ -1035,16 +1040,16 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 20, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "NomadDataset(dataset_id='NCKd75f9R9S8rnkd-GBZlg', dataset_create_time=datetime.datetime(2024, 10, 15, 20, 10, 17, 568000), dataset_name='test dataset', dataset_type='owned', dataset_modified_time=datetime.datetime(2024, 10, 15, 20, 10, 17, 568000), user=NomadUser(name='Joseph Rudzinski'), doi=None, pid=None, m_annotations=None)" + "NomadDataset(dataset_id='EfhadCxpRaGpw50rWzK22w', dataset_create_time=datetime.datetime(2024, 10, 16, 10, 15, 5, 240000), dataset_name='test dataset', dataset_type='owned', dataset_modified_time=datetime.datetime(2024, 10, 16, 10, 15, 5, 240000), user=NomadUser(name='Joseph Rudzinski'), doi=None, pid=None, m_annotations=None)" ] }, - "execution_count": 21, + "execution_count": 20, "metadata": {}, "output_type": "execute_result" } @@ -1063,18 +1068,18 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 21, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "[NomadDataset(dataset_id='NCKd75f9R9S8rnkd-GBZlg',\n", - " dataset_create_time=datetime.datetime(2024, 10, 15, 20, 10, 17, 568000),\n", + "[NomadDataset(dataset_id='EfhadCxpRaGpw50rWzK22w',\n", + " dataset_create_time=datetime.datetime(2024, 10, 16, 10, 15, 5, 240000),\n", " dataset_name='test dataset',\n", " dataset_type='owned',\n", - " dataset_modified_time=datetime.datetime(2024, 10, 15, 20, 10, 17, 568000),\n", + " dataset_modified_time=datetime.datetime(2024, 10, 16, 10, 15, 5, 240000),\n", " user=NomadUser(name='Joseph Rudzinski'),\n", " doi=None,\n", " pid=None,\n", @@ -1084,7 +1089,7 @@ ], "source": [ "my_datasets = retrieve_datasets(\n", - " user_id=nomad_user_me.user_id, url='test', max_datasets=20\n", + " dataset_params={'user_id': nomad_user_me.user_id, 'max_datasets': 20}, url='test'\n", ")\n", "pprint(my_datasets)" ] @@ -1098,19 +1103,19 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 22, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "'entry_id=Htbl78lHDSNAKbvPjEgEN_6sOcxF, upload_id=RdA_3ZsOTMqbtAhYLivVsw'\n" + "'entry_id=u4qPSfILguvJ9fabpklMTxbUJ7x2, upload_id=e2b5o4KSR5yoS1EV9E9jXQ'\n" ] } ], "source": [ - "dataset_entries = query_entries(dataset_id=dataset_id, url='test')\n", + "dataset_entries = query_entries(query_params={'dataset_id': dataset_id}, url='test')\n", "for entry in dataset_entries:\n", " pprint(f'entry_id={entry.entry_id}, upload_id={entry.upload_id}')" ] @@ -1135,14 +1140,14 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 23, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "'Upload with upload_id=zpq-JTzWQJ63jtSOlbueKA was deleted successfully.'\n" + "'Upload with upload_id=Mfp1OnhMSY65eZCfNvS8DA was deleted successfully.'\n" ] } ], @@ -1193,14 +1198,14 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 24, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "'Dataset with dataset_id=eT2WPkfCQgmwNadsurYz0A was deleted successfully.'\n" + "'Dataset with dataset_id=Cke5DQkdQ0qbLOky2zGfLw was deleted successfully.'\n" ] } ], @@ -1239,7 +1244,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 25, "metadata": {}, "outputs": [ { @@ -1249,10 +1254,15 @@ " NomadUpload(upload_id='DN61X4r7SCyzm5q1kxcEcw', upload_create_time=datetime.datetime(2024, 10, 14, 10, 55, 12, 410000), main_author=NomadUser(name='Joseph Rudzinski'), process_running=False, current_process='publish_upload', process_status='SUCCESS', last_status_message='Process publish_upload completed successfully', errors=[], warnings=[], coauthors=[], coauthor_groups=[], reviewers=[], reviewer_groups=[], writers=[NomadUser(name='Joseph Rudzinski')], writer_groups=[], viewers=[NomadUser(name='Joseph Rudzinski')], viewer_groups=[], published=True, published_to=[], with_embargo=False, embargo_length=0.0, license='CC BY 4.0', entries=1, n_entries=None, upload_files_server_path=None, publish_time=datetime.datetime(2024, 10, 14, 10, 55, 23, 52000), references=None, datasets=None, external_db=None, upload_name='Test Upload', comment=None, url='https://nomad-lab.eu/prod/v1/test/api/v1', complete_time=datetime.datetime(2024, 10, 14, 10, 55, 23, 65000)),\n", " NomadUpload(upload_id='z4QvhZ7qSCmgIFv_qJqlyQ', upload_create_time=datetime.datetime(2024, 10, 14, 20, 20, 38, 757000), main_author=NomadUser(name='Joseph Rudzinski'), process_running=False, current_process='edit_upload_metadata', process_status='SUCCESS', last_status_message='Process edit_upload_metadata completed successfully', errors=[], warnings=[], coauthors=['7c85bdf1-8b53-40a8-81a4-04f26ff56f29'], coauthor_groups=[], reviewers=[], reviewer_groups=[], writers=[NomadUser(name='Joseph Rudzinski'), NomadUser(name='Joseph Rudzinski')], writer_groups=[], viewers=[NomadUser(name='Joseph Rudzinski'), NomadUser(name='Joseph Rudzinski')], viewer_groups=[], published=True, published_to=[], with_embargo=False, embargo_length=0.0, license='CC BY 4.0', entries=1, n_entries=None, upload_files_server_path=None, publish_time=datetime.datetime(2024, 10, 15, 6, 18, 27, 700000), references=None, datasets=None, external_db=None, upload_name='Test Upload', comment=None, url='https://nomad-lab.eu/prod/v1/test/api/v1', complete_time=datetime.datetime(2024, 10, 15, 6, 22, 33, 45000)),\n", " NomadUpload(upload_id='GJdVAOCxRVe-Cwo3qMz9Kg', upload_create_time=datetime.datetime(2024, 10, 15, 10, 48, 44, 337000), main_author=NomadUser(name='Joseph Rudzinski'), process_running=False, current_process='edit_upload_metadata', process_status='SUCCESS', last_status_message='Process edit_upload_metadata completed successfully', errors=[], warnings=[], coauthors=[], coauthor_groups=[], reviewers=[], reviewer_groups=[], writers=[NomadUser(name='Joseph Rudzinski')], writer_groups=[], viewers=[NomadUser(name='Joseph Rudzinski')], viewer_groups=[], published=True, published_to=[], with_embargo=False, embargo_length=0.0, license='CC BY 4.0', entries=1, n_entries=None, upload_files_server_path=None, publish_time=datetime.datetime(2024, 10, 15, 10, 49, 24, 4000), references=None, datasets=None, external_db=None, upload_name='Test Upload', comment=None, url='https://nomad-lab.eu/prod/v1/test/api/v1', complete_time=datetime.datetime(2024, 10, 15, 10, 49, 30, 962000)),\n", - " NomadUpload(upload_id='RdA_3ZsOTMqbtAhYLivVsw', upload_create_time=datetime.datetime(2024, 10, 15, 20, 2, 10, 378000), main_author=NomadUser(name='Joseph Rudzinski'), process_running=False, current_process='edit_upload_metadata', process_status='SUCCESS', last_status_message='Process edit_upload_metadata completed successfully', errors=[], warnings=[], coauthors=[], coauthor_groups=[], reviewers=[], reviewer_groups=[], writers=[NomadUser(name='Joseph Rudzinski')], writer_groups=[], viewers=[NomadUser(name='Joseph Rudzinski')], viewer_groups=[], published=True, published_to=[], with_embargo=False, embargo_length=0.0, license='CC BY 4.0', entries=1, n_entries=None, upload_files_server_path=None, publish_time=datetime.datetime(2024, 10, 15, 20, 9, 28, 757000), references=None, datasets=None, external_db=None, upload_name='Test Upload', comment=None, url='https://nomad-lab.eu/prod/v1/test/api/v1', complete_time=datetime.datetime(2024, 10, 15, 20, 10, 33, 141000))]" + " NomadUpload(upload_id='RdA_3ZsOTMqbtAhYLivVsw', upload_create_time=datetime.datetime(2024, 10, 15, 20, 2, 10, 378000), main_author=NomadUser(name='Joseph Rudzinski'), process_running=False, current_process='edit_upload_metadata', process_status='SUCCESS', last_status_message='Process edit_upload_metadata completed successfully', errors=[], warnings=[], coauthors=[], coauthor_groups=[], reviewers=[], reviewer_groups=[], writers=[NomadUser(name='Joseph Rudzinski')], writer_groups=[], viewers=[NomadUser(name='Joseph Rudzinski')], viewer_groups=[], published=True, published_to=[], with_embargo=False, embargo_length=0.0, license='CC BY 4.0', entries=1, n_entries=None, upload_files_server_path=None, publish_time=datetime.datetime(2024, 10, 15, 20, 9, 28, 757000), references=None, datasets=None, external_db=None, upload_name='Test Upload', comment=None, url='https://nomad-lab.eu/prod/v1/test/api/v1', complete_time=datetime.datetime(2024, 10, 15, 20, 10, 33, 141000)),\n", + " NomadUpload(upload_id='8vViZoL3TYG9fMFibPkjlw', upload_create_time=datetime.datetime(2024, 10, 16, 9, 25, 53, 929000), main_author=NomadUser(name='Joseph Rudzinski'), process_running=False, current_process='publish_upload', process_status='SUCCESS', last_status_message='Process publish_upload completed successfully', errors=[], warnings=[], coauthors=[], coauthor_groups=[], reviewers=[], reviewer_groups=[], writers=[NomadUser(name='Joseph Rudzinski')], writer_groups=[], viewers=[NomadUser(name='Joseph Rudzinski')], viewer_groups=[], published=True, published_to=[], with_embargo=False, embargo_length=0.0, license='CC BY 4.0', entries=1, n_entries=None, upload_files_server_path=None, publish_time=datetime.datetime(2024, 10, 16, 9, 43, 18, 243000), references=None, datasets=None, external_db=None, upload_name='Test Upload', comment=None, url='https://nomad-lab.eu/prod/v1/test/api/v1', complete_time=datetime.datetime(2024, 10, 16, 9, 43, 18, 255000)),\n", + " NomadUpload(upload_id='cP4q5rRsQM-D60Tp3olPdQ', upload_create_time=datetime.datetime(2024, 10, 16, 9, 47, 31, 721000), main_author=NomadUser(name='Joseph Rudzinski'), process_running=False, current_process='publish_upload', process_status='SUCCESS', last_status_message='Process publish_upload completed successfully', errors=[], warnings=[], coauthors=[], coauthor_groups=[], reviewers=[], reviewer_groups=[], writers=[NomadUser(name='Joseph Rudzinski')], writer_groups=[], viewers=[NomadUser(name='Joseph Rudzinski')], viewer_groups=[], published=True, published_to=[], with_embargo=False, embargo_length=0.0, license='CC BY 4.0', entries=1, n_entries=None, upload_files_server_path=None, publish_time=datetime.datetime(2024, 10, 16, 9, 47, 46, 247000), references=None, datasets=None, external_db=None, upload_name='Test Upload', comment=None, url='https://nomad-lab.eu/prod/v1/test/api/v1', complete_time=datetime.datetime(2024, 10, 16, 9, 47, 46, 257000)),\n", + " NomadUpload(upload_id='5ADf3M4uSByqsYpkEB6UEg', upload_create_time=datetime.datetime(2024, 10, 16, 9, 52, 1, 469000), main_author=NomadUser(name='Joseph Rudzinski'), process_running=False, current_process='process_upload', process_status='SUCCESS', last_status_message='Process process_upload completed successfully', errors=[], warnings=[], coauthors=[], coauthor_groups=[], reviewers=[], reviewer_groups=[], writers=[NomadUser(name='Joseph Rudzinski')], writer_groups=[], viewers=[NomadUser(name='Joseph Rudzinski')], viewer_groups=[], published=False, published_to=[], with_embargo=False, embargo_length=0.0, license='CC BY 4.0', entries=1, n_entries=None, upload_files_server_path='/nomad/test/fs/staging/5/5ADf3M4uSByqsYpkEB6UEg', publish_time=None, references=None, datasets=None, external_db=None, upload_name=None, comment=None, url='https://nomad-lab.eu/prod/v1/test/api/v1', complete_time=datetime.datetime(2024, 10, 16, 9, 52, 2, 94000)),\n", + " NomadUpload(upload_id='_mZn0RZ8QtmBkcAlPU5bSw', upload_create_time=datetime.datetime(2024, 10, 16, 9, 52, 47, 649000), main_author=NomadUser(name='Joseph Rudzinski'), process_running=False, current_process='edit_upload_metadata', process_status='SUCCESS', last_status_message='Process edit_upload_metadata completed successfully', errors=[], warnings=[], coauthors=[], coauthor_groups=[], reviewers=[], reviewer_groups=[], writers=[NomadUser(name='Joseph Rudzinski')], writer_groups=[], viewers=[NomadUser(name='Joseph Rudzinski')], viewer_groups=[], published=True, published_to=[], with_embargo=False, embargo_length=0.0, license='CC BY 4.0', entries=1, n_entries=None, upload_files_server_path=None, publish_time=datetime.datetime(2024, 10, 16, 9, 53, 0, 835000), references=None, datasets=None, external_db=None, upload_name='Test Upload', comment=None, url='https://nomad-lab.eu/prod/v1/test/api/v1', complete_time=datetime.datetime(2024, 10, 16, 9, 53, 4, 791000)),\n", + " NomadUpload(upload_id='Cntk6OsQTvaZp7r6Jom-3g', upload_create_time=datetime.datetime(2024, 10, 16, 9, 54, 47, 426000), main_author=NomadUser(name='Joseph Rudzinski'), process_running=False, current_process='edit_upload_metadata', process_status='SUCCESS', last_status_message='Process edit_upload_metadata completed successfully', errors=[], warnings=[], coauthors=[], coauthor_groups=[], reviewers=[], reviewer_groups=[], writers=[NomadUser(name='Joseph Rudzinski')], writer_groups=[], viewers=[NomadUser(name='Joseph Rudzinski')], viewer_groups=[], published=True, published_to=[], with_embargo=False, embargo_length=0.0, license='CC BY 4.0', entries=1, n_entries=None, upload_files_server_path=None, publish_time=datetime.datetime(2024, 10, 16, 9, 54, 51, 988000), references=None, datasets=None, external_db=None, upload_name='Test Upload', comment=None, url='https://nomad-lab.eu/prod/v1/test/api/v1', complete_time=datetime.datetime(2024, 10, 16, 9, 54, 53, 838000))]" ] }, - "execution_count": 27, + "execution_count": 25, "metadata": {}, "output_type": "execute_result" } @@ -1263,7 +1273,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 26, "metadata": {}, "outputs": [ { @@ -1273,10 +1283,15 @@ " NomadEntry(entry_id='7A6lJb-14xR9lxXO8kjuYt5-vxg2', upload_id='DN61X4r7SCyzm5q1kxcEcw', references=[], origin='Joseph Rudzinski', n_quantities=34, nomad_version='1.3.7.dev55+ge83de27b3', upload_create_time=datetime.datetime(2024, 10, 14, 10, 55, 12, 410000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), nomad_commit='', processing_errors=[], entry_name='test.archive.json', last_processing_time=datetime.datetime(2024, 10, 14, 10, 55, 12, 808000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), parser_name='parsers/archive', calc_id='7A6lJb-14xR9lxXO8kjuYt5-vxg2', published=True, writers=[NomadUser(name='Joseph Rudzinski')], processed=True, mainfile='test.archive.json', main_author=NomadUser(name='Joseph Rudzinski'), entry_create_time=datetime.datetime(2024, 10, 14, 10, 55, 12, 563000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), with_embargo=False, entry_type=None, license='CC BY 4.0', domain=None, comment='This is a test upload...', upload_name='Test Upload', text_search_contents=[], publish_time=None, entry_references=None, url='https://nomad-lab.eu/prod/v1/test/api/v1'),\n", " NomadEntry(entry_id='jWSpYURP5GgPtgF9LXZJpNlDv-GL', upload_id='z4QvhZ7qSCmgIFv_qJqlyQ', references=[], origin='Joseph Rudzinski', n_quantities=0, nomad_version='1.3.7.dev55+ge83de27b3', upload_create_time=datetime.datetime(2024, 10, 14, 20, 20, 38, 757000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), nomad_commit='', processing_errors=[], entry_name='test.archive.json', last_processing_time=datetime.datetime(2024, 10, 14, 20, 20, 39, 272000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), parser_name='parsers/archive', calc_id='jWSpYURP5GgPtgF9LXZJpNlDv-GL', published=True, writers=[NomadUser(name='Joseph Rudzinski'), NomadUser(name='Joseph Rudzinski')], processed=True, mainfile='test.archive.json', main_author=NomadUser(name='Joseph Rudzinski'), entry_create_time=datetime.datetime(2024, 10, 14, 20, 20, 38, 982000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), with_embargo=False, entry_type=None, license='CC BY 4.0', domain=None, comment='This is a test upload...edited', upload_name='Test Upload', text_search_contents=[], publish_time=datetime.datetime(2024, 10, 15, 6, 18, 27, 700000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), entry_references=None, url='https://nomad-lab.eu/prod/v1/test/api/v1'),\n", " NomadEntry(entry_id='MVBIMEZOuIzH7-QFU2TtMIM6LLPp', upload_id='GJdVAOCxRVe-Cwo3qMz9Kg', references=[], origin='Joseph Rudzinski', n_quantities=0, nomad_version='1.3.7.dev55+ge83de27b3', upload_create_time=datetime.datetime(2024, 10, 15, 10, 48, 44, 337000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), nomad_commit='', processing_errors=[], entry_name='test.archive.json', last_processing_time=datetime.datetime(2024, 10, 15, 10, 48, 45, 206000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), parser_name='parsers/archive', calc_id='MVBIMEZOuIzH7-QFU2TtMIM6LLPp', published=True, writers=[NomadUser(name='Joseph Rudzinski')], processed=True, mainfile='test.archive.json', main_author=NomadUser(name='Joseph Rudzinski'), entry_create_time=datetime.datetime(2024, 10, 15, 10, 48, 44, 741000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), with_embargo=False, entry_type=None, license='CC BY 4.0', domain=None, comment='This is a test upload...', upload_name='Test Upload', text_search_contents=[], publish_time=datetime.datetime(2024, 10, 15, 10, 49, 24, 4000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), entry_references=None, url='https://nomad-lab.eu/prod/v1/test/api/v1'),\n", - " NomadEntry(entry_id='Htbl78lHDSNAKbvPjEgEN_6sOcxF', upload_id='RdA_3ZsOTMqbtAhYLivVsw', references=[], origin='Joseph Rudzinski', n_quantities=0, nomad_version='1.3.7.dev55+ge83de27b3', upload_create_time=datetime.datetime(2024, 10, 15, 20, 2, 10, 378000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), nomad_commit='', processing_errors=[], entry_name='test.archive.json', last_processing_time=datetime.datetime(2024, 10, 15, 20, 2, 10, 752000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), parser_name='parsers/archive', calc_id='Htbl78lHDSNAKbvPjEgEN_6sOcxF', published=True, writers=[NomadUser(name='Joseph Rudzinski')], processed=True, mainfile='test.archive.json', main_author=NomadUser(name='Joseph Rudzinski'), entry_create_time=datetime.datetime(2024, 10, 15, 20, 2, 10, 543000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), with_embargo=False, entry_type=None, license='CC BY 4.0', domain=None, comment='This is a test upload...', upload_name='Test Upload', text_search_contents=[], publish_time=datetime.datetime(2024, 10, 15, 20, 9, 28, 757000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), entry_references=None, url='https://nomad-lab.eu/prod/v1/test/api/v1')]" + " NomadEntry(entry_id='Htbl78lHDSNAKbvPjEgEN_6sOcxF', upload_id='RdA_3ZsOTMqbtAhYLivVsw', references=[], origin='Joseph Rudzinski', n_quantities=0, nomad_version='1.3.7.dev55+ge83de27b3', upload_create_time=datetime.datetime(2024, 10, 15, 20, 2, 10, 378000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), nomad_commit='', processing_errors=[], entry_name='test.archive.json', last_processing_time=datetime.datetime(2024, 10, 15, 20, 2, 10, 752000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), parser_name='parsers/archive', calc_id='Htbl78lHDSNAKbvPjEgEN_6sOcxF', published=True, writers=[NomadUser(name='Joseph Rudzinski')], processed=True, mainfile='test.archive.json', main_author=NomadUser(name='Joseph Rudzinski'), entry_create_time=datetime.datetime(2024, 10, 15, 20, 2, 10, 543000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), with_embargo=False, entry_type=None, license='CC BY 4.0', domain=None, comment='This is a test upload...', upload_name='Test Upload', text_search_contents=[], publish_time=datetime.datetime(2024, 10, 15, 20, 9, 28, 757000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), entry_references=None, url='https://nomad-lab.eu/prod/v1/test/api/v1'),\n", + " NomadEntry(entry_id='zyaF373NIH-igHS3TZN5FW4SaO4d', upload_id='8vViZoL3TYG9fMFibPkjlw', references=[], origin='Joseph Rudzinski', n_quantities=34, nomad_version='1.3.7.dev55+ge83de27b3', upload_create_time=datetime.datetime(2024, 10, 16, 9, 25, 53, 929000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), nomad_commit='', processing_errors=[], entry_name='test.archive.json', last_processing_time=datetime.datetime(2024, 10, 16, 9, 25, 54, 274000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), parser_name='parsers/archive', calc_id='zyaF373NIH-igHS3TZN5FW4SaO4d', published=True, writers=[NomadUser(name='Joseph Rudzinski')], processed=True, mainfile='test.archive.json', main_author=NomadUser(name='Joseph Rudzinski'), entry_create_time=datetime.datetime(2024, 10, 16, 9, 25, 54, 111000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), with_embargo=False, entry_type=None, license='CC BY 4.0', domain=None, comment='This is a test upload...', upload_name='Test Upload', text_search_contents=[], publish_time=None, entry_references=None, url='https://nomad-lab.eu/prod/v1/test/api/v1'),\n", + " NomadEntry(entry_id='Xitkh3ZVhRu11LUIk5n0cA2Wtmmy', upload_id='cP4q5rRsQM-D60Tp3olPdQ', references=[], origin='Joseph Rudzinski', n_quantities=34, nomad_version='1.3.7.dev55+ge83de27b3', upload_create_time=datetime.datetime(2024, 10, 16, 9, 47, 31, 721000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), nomad_commit='', processing_errors=[], entry_name='test.archive.json', last_processing_time=datetime.datetime(2024, 10, 16, 9, 47, 32, 28000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), parser_name='parsers/archive', calc_id='Xitkh3ZVhRu11LUIk5n0cA2Wtmmy', published=True, writers=[NomadUser(name='Joseph Rudzinski')], processed=True, mainfile='test.archive.json', main_author=NomadUser(name='Joseph Rudzinski'), entry_create_time=datetime.datetime(2024, 10, 16, 9, 47, 31, 857000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), with_embargo=False, entry_type=None, license='CC BY 4.0', domain=None, comment='This is a test upload...', upload_name='Test Upload', text_search_contents=[], publish_time=None, entry_references=None, url='https://nomad-lab.eu/prod/v1/test/api/v1'),\n", + " NomadEntry(entry_id='EaFsd7Ku9IEXOcwTcMqlWd92fZx1', upload_id='5ADf3M4uSByqsYpkEB6UEg', references=[], origin='Joseph Rudzinski', n_quantities=34, nomad_version='1.3.7.dev55+ge83de27b3', upload_create_time=datetime.datetime(2024, 10, 16, 9, 52, 1, 469000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), nomad_commit='', processing_errors=[], entry_name='test.archive.json', last_processing_time=datetime.datetime(2024, 10, 16, 9, 52, 1, 752000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), parser_name='parsers/archive', calc_id='EaFsd7Ku9IEXOcwTcMqlWd92fZx1', published=False, writers=[NomadUser(name='Joseph Rudzinski')], processed=True, mainfile='test.archive.json', main_author=NomadUser(name='Joseph Rudzinski'), entry_create_time=datetime.datetime(2024, 10, 16, 9, 52, 1, 595000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), with_embargo=False, entry_type=None, license='CC BY 4.0', domain=None, comment=None, upload_name=None, text_search_contents=[], publish_time=None, entry_references=None, url='https://nomad-lab.eu/prod/v1/test/api/v1'),\n", + " NomadEntry(entry_id='h3e0Z5FHiUetLmW8kbPW4uwrT0gH', upload_id='_mZn0RZ8QtmBkcAlPU5bSw', references=[], origin='Joseph Rudzinski', n_quantities=0, nomad_version='1.3.7.dev55+ge83de27b3', upload_create_time=datetime.datetime(2024, 10, 16, 9, 52, 47, 649000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), nomad_commit='', processing_errors=[], entry_name='test.archive.json', last_processing_time=datetime.datetime(2024, 10, 16, 9, 52, 47, 948000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), parser_name='parsers/archive', calc_id='h3e0Z5FHiUetLmW8kbPW4uwrT0gH', published=True, writers=[NomadUser(name='Joseph Rudzinski')], processed=True, mainfile='test.archive.json', main_author=NomadUser(name='Joseph Rudzinski'), entry_create_time=datetime.datetime(2024, 10, 16, 9, 52, 47, 780000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), with_embargo=False, entry_type=None, license='CC BY 4.0', domain=None, comment='This is a test upload...', upload_name='Test Upload', text_search_contents=[], publish_time=datetime.datetime(2024, 10, 16, 9, 53, 0, 835000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), entry_references=None, url='https://nomad-lab.eu/prod/v1/test/api/v1'),\n", + " NomadEntry(entry_id='lJiZnALI0ad8UKh5nt2FG1rhaZiC', upload_id='Cntk6OsQTvaZp7r6Jom-3g', references=[], origin='Joseph Rudzinski', n_quantities=0, nomad_version='1.3.7.dev55+ge83de27b3', upload_create_time=datetime.datetime(2024, 10, 16, 9, 54, 47, 426000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), nomad_commit='', processing_errors=[], entry_name='test.archive.json', last_processing_time=datetime.datetime(2024, 10, 16, 9, 54, 47, 719000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), parser_name='parsers/archive', calc_id='lJiZnALI0ad8UKh5nt2FG1rhaZiC', published=True, writers=[NomadUser(name='Joseph Rudzinski')], processed=True, mainfile='test.archive.json', main_author=NomadUser(name='Joseph Rudzinski'), entry_create_time=datetime.datetime(2024, 10, 16, 9, 54, 47, 553000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), with_embargo=False, entry_type=None, license='CC BY 4.0', domain=None, comment='This is a test upload...', upload_name='Test Upload', text_search_contents=[], publish_time=datetime.datetime(2024, 10, 16, 9, 54, 51, 988000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), entry_references=None, url='https://nomad-lab.eu/prod/v1/test/api/v1')]" ] }, - "execution_count": 28, + "execution_count": 26, "metadata": {}, "output_type": "execute_result" } diff --git a/tests/utils/how_to_use_api_functions.md b/tests/utils/how_to_use_api_functions.md index 821ea06..bb46b96 100644 --- a/tests/utils/how_to_use_api_functions.md +++ b/tests/utils/how_to_use_api_functions.md @@ -223,7 +223,9 @@ You can make an upload using the `upload_files_to_nomad()` function with input ` ```python -test_upload_fnm = './test.zip' # a dummy upload file containing a single empty json file +test_upload_fnm = ( + './test.zip' # a dummy upload file containing a single empty json file +) ``` @@ -235,7 +237,7 @@ upload_id - 'RdA_3ZsOTMqbtAhYLivVsw' + 'e2b5o4KSR5yoS1EV9E9jXQ' @@ -250,13 +252,13 @@ nomad_upload = get_upload_by_id(upload_id, url='test') pprint(nomad_upload) ``` - NomadUpload(upload_id='RdA_3ZsOTMqbtAhYLivVsw', - upload_create_time=datetime.datetime(2024, 10, 15, 20, 2, 10, 378000), + NomadUpload(upload_id='e2b5o4KSR5yoS1EV9E9jXQ', + upload_create_time=datetime.datetime(2024, 10, 16, 10, 14, 57, 722000), main_author=NomadUser(name='Joseph Rudzinski'), - process_running=False, + process_running=True, current_process='process_upload', - process_status='SUCCESS', - last_status_message='Process process_upload completed successfully', + process_status='RUNNING', + last_status_message='Cleanup', errors=[], warnings=[], coauthors=[], @@ -274,7 +276,7 @@ pprint(nomad_upload) license='CC BY 4.0', entries=1, n_entries=None, - upload_files_server_path='/nomad/test/fs/staging/R/RdA_3ZsOTMqbtAhYLivVsw', + upload_files_server_path='/nomad/test/fs/staging/e/e2b5o4KSR5yoS1EV9E9jXQ', publish_time=None, references=None, datasets=None, @@ -282,7 +284,7 @@ pprint(nomad_upload) upload_name=None, comment=None, url='https://nomad-lab.eu/prod/v1/test/api/v1', - complete_time=datetime.datetime(2024, 10, 15, 20, 2, 11, 320000)) + complete_time=None) One common usage of this function is to ensure that an upload has been processed successfully before making a subsequent action on it, e.g., editing the metadata or publishing. For this purpose, one could require the `process_running==False` or `process_status='SUCCESS'`, e.g.: @@ -330,22 +332,22 @@ For example: ```python metadata_new = {'upload_name': 'Test Upload', 'comment': 'This is a test upload...'} -edit_upload_metadata(upload_id, url='test', **metadata_new) +edit_upload_metadata(upload_id, url='test', upload_metadata=metadata_new) ``` - {'upload_id': 'RdA_3ZsOTMqbtAhYLivVsw', + {'upload_id': 'e2b5o4KSR5yoS1EV9E9jXQ', 'data': {'process_running': True, 'current_process': 'edit_upload_metadata', 'process_status': 'PENDING', 'last_status_message': 'Pending: edit_upload_metadata', 'errors': [], 'warnings': [], - 'complete_time': '2024-10-15T20:02:11.320000', - 'upload_id': 'RdA_3ZsOTMqbtAhYLivVsw', - 'upload_create_time': '2024-10-15T20:02:10.378000', + 'complete_time': '2024-10-16T10:14:58.322000', + 'upload_id': 'e2b5o4KSR5yoS1EV9E9jXQ', + 'upload_create_time': '2024-10-16T10:14:57.722000', 'main_author': '8f052e1f-1906-41fd-b2eb-690c03407788', 'coauthors': [], 'coauthor_groups': [], @@ -361,7 +363,7 @@ edit_upload_metadata(upload_id, url='test', **metadata_new) 'embargo_length': 0, 'license': 'CC BY 4.0', 'entries': 1, - 'upload_files_server_path': '/nomad/test/fs/staging/R/RdA_3ZsOTMqbtAhYLivVsw'}} + 'upload_files_server_path': '/nomad/test/fs/staging/e/e2b5o4KSR5yoS1EV9E9jXQ'}} @@ -439,8 +441,8 @@ for entry in entries: pprint(f'entry_id={entry.entry_id}') ``` - 'Entries within upload_id=RdA_3ZsOTMqbtAhYLivVsw:' - 'entry_id=Htbl78lHDSNAKbvPjEgEN_6sOcxF' + 'Entries within upload_id=e2b5o4KSR5yoS1EV9E9jXQ:' + 'entry_id=u4qPSfILguvJ9fabpklMTxbUJ7x2' To query an entry directly using the `entry_id`, use `get_entry_by_id()`: @@ -454,7 +456,7 @@ entry - NomadEntry(entry_id='Htbl78lHDSNAKbvPjEgEN_6sOcxF', upload_id='RdA_3ZsOTMqbtAhYLivVsw', references=[], origin='Joseph Rudzinski', n_quantities=0, nomad_version='1.3.7.dev55+ge83de27b3', upload_create_time=datetime.datetime(2024, 10, 15, 20, 2, 10, 378000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), nomad_commit='', processing_errors=[], entry_name='test.archive.json', last_processing_time=datetime.datetime(2024, 10, 15, 20, 2, 10, 752000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), parser_name='parsers/archive', calc_id='Htbl78lHDSNAKbvPjEgEN_6sOcxF', published=False, writers=[NomadUser(name='Joseph Rudzinski')], processed=True, mainfile='test.archive.json', main_author=NomadUser(name='Joseph Rudzinski'), entry_create_time=datetime.datetime(2024, 10, 15, 20, 2, 10, 543000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), with_embargo=False, entry_type=None, license='CC BY 4.0', domain=None, comment='This is a test upload...', upload_name='Test Upload', text_search_contents=[], publish_time=None, entry_references=None, url='https://nomad-lab.eu/prod/v1/test/api/v1') + NomadEntry(entry_id='u4qPSfILguvJ9fabpklMTxbUJ7x2', upload_id='e2b5o4KSR5yoS1EV9E9jXQ', references=[], origin='Joseph Rudzinski', n_quantities=0, nomad_version='1.3.7.dev55+ge83de27b3', upload_create_time=datetime.datetime(2024, 10, 16, 10, 14, 57, 722000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), nomad_commit='', processing_errors=[], entry_name='test.archive.json', last_processing_time=datetime.datetime(2024, 10, 16, 10, 14, 58, 23000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), parser_name='parsers/archive', calc_id='u4qPSfILguvJ9fabpklMTxbUJ7x2', published=False, writers=[NomadUser(name='Joseph Rudzinski')], processed=True, mainfile='test.archive.json', main_author=NomadUser(name='Joseph Rudzinski'), entry_create_time=datetime.datetime(2024, 10, 16, 10, 14, 57, 855000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), with_embargo=False, entry_type=None, license='CC BY 4.0', domain=None, comment='This is a test upload...', upload_name='Test Upload', text_search_contents=[], publish_time=None, entry_references=None, url='https://nomad-lab.eu/prod/v1/test/api/v1') @@ -463,7 +465,10 @@ You can download the full (meta)data stored in an entry using `download_entry_by ```python test = download_entry_by_id( - entry.entry_id, url='test', zip_file_name='./raw_entry_data.zip', with_authentication=True + entry.entry_id, + url='test', + zip_file_name='./raw_entry_data.zip', + with_authentication=True, ) test ``` @@ -477,9 +482,9 @@ test 'process_worker_id': 'BOiybXorRqW5XImFf0SyoA', 'parser': 'parsers/archive', 'logger': 'nomad.processing', - 'timestamp': '2024-10-15 20:02.10', + 'timestamp': '2024-10-16 10:14.57', 'level': 'DEBUG'}, - {'exec_time': '0.001298666000366211', + {'exec_time': '0.0013990402221679688', 'input_size': '3', 'event': 'parser executed', 'proc': 'Entry', @@ -488,7 +493,7 @@ test 'parser': 'parsers/archive', 'step': 'parsers/archive', 'logger': 'nomad.processing', - 'timestamp': '2024-10-15 20:02.10', + 'timestamp': '2024-10-16 10:14.58', 'level': 'INFO'}, {'normalizer': 'MetainfoNormalizer', 'step': 'MetainfoNormalizer', @@ -498,9 +503,9 @@ test 'process_worker_id': 'BOiybXorRqW5XImFf0SyoA', 'parser': 'parsers/archive', 'logger': 'nomad.processing', - 'timestamp': '2024-10-15 20:02.10', + 'timestamp': '2024-10-16 10:14.58', 'level': 'INFO'}, - {'exec_time': '0.0011508464813232422', + {'exec_time': '0.0006549358367919922', 'input_size': '3', 'event': 'normalizer executed', 'proc': 'Entry', @@ -510,7 +515,7 @@ test 'normalizer': 'MetainfoNormalizer', 'step': 'MetainfoNormalizer', 'logger': 'nomad.processing', - 'timestamp': '2024-10-15 20:02.10', + 'timestamp': '2024-10-16 10:14.58', 'level': 'INFO'}, {'normalizer': 'ResultsNormalizer', 'step': 'ResultsNormalizer', @@ -520,9 +525,9 @@ test 'process_worker_id': 'BOiybXorRqW5XImFf0SyoA', 'parser': 'parsers/archive', 'logger': 'nomad.processing', - 'timestamp': '2024-10-15 20:02.10', + 'timestamp': '2024-10-16 10:14.58', 'level': 'INFO'}, - {'exec_time': '0.0011796951293945312', + {'exec_time': '0.0005123615264892578', 'input_size': '3', 'event': 'normalizer executed', 'proc': 'Entry', @@ -532,32 +537,32 @@ test 'normalizer': 'ResultsNormalizer', 'step': 'ResultsNormalizer', 'logger': 'nomad.processing', - 'timestamp': '2024-10-15 20:02.10', + 'timestamp': '2024-10-16 10:14.58', 'level': 'INFO'}, - {'exec_time': '0.002213716506958008', + {'exec_time': '0.0015552043914794922', 'event': 'entry metadata saved', 'proc': 'Entry', 'process': 'process_entry', 'process_worker_id': 'BOiybXorRqW5XImFf0SyoA', 'parser': 'parsers/archive', 'logger': 'nomad.processing', - 'timestamp': '2024-10-15 20:02.10', + 'timestamp': '2024-10-16 10:14.58', 'level': 'INFO'}, - {'exec_time': '0.07823586463928223', + {'exec_time': '0.09751391410827637', 'event': 'entry metadata indexed', 'proc': 'Entry', 'process': 'process_entry', 'process_worker_id': 'BOiybXorRqW5XImFf0SyoA', 'parser': 'parsers/archive', 'logger': 'nomad.processing', - 'timestamp': '2024-10-15 20:02.10', + 'timestamp': '2024-10-16 10:14.58', 'level': 'INFO'}], - 'metadata': {'upload_id': 'RdA_3ZsOTMqbtAhYLivVsw', - 'upload_create_time': '2024-10-15T20:02:10.378000+00:00', - 'entry_id': 'Htbl78lHDSNAKbvPjEgEN_6sOcxF', + 'metadata': {'upload_id': 'e2b5o4KSR5yoS1EV9E9jXQ', + 'upload_create_time': '2024-10-16T10:14:57.722000+00:00', + 'entry_id': 'u4qPSfILguvJ9fabpklMTxbUJ7x2', 'entry_name': 'test.archive.json', 'entry_hash': 't6Zf68GLfrWxWRAIQu7QAY8LVmlL', - 'entry_create_time': '2024-10-15T20:02:10.543000+00:00', + 'entry_create_time': '2024-10-16T10:14:57.855000+00:00', 'parser_name': 'parsers/archive', 'mainfile': 'test.archive.json', 'text_search_contents': [], @@ -567,7 +572,7 @@ test 'embargo_length': 0, 'license': 'CC BY 4.0', 'processed': True, - 'last_processing_time': '2024-10-15T20:02:10.752287+00:00', + 'last_processing_time': '2024-10-16T10:14:58.023933+00:00', 'processing_errors': [], 'nomad_version': '1.3.7.dev55+ge83de27b3', 'nomad_commit': '', @@ -627,9 +632,9 @@ test 'nomad.datamodel.results.Properties', 'nomad.datamodel.results.Results'], 'entry_timestamp': {'token_seed': 't6Zf68GLfrWxWRAIQu7QAY8LVmlL', - 'token': 'MIIEQwYJKoZIhvcNAQcCoIIENDCCBDACAQMxDTALBglghkgBZQMEAgEwfAYLKoZIhvcNAQkQAQSgbQRrMGkCAQEGDCsGAQQBga0hgiwWATAvMAsGCWCGSAFlAwQCAQQgYnRB2tk2mTRtMamyedr2QFd3bb0lFM56N52xD8rv/OECFFIumicGBkskyLovIs3OBywh3EXbGA8yMDI0MTAxNTIwMDIxMFoxggOcMIIDmAIBATCBnjCBjTELMAkGA1UEBhMCREUxRTBDBgNVBAoMPFZlcmVpbiB6dXIgRm9lcmRlcnVuZyBlaW5lcyBEZXV0c2NoZW4gRm9yc2NodW5nc25ldHplcyBlLiBWLjEQMA4GA1UECwwHREZOLVBLSTElMCMGA1UEAwwcREZOLVZlcmVpbiBHbG9iYWwgSXNzdWluZyBDQQIMKQLVczMPeOL0nrS5MAsGCWCGSAFlAwQCAaCB0TAaBgkqhkiG9w0BCQMxDQYLKoZIhvcNAQkQAQQwHAYJKoZIhvcNAQkFMQ8XDTI0MTAxNTIwMDIxMFowKwYJKoZIhvcNAQk0MR4wHDALBglghkgBZQMEAgGhDQYJKoZIhvcNAQELBQAwLwYJKoZIhvcNAQkEMSIEIMF/vhL+ddm6reCVHQgPz5FLJYZKE3ag+HqzfGTfwTvPMDcGCyqGSIb3DQEJEAIvMSgwJjAkMCIEILYIjb3dCJjTSQeNfCMyp07MhBQMoINZ8CNXJUbPboLkMA0GCSqGSIb3DQEBCwUABIICAJSu4GSAVDGNwuA+Kr5Qhi7rrcQcZpAcA2TOotKVS9b8wDyCE+J7IwobbVDIVURsa0b9QsReNUZHc+U9TmlWGprwY1j1BVy+ccXNg2U2Uf0dMrnOzVWfPNAoMT1iv9tK3kMwq2Gal35yh/Arrp+XYMmLfKFRZpzNcjz0TFokjCFbrPxreLCHgSnPWy3VHncNWghyMg6Nxq0rUwvV7dp7cjt1R9Ky6eHdxQvyMmVuSRbTetg+B43KkrYXoqGNGVEqiaScZWJswM5jFApNquUZnOuRl4/bhABNGWpLXQRxDz2r4Bqvgz4DfQt/8EPg7YvWNWNzfw+oPXgh5dDqKW9DiKoa0U2J1+/YKnBcdJefDsyZHnOcXOAIaX/f8k9Wg90v+c/WrbSjbfYMMhJBGqKYgU6DAeh7p7DZQNeDS4qspVWaTb10zgxmnkCbtqzmlzJvY4pOnjirVHfMFyXxL1rkXG91swhjK6FnL/okQKVT2tGQHC4I15VnEmrhit/ptXJHpWABO6TKJCW4oCgUdxUBPMT3bY7RaT3Fe+XzjjRuau0dVd2bYIbUTlrGFBUjJf+9Zuj145FdqdSjezx7NaIy7zsF1wV/e9feu6vbu3kEu32hZOT7Agw/Ryqrqgx4KuJuelOlwTljeQVHUfblr6yhwNDzdxufsQyAbJAvRUS2tyMk', + 'token': 'MIIERAYJKoZIhvcNAQcCoIIENTCCBDECAQMxDTALBglghkgBZQMEAgEwfQYLKoZIhvcNAQkQAQSgbgRsMGoCAQEGDCsGAQQBga0hgiwWATAvMAsGCWCGSAFlAwQCAQQgYnRB2tk2mTRtMamyedr2QFd3bb0lFM56N52xD8rv/OECFQDfPonFJHlhxHevtRkQ/Z85hZIntRgPMjAyNDEwMTYxMDE0NTdaMYIDnDCCA5gCAQEwgZ4wgY0xCzAJBgNVBAYTAkRFMUUwQwYDVQQKDDxWZXJlaW4genVyIEZvZXJkZXJ1bmcgZWluZXMgRGV1dHNjaGVuIEZvcnNjaHVuZ3NuZXR6ZXMgZS4gVi4xEDAOBgNVBAsMB0RGTi1QS0kxJTAjBgNVBAMMHERGTi1WZXJlaW4gR2xvYmFsIElzc3VpbmcgQ0ECDCkC1XMzD3ji9J60uTALBglghkgBZQMEAgGggdEwGgYJKoZIhvcNAQkDMQ0GCyqGSIb3DQEJEAEEMBwGCSqGSIb3DQEJBTEPFw0yNDEwMTYxMDE0NTdaMCsGCSqGSIb3DQEJNDEeMBwwCwYJYIZIAWUDBAIBoQ0GCSqGSIb3DQEBCwUAMC8GCSqGSIb3DQEJBDEiBCA6s2JcwOqwu7132Qqq//qLCZ/RvK+vbG2KEmFaDWTOezA3BgsqhkiG9w0BCRACLzEoMCYwJDAiBCC2CI293QiY00kHjXwjMqdOzIQUDKCDWfAjVyVGz26C5DANBgkqhkiG9w0BAQsFAASCAgBq9WFjtr/M3uah2LM9SuT7Mg/clX91fkKHk+/a+Y4gZGzOGdeE4P3AlHKDsNmOqghgvX7EVXgMadc+Hyf72YGNaqrca3BFYivuI4NqbjX7+wC9vq0SZIn+5d3TBCCZisZmAtZHTj7+KZpD071qGLMmsaCi+qOTQExgB0G7DRYeGibq3slZXKTr2wY8frQDB6M36y69nPgovO4azvy+553w2EQQkAGfemSxz78S9A2jcV9frKxOIGXUb5R/FVYGfJfCM/C+d35iUGisi9ZWHl0PvSe2Zzwv+4UQRr4ljEGrnfDkdKrV1ZluxerBNrzxcYU+DpWUPs8NZdfQibSkmc9aMwvznrqB4GL3iW+8RrQrGSgiIm84HKlYp6TNCWldlGlV3Xfgo+6AO3hZv5v7tiM7GWLL6/fFi4IRPaIGCldL9D0Yun6U95vaiy+jRT1yzZx4pPlgM3JbSp4z/BkmTURBhHySUsr91LkB8OAiS5LKEZye7qJTuBdI9Ny7TNB050c5+mmfv/TKmbExIZoYvChXQKCUMCCLLSFmxN4eoVvwULziEaJd+d6e94QoEJjc3Queb/15zT0yP6X/alokIVhi3AhkHUwkFWCJlg/d9UsdiXZYe3JfzAHxwslS204ohwSQ6NhSvKIUN1ybb19ChR+QqA1JEyv33DKGAr5BPpN72A==', 'tsa_server': 'http://zeitstempel.dfn.de', - 'timestamp': '2024-10-15T20:02:10+00:00'}, + 'timestamp': '2024-10-16T10:14:57+00:00'}, 'section_defs': [{'definition_qualified_name': 'nomad.datamodel.data.ArchiveSection', 'definition_id': '7047cbff9980abff17cce4b1b6b0d1c783505b7f', 'used_directly': True}, @@ -668,17 +673,17 @@ published_upload - {'upload_id': 'RdA_3ZsOTMqbtAhYLivVsw', + {'upload_id': 'e2b5o4KSR5yoS1EV9E9jXQ', 'data': {'process_running': True, 'current_process': 'publish_upload', 'process_status': 'PENDING', 'last_status_message': 'Pending: publish_upload', 'errors': [], 'warnings': [], - 'complete_time': '2024-10-15T20:03:51.605000', - 'upload_id': 'RdA_3ZsOTMqbtAhYLivVsw', + 'complete_time': '2024-10-16T10:14:59.363000', + 'upload_id': 'e2b5o4KSR5yoS1EV9E9jXQ', 'upload_name': 'Test Upload', - 'upload_create_time': '2024-10-15T20:02:10.378000', + 'upload_create_time': '2024-10-16T10:14:57.722000', 'main_author': '8f052e1f-1906-41fd-b2eb-690c03407788', 'coauthors': [], 'coauthor_groups': [], @@ -694,7 +699,7 @@ published_upload 'embargo_length': 0, 'license': 'CC BY 4.0', 'entries': 1, - 'upload_files_server_path': '/nomad/test/fs/staging/R/RdA_3ZsOTMqbtAhYLivVsw'}} + 'upload_files_server_path': '/nomad/test/fs/staging/e/e2b5o4KSR5yoS1EV9E9jXQ'}} @@ -714,7 +719,7 @@ dataset_id - 'NCKd75f9R9S8rnkd-GBZlg' + 'EfhadCxpRaGpw50rWzK22w' @@ -723,23 +728,23 @@ The returned `dataset_id` can then be used to add individual entries (or all ent ```python metadata_new = {'dataset_id': dataset_id} -edit_upload_metadata(upload_id, url='test', **metadata_new) +edit_upload_metadata(upload_id, url='test', upload_metadata=metadata_new) ``` - {'upload_id': 'RdA_3ZsOTMqbtAhYLivVsw', + {'upload_id': 'e2b5o4KSR5yoS1EV9E9jXQ', 'data': {'process_running': True, 'current_process': 'edit_upload_metadata', 'process_status': 'PENDING', 'last_status_message': 'Pending: edit_upload_metadata', 'errors': [], 'warnings': [], - 'complete_time': '2024-10-15T20:09:28.769000', - 'upload_id': 'RdA_3ZsOTMqbtAhYLivVsw', + 'complete_time': '2024-10-16T10:15:04.112000', + 'upload_id': 'e2b5o4KSR5yoS1EV9E9jXQ', 'upload_name': 'Test Upload', - 'upload_create_time': '2024-10-15T20:02:10.378000', + 'upload_create_time': '2024-10-16T10:14:57.722000', 'main_author': '8f052e1f-1906-41fd-b2eb-690c03407788', 'coauthors': [], 'coauthor_groups': [], @@ -751,7 +756,7 @@ edit_upload_metadata(upload_id, url='test', **metadata_new) 'viewer_groups': [], 'published': True, 'published_to': [], - 'publish_time': '2024-10-15T20:09:28.757000', + 'publish_time': '2024-10-16T10:15:04.099000', 'with_embargo': False, 'embargo_length': 0, 'license': 'CC BY 4.0', @@ -797,7 +802,7 @@ nomad_dataset - NomadDataset(dataset_id='NCKd75f9R9S8rnkd-GBZlg', dataset_create_time=datetime.datetime(2024, 10, 15, 20, 10, 17, 568000), dataset_name='test dataset', dataset_type='owned', dataset_modified_time=datetime.datetime(2024, 10, 15, 20, 10, 17, 568000), user=NomadUser(name='Joseph Rudzinski'), doi=None, pid=None, m_annotations=None) + NomadDataset(dataset_id='EfhadCxpRaGpw50rWzK22w', dataset_create_time=datetime.datetime(2024, 10, 16, 10, 15, 5, 240000), dataset_name='test dataset', dataset_type='owned', dataset_modified_time=datetime.datetime(2024, 10, 16, 10, 15, 5, 240000), user=NomadUser(name='Joseph Rudzinski'), doi=None, pid=None, m_annotations=None) @@ -806,16 +811,16 @@ Alternatively, you can search for datasets, e.g., by `user_id` or `dataset_name` ```python my_datasets = retrieve_datasets( - user_id=nomad_user_me.user_id, url='test', max_datasets=20 + dataset_params={'user_id': nomad_user_me.user_id, 'max_datasets': 20}, url='test' ) pprint(my_datasets) ``` - [NomadDataset(dataset_id='NCKd75f9R9S8rnkd-GBZlg', - dataset_create_time=datetime.datetime(2024, 10, 15, 20, 10, 17, 568000), + [NomadDataset(dataset_id='EfhadCxpRaGpw50rWzK22w', + dataset_create_time=datetime.datetime(2024, 10, 16, 10, 15, 5, 240000), dataset_name='test dataset', dataset_type='owned', - dataset_modified_time=datetime.datetime(2024, 10, 15, 20, 10, 17, 568000), + dataset_modified_time=datetime.datetime(2024, 10, 16, 10, 15, 5, 240000), user=NomadUser(name='Joseph Rudzinski'), doi=None, pid=None, @@ -826,12 +831,12 @@ To get the list of entries contained within a dataset, use `query_entries()`: ```python -dataset_entries = query_entries(dataset_id=dataset_id, url='test') +dataset_entries = query_entries(query_params={'dataset_id': dataset_id}, url='test') for entry in dataset_entries: pprint(f'entry_id={entry.entry_id}, upload_id={entry.upload_id}') ``` - 'entry_id=Htbl78lHDSNAKbvPjEgEN_6sOcxF, upload_id=RdA_3ZsOTMqbtAhYLivVsw' + 'entry_id=u4qPSfILguvJ9fabpklMTxbUJ7x2, upload_id=e2b5o4KSR5yoS1EV9E9jXQ' There is no "publishing" action for datasets. Instead, when the dataset is complete (i.e., you are ready to lock the contents of the dataset), you can *assign a DOI*. There is currently no API action for this within nomad-utility-workflows. You must go to the GUI of the relevant deployment, go to `PUBLISH > Datasets`, find the dataset, and then click the "assign a DOI" banner icon to the right of the dataset entry. @@ -881,7 +886,7 @@ except Exception: pprint(f'Upload with upload_id={upload_id} was deleted successfully.') ``` - 'Upload with upload_id=zpq-JTzWQJ63jtSOlbueKA was deleted successfully.' + 'Upload with upload_id=Mfp1OnhMSY65eZCfNvS8DA was deleted successfully.' **create dataset, check for success, delete, check for success**: @@ -911,7 +916,7 @@ except Exception: pprint(f'Dataset with dataset_id={dataset_id} was deleted successfully.') ``` - 'Dataset with dataset_id=eT2WPkfCQgmwNadsurYz0A was deleted successfully.' + 'Dataset with dataset_id=Cke5DQkdQ0qbLOky2zGfLw was deleted successfully.' ## Useful Wrappers @@ -930,7 +935,12 @@ get_all_my_uploads(url='test') NomadUpload(upload_id='DN61X4r7SCyzm5q1kxcEcw', upload_create_time=datetime.datetime(2024, 10, 14, 10, 55, 12, 410000), main_author=NomadUser(name='Joseph Rudzinski'), process_running=False, current_process='publish_upload', process_status='SUCCESS', last_status_message='Process publish_upload completed successfully', errors=[], warnings=[], coauthors=[], coauthor_groups=[], reviewers=[], reviewer_groups=[], writers=[NomadUser(name='Joseph Rudzinski')], writer_groups=[], viewers=[NomadUser(name='Joseph Rudzinski')], viewer_groups=[], published=True, published_to=[], with_embargo=False, embargo_length=0.0, license='CC BY 4.0', entries=1, n_entries=None, upload_files_server_path=None, publish_time=datetime.datetime(2024, 10, 14, 10, 55, 23, 52000), references=None, datasets=None, external_db=None, upload_name='Test Upload', comment=None, url='https://nomad-lab.eu/prod/v1/test/api/v1', complete_time=datetime.datetime(2024, 10, 14, 10, 55, 23, 65000)), NomadUpload(upload_id='z4QvhZ7qSCmgIFv_qJqlyQ', upload_create_time=datetime.datetime(2024, 10, 14, 20, 20, 38, 757000), main_author=NomadUser(name='Joseph Rudzinski'), process_running=False, current_process='edit_upload_metadata', process_status='SUCCESS', last_status_message='Process edit_upload_metadata completed successfully', errors=[], warnings=[], coauthors=['7c85bdf1-8b53-40a8-81a4-04f26ff56f29'], coauthor_groups=[], reviewers=[], reviewer_groups=[], writers=[NomadUser(name='Joseph Rudzinski'), NomadUser(name='Joseph Rudzinski')], writer_groups=[], viewers=[NomadUser(name='Joseph Rudzinski'), NomadUser(name='Joseph Rudzinski')], viewer_groups=[], published=True, published_to=[], with_embargo=False, embargo_length=0.0, license='CC BY 4.0', entries=1, n_entries=None, upload_files_server_path=None, publish_time=datetime.datetime(2024, 10, 15, 6, 18, 27, 700000), references=None, datasets=None, external_db=None, upload_name='Test Upload', comment=None, url='https://nomad-lab.eu/prod/v1/test/api/v1', complete_time=datetime.datetime(2024, 10, 15, 6, 22, 33, 45000)), NomadUpload(upload_id='GJdVAOCxRVe-Cwo3qMz9Kg', upload_create_time=datetime.datetime(2024, 10, 15, 10, 48, 44, 337000), main_author=NomadUser(name='Joseph Rudzinski'), process_running=False, current_process='edit_upload_metadata', process_status='SUCCESS', last_status_message='Process edit_upload_metadata completed successfully', errors=[], warnings=[], coauthors=[], coauthor_groups=[], reviewers=[], reviewer_groups=[], writers=[NomadUser(name='Joseph Rudzinski')], writer_groups=[], viewers=[NomadUser(name='Joseph Rudzinski')], viewer_groups=[], published=True, published_to=[], with_embargo=False, embargo_length=0.0, license='CC BY 4.0', entries=1, n_entries=None, upload_files_server_path=None, publish_time=datetime.datetime(2024, 10, 15, 10, 49, 24, 4000), references=None, datasets=None, external_db=None, upload_name='Test Upload', comment=None, url='https://nomad-lab.eu/prod/v1/test/api/v1', complete_time=datetime.datetime(2024, 10, 15, 10, 49, 30, 962000)), - NomadUpload(upload_id='RdA_3ZsOTMqbtAhYLivVsw', upload_create_time=datetime.datetime(2024, 10, 15, 20, 2, 10, 378000), main_author=NomadUser(name='Joseph Rudzinski'), process_running=False, current_process='edit_upload_metadata', process_status='SUCCESS', last_status_message='Process edit_upload_metadata completed successfully', errors=[], warnings=[], coauthors=[], coauthor_groups=[], reviewers=[], reviewer_groups=[], writers=[NomadUser(name='Joseph Rudzinski')], writer_groups=[], viewers=[NomadUser(name='Joseph Rudzinski')], viewer_groups=[], published=True, published_to=[], with_embargo=False, embargo_length=0.0, license='CC BY 4.0', entries=1, n_entries=None, upload_files_server_path=None, publish_time=datetime.datetime(2024, 10, 15, 20, 9, 28, 757000), references=None, datasets=None, external_db=None, upload_name='Test Upload', comment=None, url='https://nomad-lab.eu/prod/v1/test/api/v1', complete_time=datetime.datetime(2024, 10, 15, 20, 10, 33, 141000))] + NomadUpload(upload_id='RdA_3ZsOTMqbtAhYLivVsw', upload_create_time=datetime.datetime(2024, 10, 15, 20, 2, 10, 378000), main_author=NomadUser(name='Joseph Rudzinski'), process_running=False, current_process='edit_upload_metadata', process_status='SUCCESS', last_status_message='Process edit_upload_metadata completed successfully', errors=[], warnings=[], coauthors=[], coauthor_groups=[], reviewers=[], reviewer_groups=[], writers=[NomadUser(name='Joseph Rudzinski')], writer_groups=[], viewers=[NomadUser(name='Joseph Rudzinski')], viewer_groups=[], published=True, published_to=[], with_embargo=False, embargo_length=0.0, license='CC BY 4.0', entries=1, n_entries=None, upload_files_server_path=None, publish_time=datetime.datetime(2024, 10, 15, 20, 9, 28, 757000), references=None, datasets=None, external_db=None, upload_name='Test Upload', comment=None, url='https://nomad-lab.eu/prod/v1/test/api/v1', complete_time=datetime.datetime(2024, 10, 15, 20, 10, 33, 141000)), + NomadUpload(upload_id='8vViZoL3TYG9fMFibPkjlw', upload_create_time=datetime.datetime(2024, 10, 16, 9, 25, 53, 929000), main_author=NomadUser(name='Joseph Rudzinski'), process_running=False, current_process='publish_upload', process_status='SUCCESS', last_status_message='Process publish_upload completed successfully', errors=[], warnings=[], coauthors=[], coauthor_groups=[], reviewers=[], reviewer_groups=[], writers=[NomadUser(name='Joseph Rudzinski')], writer_groups=[], viewers=[NomadUser(name='Joseph Rudzinski')], viewer_groups=[], published=True, published_to=[], with_embargo=False, embargo_length=0.0, license='CC BY 4.0', entries=1, n_entries=None, upload_files_server_path=None, publish_time=datetime.datetime(2024, 10, 16, 9, 43, 18, 243000), references=None, datasets=None, external_db=None, upload_name='Test Upload', comment=None, url='https://nomad-lab.eu/prod/v1/test/api/v1', complete_time=datetime.datetime(2024, 10, 16, 9, 43, 18, 255000)), + NomadUpload(upload_id='cP4q5rRsQM-D60Tp3olPdQ', upload_create_time=datetime.datetime(2024, 10, 16, 9, 47, 31, 721000), main_author=NomadUser(name='Joseph Rudzinski'), process_running=False, current_process='publish_upload', process_status='SUCCESS', last_status_message='Process publish_upload completed successfully', errors=[], warnings=[], coauthors=[], coauthor_groups=[], reviewers=[], reviewer_groups=[], writers=[NomadUser(name='Joseph Rudzinski')], writer_groups=[], viewers=[NomadUser(name='Joseph Rudzinski')], viewer_groups=[], published=True, published_to=[], with_embargo=False, embargo_length=0.0, license='CC BY 4.0', entries=1, n_entries=None, upload_files_server_path=None, publish_time=datetime.datetime(2024, 10, 16, 9, 47, 46, 247000), references=None, datasets=None, external_db=None, upload_name='Test Upload', comment=None, url='https://nomad-lab.eu/prod/v1/test/api/v1', complete_time=datetime.datetime(2024, 10, 16, 9, 47, 46, 257000)), + NomadUpload(upload_id='5ADf3M4uSByqsYpkEB6UEg', upload_create_time=datetime.datetime(2024, 10, 16, 9, 52, 1, 469000), main_author=NomadUser(name='Joseph Rudzinski'), process_running=False, current_process='process_upload', process_status='SUCCESS', last_status_message='Process process_upload completed successfully', errors=[], warnings=[], coauthors=[], coauthor_groups=[], reviewers=[], reviewer_groups=[], writers=[NomadUser(name='Joseph Rudzinski')], writer_groups=[], viewers=[NomadUser(name='Joseph Rudzinski')], viewer_groups=[], published=False, published_to=[], with_embargo=False, embargo_length=0.0, license='CC BY 4.0', entries=1, n_entries=None, upload_files_server_path='/nomad/test/fs/staging/5/5ADf3M4uSByqsYpkEB6UEg', publish_time=None, references=None, datasets=None, external_db=None, upload_name=None, comment=None, url='https://nomad-lab.eu/prod/v1/test/api/v1', complete_time=datetime.datetime(2024, 10, 16, 9, 52, 2, 94000)), + NomadUpload(upload_id='_mZn0RZ8QtmBkcAlPU5bSw', upload_create_time=datetime.datetime(2024, 10, 16, 9, 52, 47, 649000), main_author=NomadUser(name='Joseph Rudzinski'), process_running=False, current_process='edit_upload_metadata', process_status='SUCCESS', last_status_message='Process edit_upload_metadata completed successfully', errors=[], warnings=[], coauthors=[], coauthor_groups=[], reviewers=[], reviewer_groups=[], writers=[NomadUser(name='Joseph Rudzinski')], writer_groups=[], viewers=[NomadUser(name='Joseph Rudzinski')], viewer_groups=[], published=True, published_to=[], with_embargo=False, embargo_length=0.0, license='CC BY 4.0', entries=1, n_entries=None, upload_files_server_path=None, publish_time=datetime.datetime(2024, 10, 16, 9, 53, 0, 835000), references=None, datasets=None, external_db=None, upload_name='Test Upload', comment=None, url='https://nomad-lab.eu/prod/v1/test/api/v1', complete_time=datetime.datetime(2024, 10, 16, 9, 53, 4, 791000)), + NomadUpload(upload_id='Cntk6OsQTvaZp7r6Jom-3g', upload_create_time=datetime.datetime(2024, 10, 16, 9, 54, 47, 426000), main_author=NomadUser(name='Joseph Rudzinski'), process_running=False, current_process='edit_upload_metadata', process_status='SUCCESS', last_status_message='Process edit_upload_metadata completed successfully', errors=[], warnings=[], coauthors=[], coauthor_groups=[], reviewers=[], reviewer_groups=[], writers=[NomadUser(name='Joseph Rudzinski')], writer_groups=[], viewers=[NomadUser(name='Joseph Rudzinski')], viewer_groups=[], published=True, published_to=[], with_embargo=False, embargo_length=0.0, license='CC BY 4.0', entries=1, n_entries=None, upload_files_server_path=None, publish_time=datetime.datetime(2024, 10, 16, 9, 54, 51, 988000), references=None, datasets=None, external_db=None, upload_name='Test Upload', comment=None, url='https://nomad-lab.eu/prod/v1/test/api/v1', complete_time=datetime.datetime(2024, 10, 16, 9, 54, 53, 838000))] @@ -946,7 +956,12 @@ get_entries_of_my_uploads(url='test') NomadEntry(entry_id='7A6lJb-14xR9lxXO8kjuYt5-vxg2', upload_id='DN61X4r7SCyzm5q1kxcEcw', references=[], origin='Joseph Rudzinski', n_quantities=34, nomad_version='1.3.7.dev55+ge83de27b3', upload_create_time=datetime.datetime(2024, 10, 14, 10, 55, 12, 410000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), nomad_commit='', processing_errors=[], entry_name='test.archive.json', last_processing_time=datetime.datetime(2024, 10, 14, 10, 55, 12, 808000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), parser_name='parsers/archive', calc_id='7A6lJb-14xR9lxXO8kjuYt5-vxg2', published=True, writers=[NomadUser(name='Joseph Rudzinski')], processed=True, mainfile='test.archive.json', main_author=NomadUser(name='Joseph Rudzinski'), entry_create_time=datetime.datetime(2024, 10, 14, 10, 55, 12, 563000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), with_embargo=False, entry_type=None, license='CC BY 4.0', domain=None, comment='This is a test upload...', upload_name='Test Upload', text_search_contents=[], publish_time=None, entry_references=None, url='https://nomad-lab.eu/prod/v1/test/api/v1'), NomadEntry(entry_id='jWSpYURP5GgPtgF9LXZJpNlDv-GL', upload_id='z4QvhZ7qSCmgIFv_qJqlyQ', references=[], origin='Joseph Rudzinski', n_quantities=0, nomad_version='1.3.7.dev55+ge83de27b3', upload_create_time=datetime.datetime(2024, 10, 14, 20, 20, 38, 757000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), nomad_commit='', processing_errors=[], entry_name='test.archive.json', last_processing_time=datetime.datetime(2024, 10, 14, 20, 20, 39, 272000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), parser_name='parsers/archive', calc_id='jWSpYURP5GgPtgF9LXZJpNlDv-GL', published=True, writers=[NomadUser(name='Joseph Rudzinski'), NomadUser(name='Joseph Rudzinski')], processed=True, mainfile='test.archive.json', main_author=NomadUser(name='Joseph Rudzinski'), entry_create_time=datetime.datetime(2024, 10, 14, 20, 20, 38, 982000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), with_embargo=False, entry_type=None, license='CC BY 4.0', domain=None, comment='This is a test upload...edited', upload_name='Test Upload', text_search_contents=[], publish_time=datetime.datetime(2024, 10, 15, 6, 18, 27, 700000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), entry_references=None, url='https://nomad-lab.eu/prod/v1/test/api/v1'), NomadEntry(entry_id='MVBIMEZOuIzH7-QFU2TtMIM6LLPp', upload_id='GJdVAOCxRVe-Cwo3qMz9Kg', references=[], origin='Joseph Rudzinski', n_quantities=0, nomad_version='1.3.7.dev55+ge83de27b3', upload_create_time=datetime.datetime(2024, 10, 15, 10, 48, 44, 337000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), nomad_commit='', processing_errors=[], entry_name='test.archive.json', last_processing_time=datetime.datetime(2024, 10, 15, 10, 48, 45, 206000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), parser_name='parsers/archive', calc_id='MVBIMEZOuIzH7-QFU2TtMIM6LLPp', published=True, writers=[NomadUser(name='Joseph Rudzinski')], processed=True, mainfile='test.archive.json', main_author=NomadUser(name='Joseph Rudzinski'), entry_create_time=datetime.datetime(2024, 10, 15, 10, 48, 44, 741000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), with_embargo=False, entry_type=None, license='CC BY 4.0', domain=None, comment='This is a test upload...', upload_name='Test Upload', text_search_contents=[], publish_time=datetime.datetime(2024, 10, 15, 10, 49, 24, 4000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), entry_references=None, url='https://nomad-lab.eu/prod/v1/test/api/v1'), - NomadEntry(entry_id='Htbl78lHDSNAKbvPjEgEN_6sOcxF', upload_id='RdA_3ZsOTMqbtAhYLivVsw', references=[], origin='Joseph Rudzinski', n_quantities=0, nomad_version='1.3.7.dev55+ge83de27b3', upload_create_time=datetime.datetime(2024, 10, 15, 20, 2, 10, 378000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), nomad_commit='', processing_errors=[], entry_name='test.archive.json', last_processing_time=datetime.datetime(2024, 10, 15, 20, 2, 10, 752000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), parser_name='parsers/archive', calc_id='Htbl78lHDSNAKbvPjEgEN_6sOcxF', published=True, writers=[NomadUser(name='Joseph Rudzinski')], processed=True, mainfile='test.archive.json', main_author=NomadUser(name='Joseph Rudzinski'), entry_create_time=datetime.datetime(2024, 10, 15, 20, 2, 10, 543000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), with_embargo=False, entry_type=None, license='CC BY 4.0', domain=None, comment='This is a test upload...', upload_name='Test Upload', text_search_contents=[], publish_time=datetime.datetime(2024, 10, 15, 20, 9, 28, 757000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), entry_references=None, url='https://nomad-lab.eu/prod/v1/test/api/v1')] + NomadEntry(entry_id='Htbl78lHDSNAKbvPjEgEN_6sOcxF', upload_id='RdA_3ZsOTMqbtAhYLivVsw', references=[], origin='Joseph Rudzinski', n_quantities=0, nomad_version='1.3.7.dev55+ge83de27b3', upload_create_time=datetime.datetime(2024, 10, 15, 20, 2, 10, 378000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), nomad_commit='', processing_errors=[], entry_name='test.archive.json', last_processing_time=datetime.datetime(2024, 10, 15, 20, 2, 10, 752000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), parser_name='parsers/archive', calc_id='Htbl78lHDSNAKbvPjEgEN_6sOcxF', published=True, writers=[NomadUser(name='Joseph Rudzinski')], processed=True, mainfile='test.archive.json', main_author=NomadUser(name='Joseph Rudzinski'), entry_create_time=datetime.datetime(2024, 10, 15, 20, 2, 10, 543000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), with_embargo=False, entry_type=None, license='CC BY 4.0', domain=None, comment='This is a test upload...', upload_name='Test Upload', text_search_contents=[], publish_time=datetime.datetime(2024, 10, 15, 20, 9, 28, 757000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), entry_references=None, url='https://nomad-lab.eu/prod/v1/test/api/v1'), + NomadEntry(entry_id='zyaF373NIH-igHS3TZN5FW4SaO4d', upload_id='8vViZoL3TYG9fMFibPkjlw', references=[], origin='Joseph Rudzinski', n_quantities=34, nomad_version='1.3.7.dev55+ge83de27b3', upload_create_time=datetime.datetime(2024, 10, 16, 9, 25, 53, 929000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), nomad_commit='', processing_errors=[], entry_name='test.archive.json', last_processing_time=datetime.datetime(2024, 10, 16, 9, 25, 54, 274000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), parser_name='parsers/archive', calc_id='zyaF373NIH-igHS3TZN5FW4SaO4d', published=True, writers=[NomadUser(name='Joseph Rudzinski')], processed=True, mainfile='test.archive.json', main_author=NomadUser(name='Joseph Rudzinski'), entry_create_time=datetime.datetime(2024, 10, 16, 9, 25, 54, 111000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), with_embargo=False, entry_type=None, license='CC BY 4.0', domain=None, comment='This is a test upload...', upload_name='Test Upload', text_search_contents=[], publish_time=None, entry_references=None, url='https://nomad-lab.eu/prod/v1/test/api/v1'), + NomadEntry(entry_id='Xitkh3ZVhRu11LUIk5n0cA2Wtmmy', upload_id='cP4q5rRsQM-D60Tp3olPdQ', references=[], origin='Joseph Rudzinski', n_quantities=34, nomad_version='1.3.7.dev55+ge83de27b3', upload_create_time=datetime.datetime(2024, 10, 16, 9, 47, 31, 721000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), nomad_commit='', processing_errors=[], entry_name='test.archive.json', last_processing_time=datetime.datetime(2024, 10, 16, 9, 47, 32, 28000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), parser_name='parsers/archive', calc_id='Xitkh3ZVhRu11LUIk5n0cA2Wtmmy', published=True, writers=[NomadUser(name='Joseph Rudzinski')], processed=True, mainfile='test.archive.json', main_author=NomadUser(name='Joseph Rudzinski'), entry_create_time=datetime.datetime(2024, 10, 16, 9, 47, 31, 857000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), with_embargo=False, entry_type=None, license='CC BY 4.0', domain=None, comment='This is a test upload...', upload_name='Test Upload', text_search_contents=[], publish_time=None, entry_references=None, url='https://nomad-lab.eu/prod/v1/test/api/v1'), + NomadEntry(entry_id='EaFsd7Ku9IEXOcwTcMqlWd92fZx1', upload_id='5ADf3M4uSByqsYpkEB6UEg', references=[], origin='Joseph Rudzinski', n_quantities=34, nomad_version='1.3.7.dev55+ge83de27b3', upload_create_time=datetime.datetime(2024, 10, 16, 9, 52, 1, 469000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), nomad_commit='', processing_errors=[], entry_name='test.archive.json', last_processing_time=datetime.datetime(2024, 10, 16, 9, 52, 1, 752000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), parser_name='parsers/archive', calc_id='EaFsd7Ku9IEXOcwTcMqlWd92fZx1', published=False, writers=[NomadUser(name='Joseph Rudzinski')], processed=True, mainfile='test.archive.json', main_author=NomadUser(name='Joseph Rudzinski'), entry_create_time=datetime.datetime(2024, 10, 16, 9, 52, 1, 595000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), with_embargo=False, entry_type=None, license='CC BY 4.0', domain=None, comment=None, upload_name=None, text_search_contents=[], publish_time=None, entry_references=None, url='https://nomad-lab.eu/prod/v1/test/api/v1'), + NomadEntry(entry_id='h3e0Z5FHiUetLmW8kbPW4uwrT0gH', upload_id='_mZn0RZ8QtmBkcAlPU5bSw', references=[], origin='Joseph Rudzinski', n_quantities=0, nomad_version='1.3.7.dev55+ge83de27b3', upload_create_time=datetime.datetime(2024, 10, 16, 9, 52, 47, 649000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), nomad_commit='', processing_errors=[], entry_name='test.archive.json', last_processing_time=datetime.datetime(2024, 10, 16, 9, 52, 47, 948000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), parser_name='parsers/archive', calc_id='h3e0Z5FHiUetLmW8kbPW4uwrT0gH', published=True, writers=[NomadUser(name='Joseph Rudzinski')], processed=True, mainfile='test.archive.json', main_author=NomadUser(name='Joseph Rudzinski'), entry_create_time=datetime.datetime(2024, 10, 16, 9, 52, 47, 780000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), with_embargo=False, entry_type=None, license='CC BY 4.0', domain=None, comment='This is a test upload...', upload_name='Test Upload', text_search_contents=[], publish_time=datetime.datetime(2024, 10, 16, 9, 53, 0, 835000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), entry_references=None, url='https://nomad-lab.eu/prod/v1/test/api/v1'), + NomadEntry(entry_id='lJiZnALI0ad8UKh5nt2FG1rhaZiC', upload_id='Cntk6OsQTvaZp7r6Jom-3g', references=[], origin='Joseph Rudzinski', n_quantities=0, nomad_version='1.3.7.dev55+ge83de27b3', upload_create_time=datetime.datetime(2024, 10, 16, 9, 54, 47, 426000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), nomad_commit='', processing_errors=[], entry_name='test.archive.json', last_processing_time=datetime.datetime(2024, 10, 16, 9, 54, 47, 719000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), parser_name='parsers/archive', calc_id='lJiZnALI0ad8UKh5nt2FG1rhaZiC', published=True, writers=[NomadUser(name='Joseph Rudzinski')], processed=True, mainfile='test.archive.json', main_author=NomadUser(name='Joseph Rudzinski'), entry_create_time=datetime.datetime(2024, 10, 16, 9, 54, 47, 553000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), with_embargo=False, entry_type=None, license='CC BY 4.0', domain=None, comment='This is a test upload...', upload_name='Test Upload', text_search_contents=[], publish_time=datetime.datetime(2024, 10, 16, 9, 54, 51, 988000, tzinfo=datetime.timezone(datetime.timedelta(0), '+0000')), entry_references=None, url='https://nomad-lab.eu/prod/v1/test/api/v1')] diff --git a/tests/utils/how_to_use_api_functions.pdf b/tests/utils/how_to_use_api_functions.pdf index 623070e..41b805d 100644 Binary files a/tests/utils/how_to_use_api_functions.pdf and b/tests/utils/how_to_use_api_functions.pdf differ diff --git a/tests/utils/test_workflow.ipynb b/tests/utils/test_workflow.ipynb index 52e1bef..30733b7 100644 --- a/tests/utils/test_workflow.ipynb +++ b/tests/utils/test_workflow.ipynb @@ -21,13 +21,16 @@ "# workflow2:\n", "# inputs:\n", "# - name: input system\n", - "# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/minimEQ1.log#/run/0/system/0'\n", + "# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/minimEQ1.log#/run/0\n", + "# /system/0'\n", "\n", "# outputs:\n", "# - name: final system of production trajectory\n", "# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/PD1.log#/run/0/system/-1'\n", "# - name: final calculation of production trajectory\n", - "# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/PD1.log#/run/0/calculation/-1'\n", + "# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/PD1.log#/run/0\n", + "# /calculation\n", + "# /-1'\n", "\n", "# tasks:\n", "# - m_def: nomad.datamodel.metainfo.workflow.TaskReference\n", @@ -35,56 +38,72 @@ "# name: GeometryOpt\n", "# inputs:\n", "# - name: input system\n", - "# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/minimEQ1.log#/run/0/system/0'\n", + "# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/minimEQ1.log#/run/0\n", + "# /system/0'\n", "# outputs:\n", "# - name: relaxed system\n", - "# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/minimEQ1.log#/run/0/system/-1'\n", + "# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/minimEQ1.log#/run/0/\n", + "# system/-1'\n", "# - name: energy and pressure of the relaxed system\n", - "# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/minimEQ1.log#/run/0/calculation/-1'\n", + "# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/minimEQ1.log#/run/0/\n", + "# calculation/-1'\n", "# - m_def: nomad.datamodel.metainfo.workflow.TaskReference\n", "# task: '../upload/archive/mainfile/MOL0_160_MOL1_240/EQ1.log#/workflow2'\n", "# name: MolecularDynamics\n", "# inputs:\n", "# - name: input system\n", - "# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/minimEQ1.log#/run/0/system/-1'\n", + "# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/minimEQ1.log#/run/0\n", + "# /system/-1'\n", "# outputs:\n", "# - name: final system from high temp NVT equilibration\n", - "# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/EQ1.log#/run/0/system/-1'\n", + "# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/EQ1.log#/run/0/\n", + "# system/-1'\n", "# - name: final thermodynamic quantities of high temp NVT equilibration\n", - "# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/EQ1.log#/run/0/calculation/-1'\n", + "# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/EQ1.log#/run/0/\n", + "# calculation\n", + "# /-1'\n", "# - m_def: nomad.datamodel.metainfo.workflow.TaskReference\n", "# task: '../upload/archive/mainfile/MOL0_160_MOL1_240/EQ2.log#/workflow2'\n", "# name: MolecularDynamics\n", "# inputs:\n", "# - name: input system\n", - "# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/EQ1.log#/run/0/system/-1'\n", + "# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/EQ1.log#/run/0/system\n", + "# /-1'\n", "# outputs:\n", "# - name: final system from NVT cool down to 300k\n", - "# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/EQ2.log#/run/0/system/-1'\n", + "# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/EQ2.log#/run/0/system\n", + "# /-1'\n", "# - name: final thermodynamic quantities of NVT cool down to 300k\n", - "# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/EQ2.log#/run/0/calculation/-1'\n", + "# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/EQ2.log#/run/0/\n", + "# calculation/-1'\n", "# - m_def: nomad.datamodel.metainfo.workflow.TaskReference\n", "# task: '../upload/archive/mainfile/MOL0_160_MOL1_240/EQ3.log#/workflow2'\n", "# name: MolecularDynamics\n", "# inputs:\n", "# - name: input system\n", - "# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/EQ2.log#/run/0/system/-1'\n", + "# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/EQ2.log#/run/0\n", + "# /system/-1'\n", "# outputs:\n", "# - name: final system from NPT equilibration\n", - "# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/EQ3.log#/run/0/system/-1'\n", + "# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/EQ3.log#/run/0\n", + "# /system/-1'\n", "# - name: final thermodynamic quantities of NPT equilibration\n", - "# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/EQ3.log#/run/0/calculation/-1'\n", + "# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/EQ3.log#/run/0\n", + "# /calculation/-1'\n", "# - m_def: nomad.datamodel.metainfo.workflow.TaskReference\n", "# task: '../upload/archive/mainfile/MOL0_160_MOL1_240/PD1.log#/workflow2'\n", "# name: MolecularDynamics\n", "# inputs:\n", "# - name: input system\n", - "# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/EQ3.log#/run/0/system/-1'\n", + "# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/EQ3.log#/run/0\n", + "# /system/-1'\n", "# outputs:\n", "# - name: final system from NPT production run\n", - "# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/PD1.log#/run/0/system/-1'\n", + "# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/PD1.log#/run/0\n", + "# /system/-1'\n", "# - name: final thermodynamic quantities of NPT production run\n", - "# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/PD1.log#/run/0/calculation/-1'" + "# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/PD1.log#/run/0\n", + "# /calculation/-1'" ] }, { @@ -8668,10 +8687,13 @@ " },\n", "}\n", "# TODO there will be 2 functionalities for input/outputs and creating edges:\n", - "# TODO 1. place node references in node_attributes -- edges will be created automatically\n", - "# TODO 2. use edge attributes to set the edges and also inputs/outputs between (recommended to create edges explicitly!)\n", + "# TODO 1. place node references in node_attributes -- edges will be created\n", + "# automatically\n", + "# TODO 2. use edge attributes to set the edges and also inputs/outputs between\n", + "# (recommended to create edges explicitly!)\n", "\n", - "# TODO someone should also be able to create the graph their own way...that means we need to integrate the workflow attributes into the graph!\n", + "# TODO someone should also be able to create the graph their own way...that means we\n", + "# need to integrate the workflow attributes into the graph!\n", "\n", "edge_attributes = { # ! inputs and outputs references the first node\n", " (0, 1): {\n", @@ -13283,10 +13305,13 @@ "outputs": [], "source": [ "# TODO there will be 2 functionalities for input/outputs and creating edges:\n", - "# TODO 1. place node references in node_attributes -- edges will be created automatically\n", - "# TODO 2. use edge attributes to set the edges and also inputs/outputs between (recommended to create edges explicitly!)\n", + "# TODO 1. place node references in node_attributes -- edges will be created\n", + "# automatically\n", + "# TODO 2. use edge attributes to set the edges and also inputs/outputs between\n", + "# (recommended to create edges explicitly!)\n", "\n", - "# TODO someone should also be able to create the graph their own way...that means we need to integrate the workflow attributes into the graph!\n", + "# TODO someone should also be able to create the graph their own way...that means we\n", + "# need to integrate the workflow attributes into the graph!\n", "\n", "# ! types = MEnum('task', 'input', 'output')\n", "\n", diff --git a/tests/utils/test_workflow.py b/tests/utils/test_workflow.py index e6836e5..d44da58 100644 --- a/tests/utils/test_workflow.py +++ b/tests/utils/test_workflow.py @@ -1,15 +1,16 @@ -import numpy as np # EXAMPLE 1 - MD Equilibration Workflow # workflow2: # inputs: # - name: input system -# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/minimEQ1.log#/run/0/system/0' +# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/minimEQ1.log# +# /run/0/system/0' # outputs: # - name: final system of production trajectory # section: '../upload/archive/mainfile/MOL0_160_MOL1_240/PD1.log#/run/0/system/-1' # - name: final calculation of production trajectory -# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/PD1.log#/run/0/calculation/-1' +# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/PD1.log#/run/0 +# /calculation/-1' # tasks: # - m_def: nomad.datamodel.metainfo.workflow.TaskReference @@ -17,107 +18,122 @@ # name: GeometryOpt # inputs: # - name: input system -# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/minimEQ1.log#/run/0/system/0' +# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/minimEQ1.log#/run/0 +# /system/0' # outputs: # - name: relaxed system -# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/minimEQ1.log#/run/0/system/-1' +# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/minimEQ1.log#/run/0 +# /system/-1' # - name: energy and pressure of the relaxed system -# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/minimEQ1.log#/run/0/calculation/-1' +# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/minimEQ1.log#/run/0 +# /calculation/-1' # - m_def: nomad.datamodel.metainfo.workflow.TaskReference # task: '../upload/archive/mainfile/MOL0_160_MOL1_240/EQ1.log#/workflow2' # name: MolecularDynamics # inputs: # - name: input system -# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/minimEQ1.log#/run/0/system/-1' +# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/minimEQ1.log#/run/0 +# /system/-1' # outputs: # - name: final system from high temp NVT equilibration -# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/EQ1.log#/run/0/system/-1' +# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/EQ1.log#/run/0/system +# /-1' # - name: final thermodynamic quantities of high temp NVT equilibration -# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/EQ1.log#/run/0/calculation/-1' +# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/EQ1.log#/run/0 +# /calculation/-1' # - m_def: nomad.datamodel.metainfo.workflow.TaskReference # task: '../upload/archive/mainfile/MOL0_160_MOL1_240/EQ2.log#/workflow2' # name: MolecularDynamics # inputs: # - name: input system -# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/EQ1.log#/run/0/system/-1' +# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/EQ1.log#/run/0/system +# /-1' # outputs: # - name: final system from NVT cool down to 300k -# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/EQ2.log#/run/0/system/-1' +# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/EQ2.log#/run/0/system +# /-1' # - name: final thermodynamic quantities of NVT cool down to 300k -# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/EQ2.log#/run/0/calculation/-1' +# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/EQ2.log#/run/0 +# /calculation/-1' # - m_def: nomad.datamodel.metainfo.workflow.TaskReference # task: '../upload/archive/mainfile/MOL0_160_MOL1_240/EQ3.log#/workflow2' # name: MolecularDynamics # inputs: # - name: input system -# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/EQ2.log#/run/0/system/-1' +# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/EQ2.log#/run/0/system +# /-1' # outputs: # - name: final system from NPT equilibration -# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/EQ3.log#/run/0/system/-1' +# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/EQ3.log#/run/0/system +# /-1' # - name: final thermodynamic quantities of NPT equilibration -# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/EQ3.log#/run/0/calculation/-1' +# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/EQ3.log#/run/0 +# /calculation/-1' # - m_def: nomad.datamodel.metainfo.workflow.TaskReference # task: '../upload/archive/mainfile/MOL0_160_MOL1_240/PD1.log#/workflow2' # name: MolecularDynamics # inputs: # - name: input system -# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/EQ3.log#/run/0/system/-1' +# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/EQ3.log#/run/0/system +# /-1' # outputs: # - name: final system from NPT production run -# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/PD1.log#/run/0/system/-1' +# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/PD1.log#/run/0/system +# /-1' # - name: final thermodynamic quantities of NPT production run -# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/PD1.log#/run/0/calculation/-1' -n_nodes = 5 -adjaceny_matrix = np.zeros((n_nodes, n_nodes)) -edges = [(i, i + 1) for i in range(n_nodes - 1)] -for edge in edges: - adjaceny_matrix[edge[0], edge[1]] = 1 +# section: '../upload/archive/mainfile/MOL0_160_MOL1_240/PD1.log#/ +# run/0/calculation/-1' +# n_nodes = 5 +# adjaceny_matrix = np.zeros((n_nodes, n_nodes)) +# edges = [(i, i + 1) for i in range(n_nodes - 1)] +# for edge in edges: +# adjaceny_matrix[edge[0], edge[1]] = 1 -print(adjaceny_matrix) -node_attributes = { - 0: {'label': 'GeometryOpt', - 'type': 'GeometryOpt', - 'upload_id': None, - 'entry_id': None, - 'mainfile': None, - 'archive_path': 'workflow2', - 'non_edge_inputs': [{'name': 'input system', - 'type': 'system', - 'archive_path': 'run/0/system/0'}], - 'non_edge_outputs': [{'name': 'relaxed system'}] - }, -} -edge_attributes = { - (0, 1): {'label': 'MolecularDynamics', - 'type': 'MolecularDynamics', - 'm_def': ??, - 'edge_inputs': [{'name': 'input system', - 'type': 'system', - 'archive_path': 'run/0/system/0'}], - 'edge_outputs': [{'name': 'relaxed system'}] - }, - (1, 2): {'label': 'MolecularDynamics', - 'type': 'MolecularDynamics', - 'm_def': ??, - 'edge_inputs': [{'name': 'input system', - 'type': 'system', - 'archive_path': 'run/0/system/0'}], - 'edge_outputs': [{'name': 'relaxed system'}] - } -} +# print(adjaceny_matrix) +# node_attributes = { +# 0: {'label': 'GeometryOpt', +# 'type': 'GeometryOpt', +# 'upload_id': None, +# 'entry_id': None, +# 'mainfile': None, +# 'archive_path': 'workflow2', +# 'non_edge_inputs': [{'name': 'input system', +# 'type': 'system', +# 'archive_path': 'run/0/system/0'}], +# 'non_edge_outputs': [{'name': 'relaxed system'}] +# }, +# } +# edge_attributes = { +# (0, 1): {'label': 'MolecularDynamics', +# 'type': 'MolecularDynamics', +# 'm_def': ??, +# 'edge_inputs': [{'name': 'input system', +# 'type': 'system', +# 'archive_path': 'run/0/system/0'}], +# 'edge_outputs': [{'name': 'relaxed system'}] +# }, +# (1, 2): {'label': 'MolecularDynamics', +# 'type': 'MolecularDynamics', +# 'm_def': ??, +# 'edge_inputs': [{'name': 'input system', +# 'type': 'system', +# 'archive_path': 'run/0/system/0'}], +# 'edge_outputs': [{'name': 'relaxed system'}] +# } +# } -# Add node attributes to the graph -for node, attrs in node_attributes.items(): - G.nodes[node].update(attrs) +# # Add node attributes to the graph +# for node, attrs in node_attributes.items(): +# G.nodes[node].update(attrs) -# Add edge attributes to the graph -for edge, attrs in edge_attributes.items(): - if G.has_edge(*edge): - G.edges[edge].update(attrs) +# # Add edge attributes to the graph +# for edge, attrs in edge_attributes.items(): +# if G.has_edge(*edge): +# G.edges[edge].update(attrs) -# Check if a specific edge exists -edge_to_check = (0, 1) -if G.has_edge(*edge_to_check): - print(f"Edge {edge_to_check} exists.") -else: - print(f"Edge {edge_to_check} does not exist.") +# # Check if a specific edge exists +# edge_to_check = (0, 1) +# if G.has_edge(*edge_to_check): +# print(f"Edge {edge_to_check} exists.") +# else: +# print(f"Edge {edge_to_check} does not exist.") diff --git a/tests/utils/test_workflow_simple.ipynb b/tests/utils/test_workflow_simple.ipynb index cf87457..b361c40 100644 --- a/tests/utils/test_workflow_simple.ipynb +++ b/tests/utils/test_workflow_simple.ipynb @@ -31,7 +31,9 @@ "source": [ "# import os\n", "# import sys\n", - "# sys.path.append('/home/jfrudzinski/work/soft/plugins/nomad-simulation-workflow-utilities/src/nomad_simulation_workflow_utilities')\n", + "# sys.path.append(\n", + "# '/home/jfrudzinski/work/soft/plugins/nomad-simulation-workflow-utilities/src\n", + "# /nomad_simulation_workflow_utilities')\n", "\n", "from nomad_utility_workflows.utils import build_nomad_workflow" ] @@ -4295,7 +4297,9 @@ " 'path_info': {\n", " 'upload_id': '',\n", " 'entry_id': None,\n", - " 'mainfile_path': '', # ! Global inputs must reference the mainfile explicitly (i.e., not inferred from the edge node)\n", + " 'mainfile_path': '',\n", + " # ! Global inputs must reference the mainfile explicitly (i.e., not inferred\n", + " # from the edge node)\n", " 'supersection_index': 0,\n", " 'section_index': 0,\n", " 'section_type': 'method',\n", @@ -4334,7 +4338,9 @@ " # 'archive_path': 'run/0/calculation/-1',\n", " },\n", " }\n", - " ], # TODO In case of node reference, I should connect the mainfile inside archive_path_info\n", + " ],\n", + " # TODO In case of node reference, I should connect the mainfile inside\n", + " # archive_path_info\n", " },\n", " 2: {\n", " 'name': '2',\n", @@ -4394,7 +4400,9 @@ " },\n", " 'in_edge_nodes': [\n", " 3\n", - " ], # TODO if this is an output and connected node is a workflow, automatically reference workflow results? Or somehow need to know to put in workflow2?\n", + " ],\n", + " # TODO if this is an output and connected node is a workflow, automatically\n", + " # reference workflow results? Or somehow need to know to put in workflow2?\n", " },\n", "}\n", "\n", @@ -4652,7 +4660,8 @@ "outputs": [], "source": [ "nx.set_edge_attributes(workflow_graph, {(0, 1): {'outputs': ['test']}})\n", - "# nx.set_edge_attributes(workflow_graph, {(0, 1): {'weight': 4.2, 'label': 'edge from task 1 to task 2'}})" + "# nx.set_edge_attributes(workflow_graph, {(0, 1): {'weight': 4.2, 'label':\n", + "# 'edge from task 1 to task 2'}})" ] }, { @@ -9105,7 +9114,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.0rc1" + "version": "3.11.0" } }, "nbformat": 4,