Merge pull request #2 from FAIRmat-NFDI/add-gh-workflows

added workflows

.github/workflows/actions.yml

@@ -0,0 +1,56 @@
+name: install-and-test-workflow
+on: [push]
+jobs:
+  install-and-test:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.9
+    - name: Install dependencies
+      run: |
+        pip install --upgrade pip
+        pip install '.[dev]' --index-url https://gitlab.mpcdf.mpg.de/api/v4/projects/2187/packages/pypi/simple
+        pip install coverage coveralls
+    - name: Test with pytest
+      run: |
+        python -m coverage run -m pytest -sv
+    - name: Submit to coveralls
+      continue-on-error: true
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+        coveralls --service=github
+  build-and-install:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.9
+    - name: Build the package
+      run: |
+        pip install --upgrade pip
+        pip install build
+        python -m build --sdist
+    - name: Install the package
+      run: |
+        pip install dist/*.tar.gz --index-url https://gitlab.mpcdf.mpg.de/api/v4/projects/2187/packages/pypi/simple
+  ruff-linting:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: chartboost/ruff-action@v1
+        with:
+          args: "check ."
+  # to enable auto-formatting check, uncomment the following lines below
+  # ruff-formatting:
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #     - uses: chartboost/ruff-action@v1
+  #       with:
+  #         args: "format . --check"

.github/workflows/mkdocs-deploy.yml

@@ -0,0 +1,24 @@
+name: Deploy MkDocs Site
+
+on:
+  push:
+    branches:
+      - main # Triggers deployment on push to the main branch
+
+permissions:
+  contents: write
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Deploy docs
+        uses: mhausenblas/mkdocs-deploy-gh-pages@master
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          CONFIG_FILE: mkdocs.yml
+          REQUIREMENTS: requirements_docs.txt

docs/how_to/use_api_functions.md

@@ -342,7 +342,7 @@ For example:
 
 ```python
 metadata_new = {'upload_name': 'Test Upload', 'comment': 'This is a test upload...'}
-edit_upload_metadata(upload_id, url='test', **metadata_new)
+edit_upload_metadata(upload_id, url='test', upload_metadata=metadata_new)
 ```
 
 ??? success "output"
@@ -742,7 +742,7 @@ The returned `dataset_id` can then be used to add individual entries (or all ent
 
 ```python
 metadata_new = {'dataset_id': dataset_id}
-edit_upload_metadata(upload_id, url='test', **metadata_new)
+edit_upload_metadata(upload_id, url='test', upload_metadata=metadata_new)
 ```
 
 ??? success "output"
@@ -828,7 +828,7 @@ Alternatively, you can search for datasets, e.g., by `user_id` or `dataset_name`
 
 ```python
 my_datasets = retrieve_datasets(
-    user_id=nomad_user_me.user_id, url='test', max_datasets=20
+    dataset_params={'user_id': nomad_user_me.user_id, 'max_datasets': 20}, url='test'
 )
 pprint(my_datasets)
 ```
@@ -851,7 +851,7 @@ To get the list of entries contained within a dataset, use `query_entries()`:
 
 
 ```python
-dataset_entries = query_entries(dataset_id=dataset_id, url='test')
+dataset_entries = query_entries(query_params={'dataset_id': dataset_id}, url='test')
 for entry in dataset_entries:
     pprint(f'entry_id={entry.entry_id}, upload_id={entry.upload_id}')
 ```

src/nomad_utility_workflows/utils/__init__.py

@@ -1 +1,2 @@
-from nomad_utility_workflows.utils.workflows import build_nomad_workflow
+# from nomad_utility_workflows.utils.workflows import build_nomad_workflow
+

src/nomad_utility_workflows/utils/core.py

@@ -1,6 +1,6 @@
 import json
 import logging
-from typing import Any
+from typing import Any, TypedDict
 
 import requests
 from cachetools.func import ttl_cache
@@ -50,12 +50,32 @@ def get_authentication_token(
     return response.json().get('access_token')
 
 
+class RequestOptions(TypedDict, total=False):
+    """_summary_
+
+    Args:
+        TypedDict (_type_): _description_
+        total (bool, optional): _description_. Defaults to False.
+    """
+
+    section: str
+    url: str = None
+    timeout_in_sec: int = TIMEOUT_IN_SEC
+    headers: dict = None
+    with_authentication: bool = False
+
+
+default_request_options = {
+    'section': None,
+    'url': None,
+    'timeout_in_sec': TIMEOUT_IN_SEC,
+    'headers': None,
+    'with_authentication': False,
+}
+
+
 def get_nomad_request(
-    section: str,
-    url: str = None,
-    timeout_in_sec: int = TIMEOUT_IN_SEC,
-    headers: dict = None,
-    with_authentication: bool = False,
+    request_options: RequestOptions = default_request_options.copy(),
     return_json: bool = True,
     accept_field: str = 'application/json',
 ) -> Any:
@@ -76,6 +96,12 @@ def get_nomad_request(
     Returns:
         Any: _description_
     """
+    section = request_options.get('section')
+    url = request_options.get('url')
+    timeout_in_sec = request_options.get('timeout_in_sec')
+    headers = request_options.get('headers')
+    with_authentication = request_options.get('with_authentication')
+
     url_base = get_nomad_url(url)
     url = url_base + f"{'/' if section[0] != '/' else ''}{section}"
     logger.info('Sending get request @ %s', url)
@@ -96,6 +122,14 @@ def get_nomad_request(
 
 
 def get_nomad_url_name(url: str) -> str:
+    """_summary_
+
+    Args:
+        url (str): _description_
+
+    Returns:
+        str: _description_
+    """
     try:
         return url.split('/')[-3]
     except IndexError:
@@ -129,18 +163,42 @@ def get_nomad_url(url: str) -> str:
 
 
 def get_nomad_base_url(url: str) -> str:
+    """_summary_
+
+    Args:
+        url (str): _description_
+
+    Returns:
+        str: _description_
+    """
     return (get_nomad_url(url)).removesuffix('/api/v1')
 
 
 def post_nomad_request(
-    section: str,
-    headers: dict = None,
+    request_options: RequestOptions = default_request_options.copy(),
     data: Any = None,
     json_dict: dict = None,
-    url: str = None,
-    timeout_in_sec: int = TIMEOUT_IN_SEC,
-    with_authentication: bool = False,
 ) -> json:
+    """_summary_
+
+    Args:
+        request_options (RequestOptions, optional): _description_.
+            Defaults to default_request_options.copy().
+        data (Any, optional): _description_. Defaults to None.
+        json_dict (dict, optional): _description_. Defaults to None.
+
+    Raises:
+        ValueError: _description_
+
+    Returns:
+        json: _description_
+    """
+    section = request_options.get('section')
+    url = request_options.get('url')
+    timeout_in_sec = request_options.get('timeout_in_sec')
+    headers = request_options.get('headers')
+    with_authentication = request_options.get('with_authentication')
+
     if headers is None:
         headers = {}
     if with_authentication:
@@ -165,12 +223,26 @@ def post_nomad_request(
 
 
 def delete_nomad_request(
-    section: str,
-    headers: dict = None,
-    url: str = None,
-    timeout_in_sec: int = TIMEOUT_IN_SEC,
-    with_authentication: bool = False,
+    request_options: RequestOptions = default_request_options.copy(),
 ) -> json:
+    """_summary_
+
+    Args:
+        request_options (RequestOptions, optional): _description_.
+            Defaults to default_request_options.copy().
+
+    Raises:
+        ValueError: _description_
+
+    Returns:
+        json: _description_
+    """
+    section = request_options.get('section')
+    url = request_options.get('url')
+    timeout_in_sec = request_options.get('timeout_in_sec')
+    headers = request_options.get('headers')
+    with_authentication = request_options.get('with_authentication')
+
     if headers is None:
         headers = {}
     if with_authentication:

src/nomad_utility_workflows/utils/datasets.py

@@ -1,11 +1,12 @@
 import datetime as dt
 import logging
-from typing import Optional
+from typing import Optional, TypedDict
 
 from marshmallow import Schema, pre_load
 from marshmallow_dataclass import class_schema, dataclass
 
 from nomad_utility_workflows.utils.core import (
+    RequestOptions,
     delete_nomad_request,
     get_nomad_request,
     get_nomad_url,
@@ -25,6 +26,23 @@ def convert_users(self, data, **kwargs):
         return data
 
 
+class DatasetParams(TypedDict, total=False):
+    dataset_id: str
+    dataset_name: str
+    user_id: str
+    page_size: int
+    max_datasets: int
+
+
+default_dataset_params = {
+    'dataset_id': None,
+    'dataset_name': None,
+    'user_id': None,
+    'page_size': 10,
+    'max_datasets': 50,
+}
+
+
 @dataclass(frozen=True)
 class NomadDataset:
     dataset_id: str
@@ -39,21 +57,15 @@ class NomadDataset:
 
 
 def retrieve_datasets(
-    dataset_id: str = None,
-    dataset_name: str = None,
-    user_id: str = None,
-    page_size: int = 10,
-    max_datasets: int = 50,
+    dataset_params: DatasetParams = default_dataset_params.copy(),
     url: str = None,
 ) -> list[NomadDataset]:
     parameters = []
-    if dataset_id:
-        parameters.append(f'dataset_id={dataset_id}')
-    if dataset_name:
-        parameters.append(f'dataset_name={dataset_name}')
-    if user_id:
-        parameters.append(f'user_id={user_id}')
-    parameters.append(f'page_size={page_size}')
+    max_datasets = dataset_params.pop(
+        'max_datasets', default_dataset_params['max_datasets']
+    )
+    for key, value in dataset_params.items():
+        parameters.append(f'{key}={value}')
     url = get_nomad_url(url)
     section = '/datasets/'
     if len(parameters) > 0:
@@ -70,7 +82,9 @@ def retrieve_datasets(
             if page_after_value
             else section
         )
-        response = get_nomad_request(section, headers=headers, url=url)
+        response = get_nomad_request(
+            RequestOptions(section=section, headers=headers, url=url)
+        )
         if len(response['data']) == 0:
             break
         datasets.extend([nomad_entry_schema().load(d) for d in response['data']])
@@ -81,7 +95,7 @@ def retrieve_datasets(
 
 
 def get_dataset_by_id(dataset_id: str, url: str = None) -> NomadDataset:
-    datasets = retrieve_datasets(dataset_id=dataset_id, url=url)
+    datasets = retrieve_datasets(DatasetParams(dataset_id=dataset_id), url=url)
     if len(datasets) != 1:
         raise ValueError(f'Problem retrieving dataset {dataset_id}: {datasets}')
     return datasets[0]
@@ -93,11 +107,13 @@ def create_dataset(dataset_name: str, url: str = None, timeout_in_sec: int = 10)
     logger.info('creating dataset name %s on %s server', dataset_name, url_name)
     json_dict = {'dataset_name': dataset_name}
     response = post_nomad_request(
-        '/datasets/',
-        with_authentication=True,
+        RequestOptions(
+            section='/datasets/',
+            with_authentication=True,
+            url=url,
+            timeout_in_sec=timeout_in_sec,
+        ),
         json_dict=json_dict,
-        url=url,
-        timeout_in_sec=timeout_in_sec,
     )
     return response.get('dataset_id')
 
@@ -107,10 +123,12 @@ def delete_dataset(dataset_id: str, url: str = None, timeout_in_sec: int = 10) -
     url_name = get_nomad_url_name(url)
     logger.info('deleting dataset %s on %s server', dataset_id, url_name)
     response = delete_nomad_request(
-        f'/datasets/{dataset_id}',
-        with_authentication=True,
-        url=url,
-        timeout_in_sec=timeout_in_sec,
+        RequestOptions(
+            section=f'/datasets/{dataset_id}',
+            with_authentication=True,
+            url=url,
+            timeout_in_sec=timeout_in_sec,
+        )
     )
     if response.get('dataset_id'):
         logger.info('successfully deleted dataset %s', dataset_id)