diff --git a/README.md b/README.md index 5d2f943..5ded7c6 100644 --- a/README.md +++ b/README.md @@ -5,9 +5,15 @@ `dsjobs` is a library that simplifies the process of submitting, running, and monitoring [TAPIS v2 / AgavePy](https://agavepy.readthedocs.io/en/latest/index.html) jobs on [DesignSafe](https://designsafe-ci.org) via [Jupyter Notebooks](https://jupyter.designsafe-ci.org). +## Features + +* Simplified TAPIS v2 Calls: No need to fiddle with complex API requests. `dsjobs` abstracts away the complexities. + +* Seamless Integration with DesignSafe Jupyter Notebooks: Launch DesignSafe applications directly from the Jupyter environment. + ## Installation -Install `dsjobs` via pip (**coming soon**) +Install `dsjobs` via pip ```shell pip3 install dsjobs @@ -21,6 +27,8 @@ pip install git+https://github.com/DesignSafe-CI/dsjobs.git --quiet ## Example usage: +* [Jupyter Notebook Templates](example-notebooks/template-mpm-run.ipynb) using DSJobs. + On [DesignSafe Jupyter](https://jupyter.designsafe-ci.org/): Install the latest version of `dsjobs` and restart the kernel (Kernel >> Restart Kernel): @@ -29,7 +37,7 @@ Install the latest version of `dsjobs` and restart the kernel (Kernel >> Restart # Remove any previous installations !pip uninstall dsjobs -y # Install -!pip install git+https://github.com/DesignSafe-CI/dsjobs.git --quiet +!pip install dsjobs --quiet ``` * Import `dsjobs` library @@ -42,31 +50,14 @@ import dsjobs as ds dir(ds) ``` -### Job management +## Documentation -* Monitor job status -```python -ds.get_status(ag, job["id"]) -``` +To generate API docs: -* Get runtime information of a job ``` -ds.get_runtime(ag, job["id"]) +pdoc --html --output-dir docs dsjobs --force ``` -### Directory access - -* Access DesignSafe path URI: -```python -input_uri = ds.get_ds_path_uri(ag, '/MyData/ + + + + + +dsjobs.dir API documentation + + + + + + + + + + + +
+
+
+

Module dsjobs.dir

+
+
+
+ +Expand source code + +
import os
+
+
+def get_ds_path_uri(ag, path):
+    """
+    Given a path on DesignSafe, determine the correct input URI.
+
+    Args:
+        ag (object): Agave object to fetch profiles or metadata.
+        path (str): The directory path.
+
+    Returns:
+        str: The corresponding input URI.
+
+    Raises:
+        ValueError: If no matching directory pattern is found.
+    """
+
+    # If any of the following directory patterns are found in the path,
+    # process them accordingly.
+    directory_patterns = [
+        ("jupyter/MyData", "designsafe.storage.default", True),
+        ("jupyter/mydata", "designsafe.storage.default", True),
+        ("jupyter/CommunityData", "designsafe.storage.community", False),
+        ("/MyData", "designsafe.storage.default", True),
+        ("/mydata", "designsafe.storage.default", True),
+    ]
+
+    for pattern, storage, use_username in directory_patterns:
+        if pattern in path:
+            path = path.split(pattern).pop()
+            input_dir = ag.profiles.get()["username"] + path if use_username else path
+            input_uri = f"agave://{storage}/{input_dir}"
+            return input_uri.replace(" ", "%20")
+
+    project_patterns = [
+        ("jupyter/MyProjects", "project-"),
+        ("jupyter/projects", "project-"),
+    ]
+
+    for pattern, prefix in project_patterns:
+        if pattern in path:
+            path = path.split(pattern + "/").pop()
+            project_id = path.split("/")[0]
+            query = {"value.projectId": str(project_id)}
+            path = path.split(project_id).pop()
+            project_uuid = ag.meta.listMetadata(q=str(query))[0]["uuid"]
+            input_uri = f"agave://{prefix}{project_uuid}{path}"
+            return input_uri.replace(" ", "%20")
+
+    raise ValueError(f"No matching directory pattern found for: {path}")
+
+
+
+
+
+
+
+

Functions

+
+
+def get_ds_path_uri(ag, path) +
+
+

Given a path on DesignSafe, determine the correct input URI.

+

Args

+
+
ag : object
+
Agave object to fetch profiles or metadata.
+
path : str
+
The directory path.
+
+

Returns

+
+
str
+
The corresponding input URI.
+
+

Raises

+
+
ValueError
+
If no matching directory pattern is found.
+
+
+ +Expand source code + +
def get_ds_path_uri(ag, path):
+    """
+    Given a path on DesignSafe, determine the correct input URI.
+
+    Args:
+        ag (object): Agave object to fetch profiles or metadata.
+        path (str): The directory path.
+
+    Returns:
+        str: The corresponding input URI.
+
+    Raises:
+        ValueError: If no matching directory pattern is found.
+    """
+
+    # If any of the following directory patterns are found in the path,
+    # process them accordingly.
+    directory_patterns = [
+        ("jupyter/MyData", "designsafe.storage.default", True),
+        ("jupyter/mydata", "designsafe.storage.default", True),
+        ("jupyter/CommunityData", "designsafe.storage.community", False),
+        ("/MyData", "designsafe.storage.default", True),
+        ("/mydata", "designsafe.storage.default", True),
+    ]
+
+    for pattern, storage, use_username in directory_patterns:
+        if pattern in path:
+            path = path.split(pattern).pop()
+            input_dir = ag.profiles.get()["username"] + path if use_username else path
+            input_uri = f"agave://{storage}/{input_dir}"
+            return input_uri.replace(" ", "%20")
+
+    project_patterns = [
+        ("jupyter/MyProjects", "project-"),
+        ("jupyter/projects", "project-"),
+    ]
+
+    for pattern, prefix in project_patterns:
+        if pattern in path:
+            path = path.split(pattern + "/").pop()
+            project_id = path.split("/")[0]
+            query = {"value.projectId": str(project_id)}
+            path = path.split(project_id).pop()
+            project_uuid = ag.meta.listMetadata(q=str(query))[0]["uuid"]
+            input_uri = f"agave://{prefix}{project_uuid}{path}"
+            return input_uri.replace(" ", "%20")
+
+    raise ValueError(f"No matching directory pattern found for: {path}")
+
+
+
+
+
+
+
+ +
+ + + \ No newline at end of file diff --git a/docs/dsjobs/index.html b/docs/dsjobs/index.html new file mode 100644 index 0000000..3685c53 --- /dev/null +++ b/docs/dsjobs/index.html @@ -0,0 +1,105 @@ + + + + + + +dsjobs API documentation + + + + + + + + + + + +
+
+
+

Package dsjobs

+
+
+

dsjobs is a library that simplifies the process of submitting, running, and monitoring TAPIS v2 / AgavePy jobs on DesignSafe via Jupyter Notebooks.

+

Features

+
    +
  • +

    Simplified TAPIS v2 Calls: No need to fiddle with complex API requests. dsjobs abstracts away the complexities.

    +
    • +
  • +

    Seamless Integration with DesignSafe Jupyter Notebooks: Launch DesignSafe applications directly from the Jupyter environment.

    +
    • +
+

Installation

+
pip3 install dsjobs
+
+
+ +Expand source code + +
"""
+`dsjobs` is a library that simplifies the process of submitting, running, and monitoring [TAPIS v2 / AgavePy](https://agavepy.readthedocs.io/en/latest/index.html) jobs on [DesignSafe](https://designsafe-ci.org) via [Jupyter Notebooks](https://jupyter.designsafe-ci.org).
+
+
+## Features
+
+* Simplified TAPIS v2 Calls: No need to fiddle with complex API requests. `dsjobs` abstracts away the complexities.
+
+* Seamless Integration with DesignSafe Jupyter Notebooks: Launch DesignSafe applications directly from the Jupyter environment.
+
+## Installation
+
+```shell
+pip3 install dsjobs
+```
+
+"""
+from .dir import get_ds_path_uri
+from .jobs import get_status, runtime_summary, generate_job_info, get_archive_path
+
+
+
+

Sub-modules

+
+
dsjobs.dir
+
+
+
+
dsjobs.jobs
+
+
+
+
+
+
+
+
+
+
+
+
+ +
+ + + \ No newline at end of file diff --git a/docs/dsjobs/jobs.html b/docs/dsjobs/jobs.html new file mode 100644 index 0000000..55ad122 --- /dev/null +++ b/docs/dsjobs/jobs.html @@ -0,0 +1,576 @@ + + + + + + +dsjobs.jobs API documentation + + + + + + + + + + + +
+
+
+

Module dsjobs.jobs

+
+
+
+ +Expand source code + +
import time
+from datetime import datetime, timedelta, timezone
+from tqdm import tqdm
+import logging
+
+# Configuring the logging system
+# logging.basicConfig(
+#     level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
+# )
+
+
+def get_status(ag, job_id, time_lapse=15):
+    """
+    Retrieves and monitors the status of a job from Agave.
+
+    This function initially waits for the job to start, displaying its progress using
+    a tqdm progress bar. Once the job starts, it monitors the job's status up to
+    a maximum duration specified by the job's "maxHours". If the job completes or fails
+    before reaching this maximum duration, it returns the job's final status.
+
+    Args:
+      ag (object): The Agave job object used to interact with the job.
+      job_id (str): The unique identifier of the job to monitor.
+      time_lapse (int, optional): Time interval, in seconds, to wait between status
+        checks. Defaults to 15 seconds.
+
+    Returns:
+      str: The final status of the job. Typical values include "FINISHED", "FAILED",
+           and "STOPPED".
+
+    Raises:
+      No exceptions are explicitly raised, but potential exceptions raised by the Agave
+      job object or other called functions/methods will propagate.
+    """
+
+    previous_status = None
+    # Initially check if the job is already running
+    status = ag.jobs.getStatus(jobId=job_id)["status"]
+
+    job_details = ag.jobs.get(jobId=job_id)
+    max_hours = job_details["maxHours"]
+
+    # Using tqdm to provide visual feedback while waiting for job to start
+    with tqdm(desc="Waiting for job to start", dynamic_ncols=True) as pbar:
+        while status not in ["RUNNING", "FINISHED", "FAILED", "STOPPED"]:
+            time.sleep(time_lapse)
+            status = ag.jobs.getStatus(jobId=job_id)["status"]
+            pbar.update(1)
+            pbar.set_postfix_str(f"Status: {status}")
+
+    # Once the job is running, monitor it for up to maxHours
+    max_iterations = int(max_hours * 3600 // time_lapse)
+
+    # Using tqdm for progress bar
+    for _ in tqdm(range(max_iterations), desc="Monitoring job", ncols=100):
+        status = ag.jobs.getStatus(jobId=job_id)["status"]
+
+        # Print status if it has changed
+        if status != previous_status:
+            tqdm.write(f"\tStatus: {status}")
+            previous_status = status
+
+        # Break the loop if job reaches one of these statuses
+        if status in ["FINISHED", "FAILED", "STOPPED"]:
+            break
+
+        time.sleep(time_lapse)
+    else:
+        # This block will execute if the for loop completes without a 'break'
+        logging.warn("Warning: Maximum monitoring time reached!")
+
+    return status
+
+
+def runtime_summary(ag, job_id):
+    """Get the runtime of a job.
+
+    Args:
+        ag (object): The Agave object that has the job details.
+        job_id (str): The ID of the job for which the runtime needs to be determined.
+
+    Returns:
+        None: This function doesn't return a value, but it prints the runtime details.
+
+    """
+
+    print("Runtime Summary")
+    print("---------------")
+
+    job_history = ag.jobs.getHistory(jobId=job_id)
+    total_time = job_history[-1]["created"] - job_history[0]["created"]
+
+    status_times = {}
+
+    for i in range(
+        len(job_history) - 1
+    ):  # To avoid index out of range error in `job_history[i+1]`
+        current_status = job_history[i]["status"]
+        elapsed_time = job_history[i + 1]["created"] - job_history[i]["created"]
+
+        # Aggregate times for each status
+        if current_status in status_times:
+            status_times[current_status] += elapsed_time
+        else:
+            status_times[current_status] = elapsed_time
+
+    # Determine the max width of status names for alignment
+    max_status_width = max(len(status) for status in status_times.keys())
+
+    # Print the aggregated times for each unique status in a table format
+    for status, time in status_times.items():
+        print(f"{status.upper():<{max_status_width + 2}} time: {time}")
+
+    print(f"{'TOTAL':<{max_status_width + 2}} time: {total_time}")
+    print("---------------")
+
+
+def generate_job_info(
+    ag,
+    appid: str,
+    jobname: str = "dsjob",
+    queue: str = "skx-dev",
+    nnodes: int = 1,
+    nprocessors: int = 1,
+    runtime: str = "00:10:00",
+    inputs=None,
+    parameters=None,
+) -> dict:
+    """Generate a job information dictionary based on provided arguments.
+
+    Args:
+        ag (object): The Agave object to interact with the platform.
+        appid (str): The application ID for the job.
+        jobname (str, optional): The name of the job. Defaults to 'dsjob'.
+        queue (str, optional): The batch queue name. Defaults to 'skx-dev'.
+        nnodes (int, optional): The number of nodes required. Defaults to 1.
+        nprocessors (int, optional): The number of processors per node. Defaults to 1.
+        runtime (str, optional): The maximum runtime in the format 'HH:MM:SS'. Defaults to '00:10:00'.
+        inputs (dict, optional): The inputs for the job. Defaults to None.
+        parameters (dict, optional): The parameters for the job. Defaults to None.
+
+    Returns:
+        dict: A dictionary containing the job information.
+
+    Raises:
+        ValueError: If the provided appid is not valid.
+    """
+
+    try:
+        app = ag.apps.get(appId=appid)
+    except Exception:
+        raise ValueError(f"Invalid app ID: {appid}")
+
+    job_info = {
+        "appId": appid,
+        "name": jobname,
+        "batchQueue": queue,
+        "nodeCount": nnodes,
+        "processorsPerNode": nprocessors,
+        "memoryPerNode": "1",
+        "maxRunTime": runtime,
+        "archive": True,
+        "inputs": inputs,
+        "parameters": parameters,
+    }
+
+    return job_info
+
+
+def get_archive_path(ag, job_id):
+    """
+    Get the archive path for a given job ID and modifies the user directory
+    to '/home/jupyter/MyData'.
+
+    Args:
+        ag (object): The Agave object to interact with the platform.
+        job_id (str): The job ID to retrieve the archive path for.
+
+    Returns:
+        str: The modified archive path.
+
+    Raises:
+        ValueError: If the archivePath format is unexpected.
+    """
+
+    # Fetch the job info.
+    job_info = ag.jobs.get(jobId=job_id)
+
+    # Try to split the archive path to extract the user.
+    try:
+        user, _ = job_info.archivePath.split("/", 1)
+    except ValueError:
+        raise ValueError(f"Unexpected archivePath format for jobId={job_id}")
+
+    # Construct the new path.
+    new_path = job_info.archivePath.replace(user, "/home/jupyter/MyData")
+
+    return new_path
+
+
+
+
+
+
+
+

Functions

+
+
+def generate_job_info(ag, appid: str, jobname: str = 'dsjob', queue: str = 'skx-dev', nnodes: int = 1, nprocessors: int = 1, runtime: str = '00:10:00', inputs=None, parameters=None) ‑> dict +
+
+

Generate a job information dictionary based on provided arguments.

+

Args

+
+
ag : object
+
The Agave object to interact with the platform.
+
appid : str
+
The application ID for the job.
+
jobname : str, optional
+
The name of the job. Defaults to 'dsjob'.
+
queue : str, optional
+
The batch queue name. Defaults to 'skx-dev'.
+
nnodes : int, optional
+
The number of nodes required. Defaults to 1.
+
nprocessors : int, optional
+
The number of processors per node. Defaults to 1.
+
runtime : str, optional
+
The maximum runtime in the format 'HH:MM:SS'. Defaults to '00:10:00'.
+
inputs : dict, optional
+
The inputs for the job. Defaults to None.
+
parameters : dict, optional
+
The parameters for the job. Defaults to None.
+
+

Returns

+
+
dict
+
A dictionary containing the job information.
+
+

Raises

+
+
ValueError
+
If the provided appid is not valid.
+
+
+ +Expand source code + +
def generate_job_info(
+    ag,
+    appid: str,
+    jobname: str = "dsjob",
+    queue: str = "skx-dev",
+    nnodes: int = 1,
+    nprocessors: int = 1,
+    runtime: str = "00:10:00",
+    inputs=None,
+    parameters=None,
+) -> dict:
+    """Generate a job information dictionary based on provided arguments.
+
+    Args:
+        ag (object): The Agave object to interact with the platform.
+        appid (str): The application ID for the job.
+        jobname (str, optional): The name of the job. Defaults to 'dsjob'.
+        queue (str, optional): The batch queue name. Defaults to 'skx-dev'.
+        nnodes (int, optional): The number of nodes required. Defaults to 1.
+        nprocessors (int, optional): The number of processors per node. Defaults to 1.
+        runtime (str, optional): The maximum runtime in the format 'HH:MM:SS'. Defaults to '00:10:00'.
+        inputs (dict, optional): The inputs for the job. Defaults to None.
+        parameters (dict, optional): The parameters for the job. Defaults to None.
+
+    Returns:
+        dict: A dictionary containing the job information.
+
+    Raises:
+        ValueError: If the provided appid is not valid.
+    """
+
+    try:
+        app = ag.apps.get(appId=appid)
+    except Exception:
+        raise ValueError(f"Invalid app ID: {appid}")
+
+    job_info = {
+        "appId": appid,
+        "name": jobname,
+        "batchQueue": queue,
+        "nodeCount": nnodes,
+        "processorsPerNode": nprocessors,
+        "memoryPerNode": "1",
+        "maxRunTime": runtime,
+        "archive": True,
+        "inputs": inputs,
+        "parameters": parameters,
+    }
+
+    return job_info
+
+
+
+def get_archive_path(ag, job_id) +
+
+

Get the archive path for a given job ID and modifies the user directory +to '/home/jupyter/MyData'.

+

Args

+
+
ag : object
+
The Agave object to interact with the platform.
+
job_id : str
+
The job ID to retrieve the archive path for.
+
+

Returns

+
+
str
+
The modified archive path.
+
+

Raises

+
+
ValueError
+
If the archivePath format is unexpected.
+
+
+ +Expand source code + +
def get_archive_path(ag, job_id):
+    """
+    Get the archive path for a given job ID and modifies the user directory
+    to '/home/jupyter/MyData'.
+
+    Args:
+        ag (object): The Agave object to interact with the platform.
+        job_id (str): The job ID to retrieve the archive path for.
+
+    Returns:
+        str: The modified archive path.
+
+    Raises:
+        ValueError: If the archivePath format is unexpected.
+    """
+
+    # Fetch the job info.
+    job_info = ag.jobs.get(jobId=job_id)
+
+    # Try to split the archive path to extract the user.
+    try:
+        user, _ = job_info.archivePath.split("/", 1)
+    except ValueError:
+        raise ValueError(f"Unexpected archivePath format for jobId={job_id}")
+
+    # Construct the new path.
+    new_path = job_info.archivePath.replace(user, "/home/jupyter/MyData")
+
+    return new_path
+
+
+
+def get_status(ag, job_id, time_lapse=15) +
+
+

Retrieves and monitors the status of a job from Agave.

+

This function initially waits for the job to start, displaying its progress using +a tqdm progress bar. Once the job starts, it monitors the job's status up to +a maximum duration specified by the job's "maxHours". If the job completes or fails +before reaching this maximum duration, it returns the job's final status.

+

Args

+
+
ag : object
+
The Agave job object used to interact with the job.
+
job_id : str
+
The unique identifier of the job to monitor.
+
time_lapse : int, optional
+
Time interval, in seconds, to wait between status +checks. Defaults to 15 seconds.
+
+

Returns

+
+
str
+
The final status of the job. Typical values include "FINISHED", "FAILED", +and "STOPPED".
+
+

Raises

+

No exceptions are explicitly raised, but potential exceptions raised by the Agave +job object or other called functions/methods will propagate.

+
+ +Expand source code + +
def get_status(ag, job_id, time_lapse=15):
+    """
+    Retrieves and monitors the status of a job from Agave.
+
+    This function initially waits for the job to start, displaying its progress using
+    a tqdm progress bar. Once the job starts, it monitors the job's status up to
+    a maximum duration specified by the job's "maxHours". If the job completes or fails
+    before reaching this maximum duration, it returns the job's final status.
+
+    Args:
+      ag (object): The Agave job object used to interact with the job.
+      job_id (str): The unique identifier of the job to monitor.
+      time_lapse (int, optional): Time interval, in seconds, to wait between status
+        checks. Defaults to 15 seconds.
+
+    Returns:
+      str: The final status of the job. Typical values include "FINISHED", "FAILED",
+           and "STOPPED".
+
+    Raises:
+      No exceptions are explicitly raised, but potential exceptions raised by the Agave
+      job object or other called functions/methods will propagate.
+    """
+
+    previous_status = None
+    # Initially check if the job is already running
+    status = ag.jobs.getStatus(jobId=job_id)["status"]
+
+    job_details = ag.jobs.get(jobId=job_id)
+    max_hours = job_details["maxHours"]
+
+    # Using tqdm to provide visual feedback while waiting for job to start
+    with tqdm(desc="Waiting for job to start", dynamic_ncols=True) as pbar:
+        while status not in ["RUNNING", "FINISHED", "FAILED", "STOPPED"]:
+            time.sleep(time_lapse)
+            status = ag.jobs.getStatus(jobId=job_id)["status"]
+            pbar.update(1)
+            pbar.set_postfix_str(f"Status: {status}")
+
+    # Once the job is running, monitor it for up to maxHours
+    max_iterations = int(max_hours * 3600 // time_lapse)
+
+    # Using tqdm for progress bar
+    for _ in tqdm(range(max_iterations), desc="Monitoring job", ncols=100):
+        status = ag.jobs.getStatus(jobId=job_id)["status"]
+
+        # Print status if it has changed
+        if status != previous_status:
+            tqdm.write(f"\tStatus: {status}")
+            previous_status = status
+
+        # Break the loop if job reaches one of these statuses
+        if status in ["FINISHED", "FAILED", "STOPPED"]:
+            break
+
+        time.sleep(time_lapse)
+    else:
+        # This block will execute if the for loop completes without a 'break'
+        logging.warn("Warning: Maximum monitoring time reached!")
+
+    return status
+
+
+
+def runtime_summary(ag, job_id) +
+
+

Get the runtime of a job.

+

Args

+
+
ag : object
+
The Agave object that has the job details.
+
job_id : str
+
The ID of the job for which the runtime needs to be determined.
+
+

Returns

+
+
None
+
This function doesn't return a value, but it prints the runtime details.
+
+
+ +Expand source code + +
def runtime_summary(ag, job_id):
+    """Get the runtime of a job.
+
+    Args:
+        ag (object): The Agave object that has the job details.
+        job_id (str): The ID of the job for which the runtime needs to be determined.
+
+    Returns:
+        None: This function doesn't return a value, but it prints the runtime details.
+
+    """
+
+    print("Runtime Summary")
+    print("---------------")
+
+    job_history = ag.jobs.getHistory(jobId=job_id)
+    total_time = job_history[-1]["created"] - job_history[0]["created"]
+
+    status_times = {}
+
+    for i in range(
+        len(job_history) - 1
+    ):  # To avoid index out of range error in `job_history[i+1]`
+        current_status = job_history[i]["status"]
+        elapsed_time = job_history[i + 1]["created"] - job_history[i]["created"]
+
+        # Aggregate times for each status
+        if current_status in status_times:
+            status_times[current_status] += elapsed_time
+        else:
+            status_times[current_status] = elapsed_time
+
+    # Determine the max width of status names for alignment
+    max_status_width = max(len(status) for status in status_times.keys())
+
+    # Print the aggregated times for each unique status in a table format
+    for status, time in status_times.items():
+        print(f"{status.upper():<{max_status_width + 2}} time: {time}")
+
+    print(f"{'TOTAL':<{max_status_width + 2}} time: {total_time}")
+    print("---------------")
+
+
+
+
+
+
+
+ +
+ + + \ No newline at end of file diff --git a/dsjobs/__init__.py b/dsjobs/__init__.py index 3ac84b2..637af6e 100644 --- a/dsjobs/__init__.py +++ b/dsjobs/__init__.py @@ -1,2 +1,19 @@ +""" +`dsjobs` is a library that simplifies the process of submitting, running, and monitoring [TAPIS v2 / AgavePy](https://agavepy.readthedocs.io/en/latest/index.html) jobs on [DesignSafe](https://designsafe-ci.org) via [Jupyter Notebooks](https://jupyter.designsafe-ci.org). + + +## Features + +* Simplified TAPIS v2 Calls: No need to fiddle with complex API requests. `dsjobs` abstracts away the complexities. + +* Seamless Integration with DesignSafe Jupyter Notebooks: Launch DesignSafe applications directly from the Jupyter environment. + +## Installation + +```shell +pip3 install dsjobs +``` + +""" from .dir import get_ds_path_uri from .jobs import get_status, runtime_summary, generate_job_info, get_archive_path diff --git a/example-notebooks/template-mpm-run.ipynb b/example-notebooks/template-mpm-run.ipynb index 922bd21..3739b8e 100644 --- a/example-notebooks/template-mpm-run.ipynb +++ b/example-notebooks/template-mpm-run.ipynb @@ -18,27 +18,7 @@ }, { "cell_type": "code", - "execution_count": 1, - "id": "e70c29e6-e0eb-48cb-914e-3a228faba66d", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found existing installation: dsjobs 0.1.0\n", - "Uninstalling dsjobs-0.1.0:\n", - " Successfully uninstalled dsjobs-0.1.0\n" - ] - } - ], - "source": [ - "!pip uninstall dsjobs -y" - ] - }, - { - "cell_type": "code", - "execution_count": 2, + "execution_count": 21, "id": "b7fa3e9a-7bf3-441c-917f-fb57a94ee017", "metadata": {}, "outputs": [ @@ -53,7 +33,7 @@ } ], "source": [ - "!pip install git+https://github.com/DesignSafe-CI/dsjobs.git --quiet" + "!pip install dsjobs --quiet" ] }, { @@ -68,7 +48,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 22, "id": "8593c08c-c96a-4a66-9b52-80b8b5c27e44", "metadata": { "tags": [ @@ -97,7 +77,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 23, "id": "91b811a4-45d9-4223-a145-c0f4e393af66", "metadata": {}, "outputs": [], @@ -118,7 +98,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 24, "id": "0153c5cc-f4b9-460b-b4c6-d92eb8c6ede5", "metadata": {}, "outputs": [ @@ -177,7 +157,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 25, "id": "4e6dc010-d821-45d5-805b-84620363f468", "metadata": {}, "outputs": [], @@ -187,7 +167,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 26, "id": "06dd17dc-2540-46b3-9036-3245a9b6cfae", "metadata": {}, "outputs": [ @@ -206,6 +186,20 @@ "\tStatus: RUNNING\n" ] }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Monitoring job: 10%|████▊ | 4/40 [01:00<09:04, 15.13s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\tStatus: ARCHIVING\n" + ] + }, { "name": "stderr", "output_type": "stream", @@ -233,7 +227,7 @@ "'FINISHED'" ] }, - "execution_count": 7, + "execution_count": 26, "metadata": {}, "output_type": "execute_result" } @@ -244,7 +238,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 27, "id": "614f3568-d290-44d9-896d-608375293a58", "metadata": {}, "outputs": [ @@ -256,15 +250,15 @@ "---------------\n", "PENDING time: 0:00:00\n", "PROCESSING_INPUTS time: 0:00:03\n", - "STAGING_INPUTS time: 0:00:04\n", + "STAGING_INPUTS time: 0:00:05\n", "STAGED time: 0:00:00\n", - "STAGING_JOB time: 0:00:04\n", + "STAGING_JOB time: 0:00:03\n", "SUBMITTING time: 0:00:06\n", "QUEUED time: 0:00:05\n", "RUNNING time: 0:01:09\n", "CLEANING_UP time: 0:00:00\n", - "ARCHIVING time: 0:00:09\n", - "TOTAL time: 0:01:40\n", + "ARCHIVING time: 0:00:08\n", + "TOTAL time: 0:01:39\n", "---------------\n" ] } @@ -273,6 +267,16 @@ "ds.runtime_summary(ag, job[\"id\"])" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "5e2c642e-88c3-42be-acb6-2966cc31f90c", + "metadata": {}, + "outputs": [], + "source": [ + "ag.jobs.get(jobId=job[\"id\"])[\"lastStatusMessage\"]" + ] + }, { "cell_type": "markdown", "id": "798bd1bc-44e9-4e2f-b999-0ecf3ee769a4", @@ -283,17 +287,17 @@ }, { "cell_type": "code", - "execution_count": 9, - "id": "209aae22-98fc-454c-bbd4-1946dfb4b5be", + "execution_count": 20, + "id": "daf7f33a-594c-4a69-b2d3-bc39a434d154", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "'/home/jupyter/MyData/archive/jobs/job-cdd40c86-e9a1-43bc-8b68-934826e513aa-007'" + "'/home/jupyter/MyData/archive/jobs/job-8d70a2b2-9295-4233-bade-9cc1834e9603-007'" ] }, - "execution_count": 9, + "execution_count": 20, "metadata": {}, "output_type": "execute_result" } diff --git a/poetry.lock b/poetry.lock index dfab8a1..29d66e5 100644 --- a/poetry.lock +++ b/poetry.lock @@ -53,6 +53,21 @@ six = ">=1.12.0" [package.extras] test = ["astroid", "pytest"] +[[package]] +name = "astunparse" +version = "1.6.3" +description = "An AST unparser for Python" +optional = false +python-versions = "*" +files = [ + {file = "astunparse-1.6.3-py2.py3-none-any.whl", hash = "sha256:c2652417f2c8b5bb325c885ae329bdf3f86424075c4fd1a128674bc6fba4b8e8"}, + {file = "astunparse-1.6.3.tar.gz", hash = "sha256:5ad93a8456f0d084c3456d059fd9a92cce667963232cbf763eac3bc5b7940872"}, +] + +[package.dependencies] +six = ">=1.6.1,<2.0" +wheel = ">=0.23.0,<1.0" + [[package]] name = "backcall" version = "0.2.0" @@ -556,6 +571,26 @@ files = [ {file = "pathspec-0.11.2.tar.gz", hash = "sha256:e0d8d0ac2f12da61956eb2306b69f9469b42f4deb0f3cb6ed47b9cce9996ced3"}, ] +[[package]] +name = "pdoc" +version = "14.1.0" +description = "API Documentation for Python Projects" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pdoc-14.1.0-py3-none-any.whl", hash = "sha256:e8869dffe21296b3bd5545b28e7f07cae0656082aca43f8915323187e541b126"}, + {file = "pdoc-14.1.0.tar.gz", hash = "sha256:3a0bd921a05c39a82b1505089eb6dc99d857b71b856aa60d1aca4d9086d0e18c"}, +] + +[package.dependencies] +astunparse = {version = "*", markers = "python_version < \"3.9\""} +Jinja2 = ">=2.11.0" +MarkupSafe = "*" +pygments = ">=2.12.0" + +[package.extras] +dev = ["black", "hypothesis", "mypy", "pygments (>=2.14.0)", "pytest", "pytest-cov", "pytest-timeout", "ruff", "tox", "types-pygments"] + [[package]] name = "pexpect" version = "4.8.0" @@ -886,7 +921,21 @@ files = [ [package.dependencies] six = "*" +[[package]] +name = "wheel" +version = "0.41.3" +description = "A built-package format for Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "wheel-0.41.3-py3-none-any.whl", hash = "sha256:488609bc63a29322326e05560731bf7bfea8e48ad646e1f5e40d366607de0942"}, + {file = "wheel-0.41.3.tar.gz", hash = "sha256:4d4987ce51a49370ea65c0bfd2234e8ce80a12780820d9dc462597a6e60d0841"}, +] + +[package.extras] +test = ["pytest (>=6.0.0)", "setuptools (>=65)"] + [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "aed42b69138b93f78d650323abe94dd11132d220478c55908b587e75decbbc35" +content-hash = "ab0a34aed23b38f335bf7b4a2920e55dab6acad2a94b5325dc889b6b94b9483d" diff --git a/pyproject.toml b/pyproject.toml index 7be9865..82763b4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ exceptiongroup = "^1.1.3" [tool.poetry.group.dev.dependencies] pytest = "^7.4.2" black = {extras = ["jupyter"], version = "^23.10.0"} +pdoc = "^14.1.0" [build-system] requires = ["poetry-core"]