diff --git a/.github/workflows/complete_release.yml b/.github/workflows/complete_release.yml index cdb9fb93..0a0b30e3 100644 --- a/.github/workflows/complete_release.yml +++ b/.github/workflows/complete_release.yml @@ -37,12 +37,15 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: ./.github/actions/setup-bfabricpy + - uses: actions/setup-python@v5 with: python-version: 3.12 + - name: Install nox + run: pip install nox uv - name: Publish documentation run: | set -euxo pipefail git fetch --unshallow origin gh-pages git checkout gh-pages && git pull && git checkout - - uv run mkdocs gh-deploy + nox -s docs + nox -s publish_docs diff --git a/docs/changelog.md b/docs/changelog.md index e08d67ad..1659f84d 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -10,6 +10,21 @@ Versioning currently follows `X.Y.Z` where ## \[Unreleased\] +## \[1.13.10\] - 2024-12-13 + +### Fixed + +- If `bfabricpy.yml` contains a root-level key which is not a dictionary, a correct error message is shown instead of raising an exception. +- A bug introduced while refactoring in `slurm.py` which passed `Path` objects to `subprocess` instead of strings. +- Submitter + - does not use unset `JOB_ID` environment variable anymore. + - does not set unused `STAMP` environment variable anymore. +- Fix `bfabric_save_workflowstep.py` bugs from refactoring. + +### Added + +- Experimental bfabric-cli interface. Please do not use it in production yet as it will need a lot of refinement. + ## \[1.13.9\] - 2024-12-10 From this release onwards, the experimental app runner is not part of the main bfabric package and diff --git a/pyproject.toml b/pyproject.toml index 227aaef1..e80c546c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "bfabric" description = "Python client for the B-Fabric WSDL API" -version = "1.13.9" +version = "1.13.10" license = { text = "GPL-3.0" } authors = [ { name = "Christian Panse", email = "cp@fgcz.ethz.ch" }, @@ -78,6 +78,8 @@ Repository = "https://github.com/fgcz/bfabricPy" "bfabric_slurm_queue_status.py" = "bfabric_scripts.bfabric_slurm_queue_status:main" "bfabric_save_resource_description.py" = "bfabric_scripts.bfabric_save_resource_description:main" +"bfabric-cli" = "bfabric_scripts.cli.__main__:app" + [tool.setuptools.package-data] "*" = ["py.typed"] diff --git a/src/bfabric/config/config_file.py b/src/bfabric/config/config_file.py index 1983ba9a..aeb93728 100644 --- a/src/bfabric/config/config_file.py +++ b/src/bfabric/config/config_file.py @@ -23,13 +23,15 @@ class EnvironmentConfig(BaseModel): @classmethod def gather_config(cls, values: dict[str, Any]) -> dict[str, Any]: """Gathers all configs into the config field.""" + if not isinstance(values, dict): + return values values["config"] = {key: value for key, value in values.items() if key not in ["login", "password"]} return values @model_validator(mode="before") @classmethod def gather_auth(cls, values: dict[str, Any]) -> dict[str, Any]: - if "login" in values: + if isinstance(values, dict) and "login" in values: values["auth"] = BfabricAuth.model_validate(values) return values diff --git a/src/bfabric/wrapper_creator/bfabric_submitter.py b/src/bfabric/wrapper_creator/bfabric_submitter.py index 7ef35c1c..0e258b19 100644 --- a/src/bfabric/wrapper_creator/bfabric_submitter.py +++ b/src/bfabric/wrapper_creator/bfabric_submitter.py @@ -100,17 +100,10 @@ def compose_bash_script(self, configuration=None, configuration_parser=lambda x: #SBATCH -e {stderr_url} #SBATCH -o {stdout_url} #SBATCH --job-name=WU{workunit_id} -#SBATCH --workdir=/home/bfabric +#SBATCH --chdir=/home/bfabric #SBATCH --export=ALL,HOME=/home/bfabric -# Grid Engine Parameters -#$ -q {partition}&{nodelist} -#$ -e {stderr_url} -#$ -o {stdout_url} - - -set -e -set -o pipefail +set -euxo pipefail export EMAIL="{job_notification_emails}" export EXTERNALJOB_ID={external_job_id} @@ -118,7 +111,6 @@ def compose_bash_script(self, configuration=None, configuration_parser=lambda x: export RESSOURCEID_STDOUT_STDERR="{resource_id_stderr} {resource_id_stdout}" export OUTPUT="{output_list}" export WORKUNIT_ID="{workunit_id}" -STAMP=`/bin/date +%Y%m%d%H%M`.$$.$JOB_ID TEMPDIR="/home/bfabric/prx" _OUTPUT=`echo $OUTPUT | cut -d"," -f1` @@ -146,11 +138,11 @@ def compose_bash_script(self, configuration=None, configuration_parser=lambda x: ## interrupt here if you want to do a semi-automatic processing if [ -x /usr/bin/mutt ]; then - cat $0 > $TEMPDIR/$JOB_ID.bash + cat $0 > $TEMPDIR/$EXTERNALJOB_ID.bash (who am i; hostname; uptime; echo $0; pwd; ps;) \ - | mutt -s "JOB_ID=$JOB_ID WORKUNIT_ID=$WORKUNIT_ID EXTERNALJOB_ID=$EXTERNALJOB_ID" $EMAIL \ - -a $TEMPDIR/$JOB_ID.bash $TEMPDIR/config_WU$WORKUNIT_ID.yaml + | mutt -s "WORKUNIT_ID=$WORKUNIT_ID EXTERNALJOB_ID=$EXTERNALJOB_ID" $EMAIL \ + -a $TEMPDIR/$EXTERNALJOB_ID.bash $TEMPDIR/config_WU$WORKUNIT_ID.yaml fi # exit 0 @@ -161,7 +153,7 @@ def compose_bash_script(self, configuration=None, configuration_parser=lambda x: if [ $? -eq 0 ]; then ssh fgcz-r-035.uzh.ch "bfabric_setResourceStatus_available.py $RESSOURCEID_OUTPUT" \ - | mutt -s "JOB_ID=$JOB_ID WORKUNIT_ID=$WORKUNIT_ID EXTERNALJOB_ID=$EXTERNALJOB_ID DONE" $EMAIL + | mutt -s "WORKUNIT_ID=$WORKUNIT_ID EXTERNALJOB_ID=$EXTERNALJOB_ID DONE" $EMAIL bfabric_save_workflowstep.py $WORKUNIT_ID bfabric_setExternalJobStatus_done.py $EXTERNALJOB_ID @@ -169,7 +161,7 @@ def compose_bash_script(self, configuration=None, configuration_parser=lambda x: echo $? else echo "application failed" - mutt -s "JOB_ID=$JOB_ID WORKUNIT_ID=$WORKUNIT_ID EXTERNALJOB_ID=$EXTERNALJOB_ID failed" $EMAIL < /dev/null + mutt -s "WORKUNIT_ID=$WORKUNIT_ID EXTERNALJOB_ID=$EXTERNALJOB_ID failed" $EMAIL < /dev/null bfabric_setResourceStatus_available.py $RESSOURCEID_STDOUT_STDERR $RESSOURCEID; exit 1; fi diff --git a/src/bfabric/wrapper_creator/bfabric_wrapper_creator.py b/src/bfabric/wrapper_creator/bfabric_wrapper_creator.py index e0e59959..8090b511 100644 --- a/src/bfabric/wrapper_creator/bfabric_wrapper_creator.py +++ b/src/bfabric/wrapper_creator/bfabric_wrapper_creator.py @@ -6,6 +6,7 @@ from pathlib import Path from typing import Any, Literal +import yaml from loguru import logger from bfabric import Bfabric @@ -71,7 +72,7 @@ def create_log_resource(self, variant: Literal["out", "err"], output_resource: R "name": f"slurm_std{variant}", "workunitid": self.workunit_definition.registration.workunit_id, "storageid": self._log_storage.id, - "relativepath": f"/workunitid-{self._workunit.id}_resourceid-{output_resource.id}.{variant}", + "relativepath": f"workunitid-{self._workunit.id}_resourceid-{output_resource.id}.{variant}", }, ) return Resource(result[0]) @@ -177,3 +178,19 @@ def write_results(self, config_serialized: str) -> tuple[dict[str, Any], dict[st self._client.save("externaljob", {"id": self._external_job_id, "status": "done"}) return yaml_workunit_executable, yaml_workunit_externaljob + + def create(self) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]]: + """Creates the YAML file external job and resources, and registers everything in B-Fabric.""" + output_resource = self.create_output_resource() + stdout_resource = self.create_log_resource(variant="out", output_resource=output_resource) + stderr_resource = self.create_log_resource(variant="err", output_resource=output_resource) + + config_dict = { + "application": self.get_application_section(output_resource=output_resource), + "job_configuration": self.get_job_configuration_section( + output_resource=output_resource, stdout_resource=stdout_resource, stderr_resource=stderr_resource + ), + } + config_serialized = yaml.safe_dump(config_dict) + yaml_workunit_executable, yaml_workunit_externaljob = self.write_results(config_serialized=config_serialized) + return config_dict, yaml_workunit_executable, yaml_workunit_externaljob diff --git a/src/bfabric/wrapper_creator/slurm.py b/src/bfabric/wrapper_creator/slurm.py index 2315b492..9f875f53 100644 --- a/src/bfabric/wrapper_creator/slurm.py +++ b/src/bfabric/wrapper_creator/slurm.py @@ -61,7 +61,12 @@ def sbatch(self, script: str | Path) -> tuple[str, str] | None: env = os.environ | {"SLURMROOT": self._slurm_root} result = subprocess.run( - [self._sbatch_bin, script], env=env, check=True, shell=False, capture_output=True, encoding="utf-8" + [str(self._sbatch_bin), str(script)], + env=env, + check=True, + shell=False, + capture_output=True, + encoding="utf-8", ) # TODO the code initially had a TODO to write these two to a file, in general I think the logs of the squeue # are currently not written to a file at all. diff --git a/src/bfabric_scripts/bfabric_list_not_available_proteomics_workunits.py b/src/bfabric_scripts/bfabric_list_not_available_proteomics_workunits.py index 94ed4452..063c59b9 100755 --- a/src/bfabric_scripts/bfabric_list_not_available_proteomics_workunits.py +++ b/src/bfabric_scripts/bfabric_list_not_available_proteomics_workunits.py @@ -13,77 +13,9 @@ from __future__ import annotations from argparse import ArgumentParser -from datetime import datetime, timedelta -from loguru import logger -from rich.console import Console -from rich.table import Column, Table - -from bfabric import Bfabric from bfabric.cli_formatting import setup_script_logging -from bfabric.entities import Parameter, Workunit, Application - - -def render_output(workunits_by_status: dict[str, list[Workunit]], client: Bfabric) -> None: - """Renders the output as a table.""" - table = Table( - Column("Application", no_wrap=False), - Column("WU ID", no_wrap=True), - Column("Created", no_wrap=True), - Column("Status", no_wrap=True), - Column("Created by", no_wrap=True, max_width=12), - Column("Name", no_wrap=False), - Column("Nodelist", no_wrap=False), - ) - - workunit_ids = [wu.id for wu_list in workunits_by_status.values() for wu in wu_list] - app_ids = {wu["application"]["id"] for wu_list in workunits_by_status.values() for wu in wu_list} - - nodelist_params = Parameter.find_by({"workunitid": workunit_ids, "key": "nodelist"}, client) - nodelist_values = {param["workunit"]["id"]: param.value for param in nodelist_params.values()} - application_values = Application.find_all(ids=sorted(app_ids), client=client) - - for status, workunits_all in workunits_by_status.items(): - workunits = [x for x in workunits_all if x["createdby"] not in ["gfeeder", "itfeeder"]] - status_color = { - "Pending": "yellow", - "Processing": "blue", - "Failed": "red", - }.get(status, "black") - - for wu in workunits: - app = application_values[wu["application"]["id"]] - table.add_row( - f"[link={app.web_url}]A{wu['application']['id']:3} {app['name']}[/link]", - f"[link={wu.web_url}&tab=details]WU{wu['id']}[/link]", - wu["created"], - f"[{status_color}]{status}[/{status_color}]", - wu["createdby"], - wu["name"], - nodelist_values.get(wu.id, "N/A"), - ) - - console = Console() - console.print(table) - - -def list_not_available_proteomics_workunits(date_cutoff: datetime) -> None: - """Lists proteomics work units that are not available on bfabric.""" - client = Bfabric.from_config() - console = Console() - with console.capture() as capture: - console.print( - f"listing not available proteomics work units created after {date_cutoff}", style="bright_yellow", end="" - ) - logger.info(capture.get()) - - workunits_by_status = {} - for status in ["Pending", "Processing", "Failed"]: - workunits_by_status[status] = Workunit.find_by( - {"status": status, "createdafter": date_cutoff.isoformat()}, client=client - ).values() - - render_output(workunits_by_status, client=client) +from bfabric_scripts.cli.workunit.not_available import list_not_available_proteomics_workunits def main() -> None: @@ -92,8 +24,7 @@ def main() -> None: parser = ArgumentParser(description="Lists proteomics work units that are not available on bfabric.") parser.add_argument("--max-age", type=int, help="Max age of work units in days", default=14) args = parser.parse_args() - date_cutoff = datetime.today() - timedelta(days=args.max_age) - list_not_available_proteomics_workunits(date_cutoff) + list_not_available_proteomics_workunits(max_age=args.max_age) if __name__ == "__main__": diff --git a/src/bfabric_scripts/bfabric_save_workflowstep.py b/src/bfabric_scripts/bfabric_save_workflowstep.py index f74e66ef..5ff8142c 100755 --- a/src/bfabric_scripts/bfabric_save_workflowstep.py +++ b/src/bfabric_scripts/bfabric_save_workflowstep.py @@ -42,8 +42,8 @@ def save_workflowstep(workunit_id: int | None = None) -> None: } workunit = client.read("workunit", obj={"id": workunit_id}).to_list_dict()[0] - application_id = workunit["application"]["_id"] - container_id = workunit["container"]["_id"] + application_id = workunit["application"]["id"] + container_id = workunit["container"]["id"] if application_id in workflowtemplatestep_ids and application_id in workflowtemplate_ids: workflows = client.read("workflow", obj={"containerid": container_id}).to_list_dict() @@ -70,7 +70,7 @@ def save_workflowstep(workunit_id: int | None = None) -> None: "workflowtemplatestepid": workflowtemplatestep_ids[application_id], "workunitid": workunit_id, }, - ).to_list_dict() + ) print(res[0]) diff --git a/src/bfabric_scripts/bfabric_upload_submitter_executable.py b/src/bfabric_scripts/bfabric_upload_submitter_executable.py index 052a4d77..46cfa333 100755 --- a/src/bfabric_scripts/bfabric_upload_submitter_executable.py +++ b/src/bfabric_scripts/bfabric_upload_submitter_executable.py @@ -35,65 +35,11 @@ from __future__ import annotations import argparse -import base64 from pathlib import Path -import yaml - from bfabric import Bfabric from bfabric.cli_formatting import setup_script_logging - - -def slurm_parameters() -> list[dict[str, str]]: - parameters = [{"modifiable": "true", "required": "true", "type": "STRING"} for _ in range(3)] - parameters[0]["description"] = "Which Slurm partition should be used." - parameters[0]["enumeration"] = ["prx"] - parameters[0]["key"] = "partition" - parameters[0]["label"] = "partition" - parameters[0]["value"] = "prx" - parameters[1]["description"] = "Which Slurm nodelist should be used." - parameters[1]["enumeration"] = ["fgcz-r-033"] - parameters[1]["key"] = "nodelist" - parameters[1]["label"] = "nodelist" - parameters[1]["value"] = "fgcz-r-[035,028]" - parameters[2]["description"] = "Which Slurm memory should be used." - parameters[2]["enumeration"] = ["10G", "50G", "128G", "256G", "512G", "960G"] - parameters[2]["key"] = "memory" - parameters[2]["label"] = "memory" - parameters[2]["value"] = "10G" - return parameters - - -def main_upload_submitter_executable( - client: Bfabric, filename: Path, engine: str, name: str | None, description: str | None -) -> None: - executable = filename.read_text() - - attr = { - "context": "SUBMITTER", - "parameter": [], - "masterexecutableid": 11871, - "status": "available", - "enabled": "true", - "valid": "true", - "base64": base64.b64encode(executable.encode()).decode(), - } - - if engine == "slurm": - name = name or "yaml / Slurm executable" - description = description or "Submitter executable for the bfabric functional test using Slurm." - attr["version"] = "1.03" - attr["parameter"] = slurm_parameters() - else: - raise NotImplementedError - - if name: - attr["name"] = name - if description: - attr["description"] = description - - res = client.save("executable", attr) - print(yaml.dump(res)) +from bfabric_scripts.cli.external_job.upload_submitter_executable import upload_submitter_executable_impl def main() -> None: @@ -111,7 +57,7 @@ def main() -> None: parser.add_argument("--name", type=str, help="Name of the submitter", required=False) parser.add_argument("--description", type=str, help="Description about the submitter", required=False) options = parser.parse_args() - main_upload_submitter_executable(client=client, **vars(options)) + upload_submitter_executable_impl(client=client, **vars(options)) if __name__ == "__main__": diff --git a/src/bfabric_scripts/cli/__init__.py b/src/bfabric_scripts/cli/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/bfabric_scripts/cli/__main__.py b/src/bfabric_scripts/cli/__main__.py new file mode 100644 index 00000000..0fdf9bf9 --- /dev/null +++ b/src/bfabric_scripts/cli/__main__.py @@ -0,0 +1,13 @@ +import cyclopts + +from bfabric_scripts.cli.cli_external_job import app as _app_external_job +from bfabric_scripts.cli.cli_read import app as _app_read +from bfabric_scripts.cli.cli_workunit import app as _app_workunit + +app = cyclopts.App() +app.command(_app_read, name="read") +app.command(_app_external_job, name="external-job") +app.command(_app_workunit, name="workunit") + +if __name__ == "__main__": + app() diff --git a/src/bfabric_scripts/cli/cli_external_job.py b/src/bfabric_scripts/cli/cli_external_job.py new file mode 100644 index 00000000..8afb1329 --- /dev/null +++ b/src/bfabric_scripts/cli/cli_external_job.py @@ -0,0 +1,61 @@ +import shutil +from pathlib import Path +from typing import Literal + +import cyclopts +from rich.pretty import pprint + +from bfabric import Bfabric +from bfabric.wrapper_creator.bfabric_submitter import BfabricSubmitter +from bfabric.wrapper_creator.bfabric_wrapper_creator import BfabricWrapperCreator +from bfabric_scripts.cli.external_job.upload_submitter_executable import upload_submitter_executable_impl +from bfabric_scripts.cli.external_job.upload_wrapper_creator_executable import upload_wrapper_creator_executable_impl + +app = cyclopts.App() + + +def find_slurm_root() -> str: + sbatch_path = shutil.which("sbatch") + if sbatch_path: + return str(Path(sbatch_path).parents[1]) + else: + return "/usr/" + + +@app.command +def submitter(external_job_id: int, scheduler: Literal["Slurm"] = "Slurm") -> None: + if scheduler != "Slurm": + raise NotImplementedError(f"Unsupported scheduler: {scheduler}") + slurm_root = find_slurm_root() + client = Bfabric.from_config() + submitter = BfabricSubmitter( + client=client, externaljobid=external_job_id, scheduleroot=slurm_root, scheduler="Slurm" + ) + submitter.submitter_yaml() + + +@app.command +def wrapper_creator(external_job_id: int) -> None: + client = Bfabric.from_config() + creator = BfabricWrapperCreator(client=client, external_job_id=external_job_id) + pprint(creator.create()) + + +@app.command +def upload_submitter_executable( + filename: Path, + *, + engine: Literal["slurm"] = "slurm", + name: str = "yaml / Slurm executable", + description: str = "Submitter executable for the bfabric functional test using Slurm.", +) -> None: + client = Bfabric.from_config() + upload_submitter_executable_impl( + client=client, filename=filename, engine=engine, name=name, description=description + ) + + +@app.command +def upload_wrapper_creator_executable(filename: Path) -> None: + client = Bfabric.from_config() + upload_wrapper_creator_executable_impl(client=client, filename=filename) diff --git a/src/bfabric_scripts/cli/cli_read.py b/src/bfabric_scripts/cli/cli_read.py new file mode 100644 index 00000000..495fb946 --- /dev/null +++ b/src/bfabric_scripts/cli/cli_read.py @@ -0,0 +1,165 @@ +from __future__ import annotations + +import json +import time +from enum import Enum +from typing import Any + +import cyclopts +import yaml +from loguru import logger +from pydantic import BaseModel +from rich.console import Console +from rich.pretty import pprint +from rich.table import Table + +from bfabric import Bfabric, BfabricClientConfig +from bfabric.cli_formatting import setup_script_logging + + +class OutputFormat(Enum): + JSON = "json" + YAML = "yaml" + TABLE_RICH = "table_rich" + AUTO = "auto" + + +app = cyclopts.App() + + +class CommandRead(BaseModel): + format: OutputFormat + limit: int + query: dict[str, str] + columns: list[str] | None + max_columns: int + + +@app.default +def bfabric_read( + endpoint: str, + attributes: list[tuple[str, str]] | None = None, + *, + output_format: OutputFormat = OutputFormat.AUTO, + limit: int = 100, + columns: list[str] | None = None, + max_columns: int = 7, +) -> None: + """Reads one or several items from a B-Fabric endpoint and prints them. + + Example usage: + read workunit name "DIANN%" createdafter 2024-10-31T19:00:00 + + :param endpoint: The endpoint to query. + :param attributes: A list of attribute-value pairs to filter the results by. + :param output_format: The output format to use. + :param limit: The maximum number of results to return. + :param columns: The columns to return (separate arguments). + :param max_columns: The maximum number of columns to return (only relevant if no columns are passed explicitly and a table output is chosen). + """ + setup_script_logging() + client = Bfabric.from_config() + console_out = Console() + + query = {attribute: value for attribute, value in attributes or []} + results = _get_results(client=client, endpoint=endpoint, query=query, limit=limit) + output_format = _determine_output_format( + console_out=console_out, output_format=output_format, n_results=len(results) + ) + output_columns = _determine_output_columns( + results=results, columns=columns, max_columns=max_columns, output_format=output_format + ) + + if output_format == OutputFormat.JSON: + print(json.dumps(results, indent=2)) + elif output_format == OutputFormat.YAML: + print(yaml.dump(results)) + elif output_format == OutputFormat.TABLE_RICH: + pprint( + CommandRead( + format=output_format, + limit=limit, + query=query, + columns=output_columns, + max_columns=max_columns, + ), + console=console_out, + ) + # _print_query_rich(console_out, query) + _print_table_rich(client.config, console_out, endpoint, results, output_columns=output_columns) + else: + raise ValueError(f"output format {output_format} not supported") + + +def _determine_output_columns( + results: list[dict[str, Any]], columns: list[str] | None, max_columns: int, output_format: OutputFormat +) -> list[str]: + if not columns: + if max_columns < 1: + raise ValueError("max_columns must be at least 1") + columns = ["id"] + available_columns = sorted(set(results[0].keys()) - {"id"}) + columns += available_columns[:max_columns] + + logger.info(f"columns = {columns}") + return columns + + +def _get_results(client: Bfabric, endpoint: str, query: dict[str, str], limit: int) -> list[dict[str, Any]]: + start_time = time.time() + results = client.read(endpoint=endpoint, obj=query, max_results=limit) + end_time = time.time() + logger.info(f"number of query result items = {len(results)}") + logger.info(f"query time = {end_time - start_time:.2f} seconds") + + results = sorted(results.to_list_dict(drop_empty=False), key=lambda x: x["id"]) + if results: + possible_attributes = sorted(set(results[0].keys())) + logger.info(f"possible attributes = {possible_attributes}") + + return results + + +def _print_query_rich( + console_out: Console, + query: dict[str, str], +) -> None: + pprint(query, console=console_out) + + +def _print_table_rich( + config: BfabricClientConfig, + console_out: Console, + endpoint: str, + res: list[dict[str, Any]], + output_columns: list[str], +) -> None: + """Prints the results as a rich table to the console.""" + table = Table(*output_columns) + for x in res: + entry_url = f"{config.base_url}/{endpoint}/show.html?id={x['id']}" + values = [] + for column in output_columns: + if column == "id": + values.append(f"[link={entry_url}]{x['id']}[/link]") + elif column == "groupingvar": + values.append(x.get(column, {}).get("name", "") or "") + elif isinstance(x.get(column), dict): + values.append(str(x.get(column, {}).get("id", ""))) + else: + values.append(str(x.get(column, ""))) + table.add_row(*values) + console_out.print(table) + + +def _determine_output_format(console_out: Console, output_format: OutputFormat, n_results: int) -> OutputFormat: + """Returns the format to use, based on the number of results, and whether the output is an interactive console. + If the format is already set to a concrete value instead of "auto", it will be returned unchanged. + """ + if output_format == OutputFormat.AUTO: + if n_results < 2: + output_format = OutputFormat.YAML + else: + output_format = OutputFormat.TABLE_RICH + logger.info(f"output format = {output_format}") + return output_format diff --git a/src/bfabric_scripts/cli/cli_workunit.py b/src/bfabric_scripts/cli/cli_workunit.py new file mode 100644 index 00000000..7f9f5ac1 --- /dev/null +++ b/src/bfabric_scripts/cli/cli_workunit.py @@ -0,0 +1,16 @@ +import cyclopts + +from bfabric.cli_formatting import setup_script_logging +from bfabric_scripts.cli.workunit.not_available import list_not_available_proteomics_workunits + +app = cyclopts.App() + + +@app.command +def not_available(max_age: float = 14.0) -> None: + """Lists not available analysis work units. + + :param max_age: The maximum age of work units in days. + """ + setup_script_logging() + list_not_available_proteomics_workunits(max_age=max_age) diff --git a/src/bfabric_scripts/cli/external_job/__init__.py b/src/bfabric_scripts/cli/external_job/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/bfabric_scripts/cli/external_job/upload_submitter_executable.py b/src/bfabric_scripts/cli/external_job/upload_submitter_executable.py new file mode 100644 index 00000000..574f382c --- /dev/null +++ b/src/bfabric_scripts/cli/external_job/upload_submitter_executable.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +import base64 +from pathlib import Path + +import yaml + +from bfabric import Bfabric + + +def slurm_parameters() -> list[dict[str, str]]: + parameters = [{"modifiable": "true", "required": "true", "type": "STRING"} for _ in range(3)] + parameters[0]["description"] = "Which Slurm partition should be used." + parameters[0]["enumeration"] = ["prx", "mascot"] + parameters[0]["key"] = "partition" + parameters[0]["label"] = "partition" + parameters[0]["value"] = "prx" + parameters[1]["description"] = "Which Slurm nodelist should be used." + parameters[1]["enumeration"] = ["fgcz-r-024", "fgcz-r-033", "fgcz-c-072", "fgcz-c-073"] + parameters[1]["key"] = "nodelist" + parameters[1]["label"] = "nodelist" + parameters[1]["value"] = "fgcz-r-[035,028]" + parameters[2]["description"] = "Which Slurm memory should be used." + parameters[2]["enumeration"] = ["10G", "50G", "128G", "256G", "512G", "960G"] + parameters[2]["key"] = "memory" + parameters[2]["label"] = "memory" + parameters[2]["value"] = "10G" + return parameters + + +def upload_submitter_executable_impl( + client: Bfabric, filename: Path, engine: str, name: str | None, description: str | None +) -> None: + executable = filename.read_text() + + attr = { + "context": "SUBMITTER", + "parameter": [], + "masterexecutableid": 11871, + "status": "available", + "enabled": "true", + "valid": "true", + "base64": base64.b64encode(executable.encode()).decode(), + } + + if engine == "slurm": + name = name or "yaml / Slurm executable" + description = description or "Submitter executable for the bfabric functional test using Slurm." + attr["version"] = "1.03" + attr["parameter"] = slurm_parameters() + else: + raise NotImplementedError + + attr["name"] = name + attr["description"] = description + + res = client.save("executable", attr) + print(yaml.dump(res)) diff --git a/src/bfabric_scripts/cli/external_job/upload_wrapper_creator_executable.py b/src/bfabric_scripts/cli/external_job/upload_wrapper_creator_executable.py new file mode 100644 index 00000000..3d0176da --- /dev/null +++ b/src/bfabric_scripts/cli/external_job/upload_wrapper_creator_executable.py @@ -0,0 +1,20 @@ +import base64 +from pathlib import Path + +from rich.pretty import pprint + +from bfabric import Bfabric + + +def upload_wrapper_creator_executable_impl(client: Bfabric, filename: Path): + executable_content = filename.read_text() + attr = { + "name": "yaml 004", + "context": "WRAPPERCREATOR", + "parameter": None, + "description": "None.", + "masterexecutableid": 11851, + "base64": base64.b64encode(executable_content.encode()).decode(), + } + result = client.save("executable", attr) + pprint(result) diff --git a/src/bfabric_scripts/cli/workunit/__init__.py b/src/bfabric_scripts/cli/workunit/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/bfabric_scripts/cli/workunit/not_available.py b/src/bfabric_scripts/cli/workunit/not_available.py new file mode 100644 index 00000000..6fa4d8ef --- /dev/null +++ b/src/bfabric_scripts/cli/workunit/not_available.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +from datetime import datetime, timedelta + +from loguru import logger +from rich.console import Console +from rich.table import Column, Table + +from bfabric import Bfabric +from bfabric.entities import Parameter, Workunit, Application + + +def render_output(workunits_by_status: dict[str, list[Workunit]], client: Bfabric) -> None: + """Renders the output as a table.""" + table = Table( + Column("Application", no_wrap=False), + Column("WU ID", no_wrap=True), + Column("Created", no_wrap=True), + Column("Status", no_wrap=True), + Column("Created by", no_wrap=True, max_width=12), + Column("Name", no_wrap=False), + Column("Nodelist", no_wrap=False), + ) + + workunit_ids = [wu.id for wu_list in workunits_by_status.values() for wu in wu_list] + app_ids = {wu["application"]["id"] for wu_list in workunits_by_status.values() for wu in wu_list} + + nodelist_params = Parameter.find_by({"workunitid": workunit_ids, "key": "nodelist"}, client) + nodelist_values = {param["workunit"]["id"]: param.value for param in nodelist_params.values()} + application_values = Application.find_all(ids=sorted(app_ids), client=client) + + for status, workunits_all in workunits_by_status.items(): + workunits = [x for x in workunits_all if x["createdby"] not in ["gfeeder", "itfeeder"]] + status_color = { + "Pending": "yellow", + "Processing": "blue", + "Failed": "red", + }.get(status, "black") + + for wu in workunits: + app = application_values[wu["application"]["id"]] + table.add_row( + f"[link={app.web_url}]A{wu['application']['id']:3} {app['name']}[/link]", + f"[link={wu.web_url}&tab=details]WU{wu['id']}[/link]", + wu["created"], + f"[{status_color}]{status}[/{status_color}]", + wu["createdby"], + wu["name"], + nodelist_values.get(wu.id, "N/A"), + ) + + console = Console() + console.print(table) + + +def list_not_available_proteomics_workunits(max_age: float) -> None: + """Lists proteomics work units that are not available on bfabric.""" + client = Bfabric.from_config() + date_cutoff = datetime.today() - timedelta(days=max_age) + console = Console() + with console.capture() as capture: + console.print( + f"listing not available proteomics work units created after {date_cutoff}", style="bright_yellow", end="" + ) + logger.info(capture.get()) + + workunits_by_status = {} + for status in ["Pending", "Processing", "Failed"]: + workunits_by_status[status] = Workunit.find_by( + {"status": status, "createdafter": date_cutoff.isoformat()}, client=client + ).values() + + render_output(workunits_by_status, client=client) diff --git a/tests/bfabric/wrapper_creator/test_slurm.py b/tests/bfabric/wrapper_creator/test_slurm.py index 3f375609..bc561982 100644 --- a/tests/bfabric/wrapper_creator/test_slurm.py +++ b/tests/bfabric/wrapper_creator/test_slurm.py @@ -23,7 +23,7 @@ def test_sbatch_when_success(mocker: MockerFixture, mock_slurm: SLURM, path: Pat assert stdout == "stdout" assert stderr == "stderr" mock_run.assert_called_once_with( - [Path("/tmp/test_slurm/bin/sbatch"), Path(path)], + ["/tmp/test_slurm/bin/sbatch", str(path)], env={"SLURMROOT": Path("/tmp/test_slurm"), "x": "y"}, check=True, shell=False,