From a7c5bd5a0f84a087a848a2789b782bc9f9cebac3 Mon Sep 17 00:00:00 2001 From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com> Date: Mon, 30 Dec 2024 13:23:06 -0500 Subject: [PATCH] add ability to specify path for report and table commands #534 --- looper/command_models/arguments.py | 6 +++++ looper/command_models/commands.py | 5 ++++- looper/divvy.py | 2 +- looper/exceptions.py | 3 ++- looper/looper.py | 35 +++++++++++++++++++++--------- requirements/requirements-all.txt | 2 +- tests/smoketests/test_other.py | 12 ++++++---- 7 files changed, 47 insertions(+), 18 deletions(-) diff --git a/looper/command_models/arguments.py b/looper/command_models/arguments.py index 2d6b2fbc..68c32977 100644 --- a/looper/command_models/arguments.py +++ b/looper/command_models/arguments.py @@ -184,6 +184,12 @@ class ArgumentEnum(enum.Enum): default=(str, None), description="Output directory", ) + REPORT_OUTPUT_DIR = Argument( + name="report_dir", + alias="-r", + default=(str, None), + description="Set location for looper report and looper table outputs", + ) GENERIC = Argument( name="generic", diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py index 1df4662c..69312f0d 100644 --- a/looper/command_models/commands.py +++ b/looper/command_models/commands.py @@ -124,7 +124,9 @@ def create_model(self) -> Type[pydantic.BaseModel]: TableParser = Command( "table", MESSAGE_BY_SUBCOMMAND["table"], - [], + [ + ArgumentEnum.REPORT_OUTPUT_DIR.value, + ], ) @@ -134,6 +136,7 @@ def create_model(self) -> Type[pydantic.BaseModel]: MESSAGE_BY_SUBCOMMAND["report"], [ ArgumentEnum.PORTABLE.value, + ArgumentEnum.REPORT_OUTPUT_DIR.value, ], ) diff --git a/looper/divvy.py b/looper/divvy.py index c9f97258..e458bad6 100644 --- a/looper/divvy.py +++ b/looper/divvy.py @@ -158,7 +158,7 @@ def activate_package(self, package_name): # but now, it makes more sense to do it here so we can piggyback on # the default update() method and not even have to do that. if not os.path.isabs(self.compute["submission_template"]): - + try: if self.filepath: self.compute["submission_template"] = os.path.join( diff --git a/looper/exceptions.py b/looper/exceptions.py index d0a4b5d2..7d478feb 100644 --- a/looper/exceptions.py +++ b/looper/exceptions.py @@ -111,8 +111,9 @@ def __init__(self, typename_by_requirement): ) self.error_specs = typename_by_requirement + class LooperReportError(LooperError): """Looper reporting errors""" def __init__(self, reason): - super(LooperReportError, self).__init__(reason) \ No newline at end of file + super(LooperReportError, self).__init__(reason) diff --git a/looper/looper.py b/looper/looper.py index a9d45d05..cb3cb301 100755 --- a/looper/looper.py +++ b/looper/looper.py @@ -99,11 +99,15 @@ def __call__(self, args): psms[piface.psm.pipeline_name] = piface.psm for pl_name, psm in psms.items(): all_project_level_records = psm.select_records() - for record in all_project_level_records['records']: - s = piface.psm.get_status(record_identifier=record['record_identifier']) + for record in all_project_level_records["records"]: + s = piface.psm.get_status( + record_identifier=record["record_identifier"] + ) status.setdefault(piface.psm.pipeline_name, {}) - status[piface.psm.pipeline_name][record['record_identifier']] = s - _LOGGER.debug(f"{self.prj.name} ({record['record_identifier']}): {s}") + status[piface.psm.pipeline_name][record["record_identifier"]] = s + _LOGGER.debug( + f"{self.prj.name} ({record['record_identifier']}): {s}" + ) else: for sample in self.prj.samples: @@ -564,6 +568,8 @@ def __call__(self, args): portable = args.portable + report_dir = getattr(args, "report_dir", None) + psms = {} if project_level: @@ -575,10 +581,14 @@ def __call__(self, args): for pl_name, psm in psms.items(): try: report_directory = psm.summarize( - looper_samples=self.prj.samples, portable=portable + looper_samples=self.prj.samples, + portable=portable, + output_dir=report_dir, ) except PipestatSummarizeError as e: - raise LooperReportError(f"Looper report error due to the following exception: {e}") + raise LooperReportError( + f"Looper report error due to the following exception: {e}" + ) print(f"Report directory: {report_directory}") self.debug["report_directory"] = report_directory return self.debug @@ -590,10 +600,14 @@ def __call__(self, args): for pl_name, psm in psms.items(): try: report_directory = psm.summarize( - looper_samples=self.prj.samples, portable=portable + looper_samples=self.prj.samples, + portable=portable, + output_dir=report_dir, ) except PipestatSummarizeError as e: - raise LooperReportError(f"Looper report error due to the following exception: {e}") + raise LooperReportError( + f"Looper report error due to the following exception: {e}" + ) print(f"Report directory: {report_directory}") self.debug["report_directory"] = report_directory return self.debug @@ -633,6 +647,7 @@ class Tabulator(Executor): def __call__(self, args): # p = self.prj project_level = getattr(args, "project", None) + report_dir = getattr(args, "report_dir", None) results = [] psms = {} if project_level: @@ -641,14 +656,14 @@ def __call__(self, args): if piface.psm.pipeline_name not in psms: psms[piface.psm.pipeline_name] = piface.psm for pl_name, psm in psms.items(): - results = psm.table() + results = psm.table(output_dir=report_dir) else: for piface in self.prj.pipeline_interfaces: if piface.psm.pipeline_type == PipelineLevel.SAMPLE.value: if piface.psm.pipeline_name not in psms: psms[piface.psm.pipeline_name] = piface.psm for pl_name, psm in psms.items(): - results = psm.table() + results = psm.table(output_dir=report_dir) # Results contains paths to stats and object summaries. return results diff --git a/requirements/requirements-all.txt b/requirements/requirements-all.txt index 7351b913..5be6ead7 100644 --- a/requirements/requirements-all.txt +++ b/requirements/requirements-all.txt @@ -4,7 +4,7 @@ jinja2 logmuse>=0.2.0 pandas>=2.0.2 pephubclient>=0.4.0 -pipestat>=0.11.1a1 +pipestat>=0.12.0a1 peppy>=0.40.6 pyyaml>=3.12 rich>=9.10.0 diff --git a/tests/smoketests/test_other.py b/tests/smoketests/test_other.py index 85f65f3a..bc23bfb6 100644 --- a/tests/smoketests/test_other.py +++ b/tests/smoketests/test_other.py @@ -3,7 +3,11 @@ import pytest from peppy import Project -from looper.exceptions import PipestatConfigurationException, MisconfigurationException, LooperReportError +from looper.exceptions import ( + PipestatConfigurationException, + MisconfigurationException, + LooperReportError, +) from tests.conftest import * from looper.cli_pydantic import main import pandas as pd @@ -86,12 +90,12 @@ def test_pipestat_configured(self, prep_temp_pep_pipestat, cmd): except Exception: raise pytest.fail("DID RAISE {0}".format(Exception)) else: - with pytest.raises(expected_exception=LooperReportError): # Looper report will and should raise exception if there are no results reported. + with pytest.raises( + expected_exception=LooperReportError + ): # Looper report will and should raise exception if there are no results reported. result = main(test_args=x) - - class TestLooperRerun: @pytest.mark.parametrize( "flags", [FLAGS[2], FLAGS[3]]