From beaad3c50ff7af0dfc4102ccc2baf81bf7452d17 Mon Sep 17 00:00:00 2001 From: Eric Blanc Date: Fri, 13 Dec 2024 13:59:39 +0100 Subject: [PATCH 1/5] feat: initial implementation of a simple wrapper class for all snappy wrappers --- snappy_wrappers/simple_wrapper.py | 126 ++++++++++++++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 snappy_wrappers/simple_wrapper.py diff --git a/snappy_wrappers/simple_wrapper.py b/snappy_wrappers/simple_wrapper.py new file mode 100644 index 00000000..26a9e593 --- /dev/null +++ b/snappy_wrappers/simple_wrapper.py @@ -0,0 +1,126 @@ +"""Abstract wrapper for cnvkit.py""" + +import os +import shutil +import stat +import tempfile +import textwrap + +from snakemake.shell import shell + +__author__ = "Eric Blanc" +__email__ = "eric.blanc@bih-charite.de" + + +class SimpleWrapper: + header = r""" + # Pipe everything to log file + if [[ -n "{snakemake.log.log}" ]]; then + if [[ "$(set +e; tty; set -e)" != "" ]]; then + rm -f "{snakemake.log.log}" && mkdir -p $(dirname {snakemake.log.log}) + exec &> >(tee -a "{snakemake.log.log}" >&2) + else + rm -f "{snakemake.log.log}" && mkdir -p $(dirname {snakemake.log.log}) + echo "No tty, logging disabled" >"{snakemake.log.log}" + fi + fi + + # Compute md5 except when filename ends with .md5 + compute_md5() {{ + fn=$1 + f=$(basename $fn) + d=$(dirname $fn) + pushd $d 1> /dev/null 2>&1 + md5sum $f > $f.md5 + popd 1> /dev/null 2>&1 + }} + + # Write out information about conda installation. + conda list >{snakemake.log.conda_list} + conda info >{snakemake.log.conda_info} + compute_md5 {snakemake.log.conda_list} + compute_md5 {snakemake.log.conda_info} + + # Create temp directory + TMPDIR=$(mktemp -d) + + set -x + + # --------------------------------- Start command ----------------------------------------- + """ + + footer = r""" + # --------------------------------- End command ------------------------------------------- + + set +x + + for fn in {snakemake.output} + do + if ! [[ $fn =~ \.md5$ ]] + then + compute_md5 $fn + fi + done + """ + + md5_log = r""" + f=$(basename {log}) + d=$(dirname {log}) + pushd $d 1> /dev/null 2>&1 + md5sum $f > $f.md5 + popd 1> /dev/null 2>&1 + """ + + def __init__(self, snakemake) -> None: + self.snakemake = snakemake + + def run_bash(self, cmd: str) -> None: + self._run(cmd, self.snakemake.log.sh) + shell(SimpleWrapper.md5_log.format(log=self.snakemake.log.sh)) + + def run_R(self, cmd: str) -> None: + with open(self.snakemake.log.R, "wt") as f: + print(cmd, file=f) + shell(SimpleWrapper.md5_log.format(log=self.snakemake.log.R)) + self._run(f"R --vanilla < {self.snakemake.log.R}", None) + + def _run(self, cmd: str, filename: str | None) -> None: + """ + Creates a temp file for the script, executes it & computes the md5 sum of the log + + The shell script is first created as a temporary file, and then copied over to + the log directory. + This allows R scripts to be saved in the log directory, rather than the uninformative + shell script starting R. + + : param cmd: The command string (after snakemake input/output/params expansion) + : param filename: the path where to save the script + """ + with tempfile.NamedTemporaryFile(mode="wt", delete_on_close=False) as f: + tempfilename = f.name + + print( + textwrap.dedent( + "\n".join( + ( + SimpleWrapper.header.format(snakemake=self.snakemake), + cmd, + SimpleWrapper.footer.format(snakemake=self.snakemake), + ) + ) + ), + file=f, + ) + + f.flush() + f.close() + + current_permissions = stat.S_IMODE(os.lstat(tempfilename).st_mode) + os.chmod(tempfilename, current_permissions | stat.S_IXUSR) + + if filename is not None: + shutil.copy(tempfilename, filename) + + shell(tempfilename) + + shell(SimpleWrapper.md5_log.format(log=str(self.snakemake.log.log))) From 9110360fb7ebafac06f6c8b5292946d26a2871e1 Mon Sep 17 00:00:00 2001 From: Till Hartmann Date: Fri, 13 Dec 2024 15:26:04 +0100 Subject: [PATCH 2/5] Introduce ShellWrapper and RWrapper as subclasses of SnappyWrapper --- .../{simple_wrapper.py => snappy_wrapper.py} | 60 ++++++++++++++----- 1 file changed, 45 insertions(+), 15 deletions(-) rename snappy_wrappers/{simple_wrapper.py => snappy_wrapper.py} (67%) diff --git a/snappy_wrappers/simple_wrapper.py b/snappy_wrappers/snappy_wrapper.py similarity index 67% rename from snappy_wrappers/simple_wrapper.py rename to snappy_wrappers/snappy_wrapper.py index 26a9e593..b6deaf6e 100644 --- a/snappy_wrappers/simple_wrapper.py +++ b/snappy_wrappers/snappy_wrapper.py @@ -5,6 +5,7 @@ import stat import tempfile import textwrap +from abc import abstractmethod, ABCMeta from snakemake.shell import shell @@ -12,7 +13,7 @@ __email__ = "eric.blanc@bih-charite.de" -class SimpleWrapper: +class SnappyWrapper(metaclass=ABCMeta): header = r""" # Pipe everything to log file if [[ -n "{snakemake.log.log}" ]]; then @@ -71,18 +72,21 @@ class SimpleWrapper: popd 1> /dev/null 2>&1 """ - def __init__(self, snakemake) -> None: - self.snakemake = snakemake + output_links = r""" + for path in {snakemake.output.output_links}; do + dst=$path + src=work/${{dst#output/}} + ln -sr $src $dst + done + """ - def run_bash(self, cmd: str) -> None: - self._run(cmd, self.snakemake.log.sh) - shell(SimpleWrapper.md5_log.format(log=self.snakemake.log.sh)) + def __init__(self, snakemake, with_output_links: bool = True) -> None: + self._snakemake = snakemake + self._with_output_links = with_output_links - def run_R(self, cmd: str) -> None: - with open(self.snakemake.log.R, "wt") as f: - print(cmd, file=f) - shell(SimpleWrapper.md5_log.format(log=self.snakemake.log.R)) - self._run(f"R --vanilla < {self.snakemake.log.R}", None) + @abstractmethod + def run(self, cmd: str) -> None: + pass def _run(self, cmd: str, filename: str | None) -> None: """ @@ -103,9 +107,9 @@ def _run(self, cmd: str, filename: str | None) -> None: textwrap.dedent( "\n".join( ( - SimpleWrapper.header.format(snakemake=self.snakemake), + SnappyWrapper.header.format(snakemake=self._snakemake), cmd, - SimpleWrapper.footer.format(snakemake=self.snakemake), + SnappyWrapper.footer.format(snakemake=self._snakemake), ) ) ), @@ -114,7 +118,7 @@ def _run(self, cmd: str, filename: str | None) -> None: f.flush() f.close() - + current_permissions = stat.S_IMODE(os.lstat(tempfilename).st_mode) os.chmod(tempfilename, current_permissions | stat.S_IXUSR) @@ -123,4 +127,30 @@ def _run(self, cmd: str, filename: str | None) -> None: shell(tempfilename) - shell(SimpleWrapper.md5_log.format(log=str(self.snakemake.log.log))) + shell(SnappyWrapper.md5_log.format(log=str(self._snakemake.log.log))) + + if ( + self._with_output_links + and getattr(self._snakemake.output, "output_links", None) is not None + ): + shell(SnappyWrapper.output_links.format(snakemake=self._snakemake)) + + +class ShellWrapper(SnappyWrapper): + def _run_bash(self, cmd: str) -> None: + self._run(cmd, self._snakemake.log.sh) + shell(SnappyWrapper.md5_log.format(log=self._snakemake.log.sh)) + + def run(self, cmd: str) -> None: + self._run_bash(cmd) + + +class RWrapper(SnappyWrapper): + def _run_R(self, cmd: str) -> None: + with open(self._snakemake.log.R, "wt") as f: + print(cmd, file=f) + shell(SnappyWrapper.md5_log.format(log=self._snakemake.log.R)) + self._run(f"R --vanilla < {self._snakemake.log.R}", None) + + def run(self, cmd: str) -> None: + self._run_R(cmd) From cb08a29a2f1ef7b8b07118a09aaeaa513d9c59c0 Mon Sep 17 00:00:00 2001 From: Till Hartmann Date: Fri, 13 Dec 2024 15:26:48 +0100 Subject: [PATCH 3/5] update doc comment --- snappy_wrappers/snappy_wrapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snappy_wrappers/snappy_wrapper.py b/snappy_wrappers/snappy_wrapper.py index b6deaf6e..d14b8b52 100644 --- a/snappy_wrappers/snappy_wrapper.py +++ b/snappy_wrappers/snappy_wrapper.py @@ -1,4 +1,4 @@ -"""Abstract wrapper for cnvkit.py""" +"""Abstract wrapper classes as utilities for snappy specific wrappers.""" import os import shutil From 905231d6e8c89285171ca1b70a8c75c9a03811c9 Mon Sep 17 00:00:00 2001 From: Till Hartmann Date: Fri, 13 Dec 2024 16:26:06 +0100 Subject: [PATCH 4/5] unify log.R and log.sh to log.script, check snakemake attributes --- snappy_wrappers/snappy_wrapper.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/snappy_wrappers/snappy_wrapper.py b/snappy_wrappers/snappy_wrapper.py index d14b8b52..2c10c2f6 100644 --- a/snappy_wrappers/snappy_wrapper.py +++ b/snappy_wrappers/snappy_wrapper.py @@ -83,6 +83,19 @@ class SnappyWrapper(metaclass=ABCMeta): def __init__(self, snakemake, with_output_links: bool = True) -> None: self._snakemake = snakemake self._with_output_links = with_output_links + self._check_snakemake_attributes() + + def _check_snakemake_attributes(self): + if not getattr(self._snakemake, "log", None): + raise AttributeError("snakemake.log is not defined") + if not getattr(self._snakemake.log, "log", None): + raise AttributeError("snakemake.log.log is not defined") + if not getattr(self._snakemake.log, "conda_list", None): + raise AttributeError("snakemake.log.conda_list is not defined") + if not getattr(self._snakemake.log, "conda_info", None): + raise AttributeError("snakemake.log.conda_info is not defined") + if not getattr(self._snakemake.log, "script", None): + raise AttributeError("snakemake.log.script is not defined") @abstractmethod def run(self, cmd: str) -> None: @@ -138,8 +151,8 @@ def _run(self, cmd: str, filename: str | None) -> None: class ShellWrapper(SnappyWrapper): def _run_bash(self, cmd: str) -> None: - self._run(cmd, self._snakemake.log.sh) - shell(SnappyWrapper.md5_log.format(log=self._snakemake.log.sh)) + self._run(cmd, self._snakemake.log.script) + shell(SnappyWrapper.md5_log.format(log=self._snakemake.log.script)) def run(self, cmd: str) -> None: self._run_bash(cmd) @@ -147,10 +160,10 @@ def run(self, cmd: str) -> None: class RWrapper(SnappyWrapper): def _run_R(self, cmd: str) -> None: - with open(self._snakemake.log.R, "wt") as f: + with open(self._snakemake.log.script, "wt") as f: print(cmd, file=f) - shell(SnappyWrapper.md5_log.format(log=self._snakemake.log.R)) - self._run(f"R --vanilla < {self._snakemake.log.R}", None) + shell(SnappyWrapper.md5_log.format(log=self._snakemake.log.script)) + self._run(f"R --vanilla < {self._snakemake.log.script}", None) def run(self, cmd: str) -> None: self._run_R(cmd) From e986d12487fd52b97a786a3572866bba762532f9 Mon Sep 17 00:00:00 2001 From: Till Hartmann Date: Fri, 13 Dec 2024 16:26:40 +0100 Subject: [PATCH 5/5] use RScript instead of R --- snappy_wrappers/snappy_wrapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snappy_wrappers/snappy_wrapper.py b/snappy_wrappers/snappy_wrapper.py index 2c10c2f6..9e6e9bdb 100644 --- a/snappy_wrappers/snappy_wrapper.py +++ b/snappy_wrappers/snappy_wrapper.py @@ -163,7 +163,7 @@ def _run_R(self, cmd: str) -> None: with open(self._snakemake.log.script, "wt") as f: print(cmd, file=f) shell(SnappyWrapper.md5_log.format(log=self._snakemake.log.script)) - self._run(f"R --vanilla < {self._snakemake.log.script}", None) + self._run(f"Rscript --vanilla {self._snakemake.log.script}", None) def run(self, cmd: str) -> None: self._run_R(cmd)