diff --git a/.github/workflows/build-nextflow.yml b/.github/workflows/build-nextflow.yml index 84082e7..f1a5dd2 100644 --- a/.github/workflows/build-nextflow.yml +++ b/.github/workflows/build-nextflow.yml @@ -1,4 +1,5 @@ name: build +# TODO replace tool_name with the name of your tool on: push: @@ -30,17 +31,18 @@ jobs: run: | python -m pip install --upgrade pip setuptools pip install .[dev,test] - - name: Check CLI flags + - name: Check CLI basics run: | + which tool_name tool_name --help tool_name --version tool_name --citation - - name: Test - run: | - python -m pytest - name: Stub run run: | - tool_name run -profile ci_stub,docker -stub + mkdir -p tmp && pushd tmp + tool_name init + tool_name run -c conf/ci_stub.config -stub + popd - name: "Upload Artifact" uses: actions/upload-artifact@v3 if: always() # run even if previous steps fail diff --git a/README.md b/README.md index 593d89c..4f4e6e0 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,13 @@ CCBR template for creating Nextflow pipelines -[![build](https://github.com/CCBR/CCBR_NextflowTemplate/actions/workflows/build.yml/badge.svg)](https://github.com/CCBR/CCBR_NextflowTemplate/actions/workflows/build.yml) + + +[![build](https://github.com/CCBR/CCBR_NextflowTemplate/actions/workflows/build-nextflow.yml/badge.svg)](https://github.com/CCBR/CCBR_NextflowTemplate/actions/workflows/build-nextflow.yml) +[![docs](https://github.com/CCBR/CCBR_NextflowTemplate/actions/workflows/docs-mkdocs.yml/badge.svg)](https://github.com/CCBR/CCBR_NextflowTemplate/actions/workflows/docs-mkdocs.yml) + +See the website for detailed information, documentation, and examples: + ## Using this template @@ -77,6 +83,18 @@ Install the tool in edit mode: pip3 install -e . ``` +View CLI options: + +```sh +tool_name --help +``` + +Navigate to your project directory and initialize required config files: + +```sh +tool_name init +``` + Run the example ```sh diff --git a/docs/index.md b/docs/index.md index b4655ed..4ef7e8e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -3,9 +3,3 @@ --8<-- "README.md" - -Information on who the pipeline was developed for, and a statement if it's only been tested on Biowulf. For example: - -It has been developed and tested solely on NIH [HPC Biowulf](https://hpc.nih.gov/). - -Also include a workflow image to summarize the pipeline. diff --git a/pyproject.toml b/pyproject.toml index cc11612..28bd519 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,11 +31,12 @@ classifiers = [ "Programming Language :: Python :: 3.9", "Topic :: Scientific/Engineering :: Bio-Informatics", ] -requires-python = ">=3.8" +requires-python = ">=3.10" dependencies = [ - "pyyaml >= 6.0", + "ccbr_tools@git+https://github.com/CCBR/Tools", + "cffconvert >= 2.0.0", "Click >= 8.1.3", - "cffconvert >= 2.0.0" + "pyyaml >= 6.0" ] [project.optional-dependencies] diff --git a/src/__main__.py b/src/__main__.py index cffd05d..8465a3d 100644 --- a/src/__main__.py +++ b/src/__main__.py @@ -4,17 +4,28 @@ Check out the wiki for a detailed look at customizing this file: https://github.com/beardymcjohnface/Snaketool/wiki/Customising-your-Snaketool """ - -import os +import cffconvert.cli.cli import click -from .util import ( - nek_base, - get_version, - copy_config, - OrderedCommands, - run_nextflow, - print_citation, -) +import os +import pathlib + +import ccbr_tools.pkg_util +import ccbr_tools.pipeline.util +import ccbr_tools.pipeline.nextflow + + +def repo_base(*paths): + basedir = pathlib.Path(__file__).absolute().parent.parent + return basedir.joinpath(*paths) + + +def print_citation_flag(ctx, param, value): + if not value or ctx.resilient_parsing: + return + ccbr_tools.pkg_util.print_citation( + citation_file=repo_base("CITATION.cff"), output_format="bibtex" + ) + ctx.exit() def common_options(func): @@ -28,15 +39,22 @@ def common_options(func): @click.group( - cls=OrderedCommands, context_settings=dict(help_option_names=["-h", "--help"]) + cls=ccbr_tools.pkg_util.CustomClickGroup, + context_settings=dict(help_option_names=["-h", "--help"]), +) +@click.version_option( + ccbr_tools.pkg_util.get_version(repo_base=repo_base), + "-v", + "--version", + is_flag=True, ) -@click.version_option(get_version(), "-v", "--version", is_flag=True) @click.option( + "-c", "--citation", + callback=print_citation_flag, + is_eager=True, is_flag=True, - callback=print_citation, expose_value=False, - is_eager=True, help="Print the citation in bibtex format and exit.", ) def cli(): @@ -74,7 +92,7 @@ def cli(): "main_path", help="Path to the tool_name main.nf file or the GitHub repo (CCBR/TOOL_NAME). Defaults to the version installed in the $PATH.", type=str, - default=nek_base(os.path.join("main.nf")), + default=repo_base("main.nf"), show_default=True, ) @click.option( @@ -97,9 +115,10 @@ def run(main_path, _mode, **kwargs): f"Path to the tool_name main.nf file not found: {main_path}" ) - run_nextflow( + ccbr_tools.pipeline.nextflow.run( nextfile_path=main_path, mode=_mode, + pipeline_name="TOOL_NAME", **kwargs, ) @@ -108,13 +127,38 @@ def run(main_path, _mode, **kwargs): def init(**kwargs): """Initialize the working directory by copying the system default config files""" paths = ("nextflow.config", "conf/", "assets/") - copy_config(paths) - if not os.path.exists("log/"): - os.mkdir("log/") + ccbr_tools.pipeline.util.copy_config(paths, repo_base=repo_base) + os.makedirs("log", exist_ok=True) + + +@click.command() +@click.argument( + "citation_file", + type=click.Path(exists=True), + required=True, + default=repo_base("CITATION.cff"), +) +@click.option( + "--output-format", + "-f", + default="bibtex", + help="Output format for the citation", + type=cffconvert.cli.cli.options["outputformat"]["type"], +) +def cite(citation_file, output_format): + """ + Print the citation in the desired format + + citation_file : Path to a file in Citation File Format (CFF) [default: the CFF for ccbr_tools] + """ + ccbr_tools.pkg_util.print_citation( + citation_file=citation_file, output_format=output_format + ) cli.add_command(run) cli.add_command(init) +cli.add_command(cite) def main(): diff --git a/src/util.py b/src/util.py deleted file mode 100644 index 0df6bb3..0000000 --- a/src/util.py +++ /dev/null @@ -1,208 +0,0 @@ -from cffconvert.cli.create_citation import create_citation -from cffconvert.cli.validate_or_write_output import validate_or_write_output -from time import localtime, strftime - -import click -import collections.abc -import os -import pprint -import shutil -import stat -import subprocess -import sys -import yaml - - -def nek_base(rel_path): - basedir = os.path.split(os.path.dirname(os.path.realpath(__file__)))[0] - return os.path.join(basedir, rel_path) - - -def get_version(): - with open(nek_base("VERSION"), "r") as f: - version = f.readline() - return version - - -def print_citation(context, param, value): - if not value or context.resilient_parsing: - return - citation = create_citation(nek_base("CITATION.cff"), None) - # click.echo(citation._implementation.cffobj['message']) - validate_or_write_output(None, "bibtex", False, citation) - context.exit() - - -def msg(err_message): - tstamp = strftime("[%Y:%m:%d %H:%M:%S] ", localtime()) - click.echo(tstamp + err_message, err=True) - - -def msg_box(splash, errmsg=None): - msg("-" * (len(splash) + 4)) - msg(f"| {splash} |") - msg(("-" * (len(splash) + 4))) - if errmsg: - click.echo("\n" + errmsg, err=True) - - -def append_config_block(nf_config="nextflow.config", scope=None, **kwargs): - with open(nf_config, "a") as f: - f.write(scope.rstrip() + "{" + "\n") - for k in kwargs: - f.write(f"{k} = {kwargs[k]}\n") - f.write("}\n") - - -def copy_config(config_paths, overwrite=True): - msg(f"Copying default config files to current working directory") - for local_config in config_paths: - system_config = nek_base(local_config) - if os.path.isfile(system_config): - shutil.copyfile(system_config, local_config) - elif os.path.isdir(system_config): - shutil.copytree(system_config, local_config, dirs_exist_ok=overwrite) - else: - raise FileNotFoundError(f"Cannot copy {system_config} to {local_config}") - - -def read_config(file): - with open(file, "r") as stream: - _config = yaml.safe_load(stream) - return _config - - -def update_config(config, overwrite_config): - def _update(d, u): - for key, value in u.items(): - if isinstance(value, collections.abc.Mapping): - d[key] = _update(d.get(key, {}), value) - else: - d[key] = value - return d - - _update(config, overwrite_config) - - -def write_config(_config, file): - msg(f"Writing runtime config file to {file}") - with open(file, "w") as stream: - yaml.dump(_config, stream) - - -def chmod_bins_exec(): - """Ensure that all files in bin/ are executable. - - It appears that setuptools strips executable permissions from package_data files, - yet post-install scripts are not possible with the pyproject.toml format. - So this function will run when `run()` is called. - Without this hack, nextflow processes that call scripts in bin/ fail. - - https://stackoverflow.com/questions/18409296/package-data-files-with-executable-permissions - https://github.com/pypa/setuptools/issues/2041 - https://stackoverflow.com/questions/76320274/post-install-script-for-pyproject-toml-projects - """ - bin_dir = nek_base("bin/") - for filename in os.listdir(bin_dir): - bin_path = os.path.join(bin_dir, filename) - if os.path.isfile(bin_path): - file_stat = os.stat(bin_path) - # below is equivalent to `chmod +x` - os.chmod( - bin_path, file_stat.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH - ) - - -class OrderedCommands(click.Group): - """Preserve the order of subcommands when printing --help""" - - def list_commands(self, ctx: click.Context): - return list(self.commands) - - -def scontrol_show(): - scontrol_dict = dict() - scontrol_out = subprocess.run( - "scontrol show config", shell=True, capture_output=True, text=True - ).stdout - if len(scontrol_out) > 0: - for line in scontrol_out.split("\n"): - line_split = line.split("=") - if len(line_split) > 1: - scontrol_dict[line_split[0].strip()] = line_split[1].strip() - return scontrol_dict - - -hpc_options = { - "biowulf": {"profile": "biowulf", "slurm": "assets/slurm_header_biowulf.sh"}, - "fnlcr": { - "profile": "frce", - "slurm": "assets/slurm_header_frce.sh", - }, -} - - -def get_hpc(): - scontrol_out = scontrol_show() - if "ClusterName" in scontrol_out.keys(): - hpc = scontrol_out["ClusterName"] - else: - hpc = None - return hpc - - -def run_nextflow( - nextfile_path=None, - merge_config=None, - threads=None, - nextflow_args=None, - mode="local", -): - """Run a Nextflow workflow""" - nextflow_command = ["nextflow", "run", nextfile_path] - - hpc = get_hpc() - if mode == "slurm" and not hpc: - raise ValueError("mode is 'slurm' but no HPC environment was detected") - # add any additional Nextflow commands - args_dict = dict() - prev_arg = "" - for arg in nextflow_args: - if arg.startswith("-"): - args_dict[arg] = "" - elif prev_arg.startswith("-"): - args_dict[prev_arg] = arg - prev_arg = arg - # make sure profile matches biowulf or frce - profiles = ( - set(args_dict["-profile"].split(",")) - if "-profile" in args_dict.keys() - else set() - ) - if mode == "slurm": - profiles.add("slurm") - if hpc: - profiles.add(hpc_options[hpc]["profile"]) - args_dict["-profile"] = ",".join(sorted(profiles)) - nextflow_command += list(f"{k} {v}" for k, v in args_dict.items()) - - # Print nextflow command - nextflow_command = " ".join(str(nf) for nf in nextflow_command) - msg_box("Nextflow command", errmsg=nextflow_command) - - if mode == "slurm": - slurm_filename = "submit_slurm.sh" - with open(slurm_filename, "w") as sbatch_file: - with open(nek_base(hpc_options[hpc]["slurm"]), "r") as template: - sbatch_file.writelines(template.readlines()) - sbatch_file.write(nextflow_command) - run_command = f"sbatch {slurm_filename}" - msg_box("Slurm batch job", errmsg=run_command) - elif mode == "local": - if hpc: - nextflow_command = f'bash -c "module load nextflow && {nextflow_command}"' - run_command = nextflow_command - else: - raise ValueError(f"mode {mode} not recognized") - # Run Nextflow!!! - subprocess.run(run_command, shell=True, check=True)