From bcb1abf6a4d4302f4977755a14b7f95e687a4425 Mon Sep 17 00:00:00 2001 From: Michael Hall Date: Thu, 24 Mar 2022 10:26:12 +1000 Subject: [PATCH 01/33] support quoted jobid from snakemake v7.1.1 --- {{cookiecutter.profile_name}}/lsf_status.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/{{cookiecutter.profile_name}}/lsf_status.py b/{{cookiecutter.profile_name}}/lsf_status.py index 5589f44..a0d04e9 100755 --- a/{{cookiecutter.profile_name}}/lsf_status.py +++ b/{{cookiecutter.profile_name}}/lsf_status.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import shlex import sys import time from pathlib import Path @@ -198,8 +199,12 @@ def get_status(self) -> str: if __name__ == "__main__": - jobid = int(sys.argv[1]) - outlog = sys.argv[2] + # need to support quoted and unquoted jobid + # see https://github.com/Snakemake-Profiles/lsf/issues/45 + split_args = shlex.split(" ".join(sys.argv[1:])) + jobid = int(split_args[0]) + outlog = split_args[1] + if CookieCutter.get_unknwn_behaviour().lower() == "wait": kill_unknown = False elif CookieCutter.get_unknwn_behaviour().lower() == "kill": From 890760767abb6e963d727d9f2700a7396f8e4bfe Mon Sep 17 00:00:00 2001 From: Michael Hall Date: Thu, 24 Mar 2022 10:28:13 +1000 Subject: [PATCH 02/33] Update CHANGELOG.md --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07f036a..7e5750d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ This document tracks changes to the `master` branch of the profile. ## [Unreleased] +### Fixed + +- Support quoted jobid from `snakemake>=v7.1.1` [[#45][45]] + ## [0.1.2] - 01/04/2021 ### Added @@ -71,6 +75,7 @@ This document tracks changes to the `master` branch of the profile. [9]: https://github.com/Snakemake-Profiles/lsf/pull/9 [36]: https://github.com/Snakemake-Profiles/lsf/issues/36 [39]: https://github.com/Snakemake-Profiles/lsf/issues/39 +[45]: https://github.com/Snakemake-Profiles/lsf/issues/45 [0.1.0]: https://github.com/Snakemake-Profiles/lsf/releases/tag/0.1.0 [0.1.1]: https://github.com/Snakemake-Profiles/lsf/releases/tag/0.1.1 [0.1.2]: https://github.com/Snakemake-Profiles/lsf/releases/tag/0.1.2 From fcc66ee71e727f01500e094617dec8199b946ed3 Mon Sep 17 00:00:00 2001 From: Brian Fulton-Howard Date: Mon, 22 Nov 2021 16:46:13 -0500 Subject: [PATCH 03/33] remove useless default threads and add default project --- cookiecutter.json | 1 + {{cookiecutter.profile_name}}/CookieCutter.py | 8 ++++---- {{cookiecutter.profile_name}}/lsf_submit.py | 13 ++++++++++++- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/cookiecutter.json b/cookiecutter.json index 23a189d..8a254e7 100644 --- a/cookiecutter.json +++ b/cookiecutter.json @@ -12,6 +12,7 @@ "default_threads": 1, "default_cluster_logdir": "logs/cluster", "default_queue": "", + "default_project": "", "max_status_checks_per_second": 10, "max_jobs_per_second": 10, "profile_name": "lsf" diff --git a/{{cookiecutter.profile_name}}/CookieCutter.py b/{{cookiecutter.profile_name}}/CookieCutter.py index 9c1709c..2be08d1 100644 --- a/{{cookiecutter.profile_name}}/CookieCutter.py +++ b/{{cookiecutter.profile_name}}/CookieCutter.py @@ -3,10 +3,6 @@ class CookieCutter: Cookie Cutter wrapper """ - @staticmethod - def get_default_threads() -> int: - return int("{{cookiecutter.default_threads}}") - @staticmethod def get_default_mem_mb() -> int: return int("{{cookiecutter.default_mem_mb}}") @@ -19,6 +15,10 @@ def get_log_dir() -> str: def get_default_queue() -> str: return "{{cookiecutter.default_queue}}" + @staticmethod + def get_default_project() -> str: + return int("{{cookiecutter.default_project}}") + @staticmethod def get_lsf_unit_for_limits() -> str: return "{{cookiecutter.LSF_UNIT_FOR_LIMITS}}" diff --git a/{{cookiecutter.profile_name}}/lsf_submit.py b/{{cookiecutter.profile_name}}/lsf_submit.py index ec469b0..4929885 100755 --- a/{{cookiecutter.profile_name}}/lsf_submit.py +++ b/{{cookiecutter.profile_name}}/lsf_submit.py @@ -65,7 +65,7 @@ def cluster(self) -> dict: @property def threads(self) -> int: - return self.job_properties.get("threads", CookieCutter.get_default_threads()) + return self.job_properties.get("threads", 1) @property def resources(self) -> dict: @@ -175,6 +175,16 @@ def queue_cmd(self) -> str: def rule_specific_params(self) -> str: return self.lsf_config.params_for_rule(self.rule_name) + @property + def proj(self) -> str: + if re.match(r'-P ', self.rule_specific_params): + return "" + return self.cluster.get("project", CookieCutter.get_default_project()) + + @property + def proj_cmd(self) -> str: + return "-P {}".format(self.proj) if self.proj else "" + @property def cluster_cmd(self) -> str: return self._cluster_cmd @@ -186,6 +196,7 @@ def submit_cmd(self) -> str: self.resources_cmd, self.jobinfo_cmd, self.queue_cmd, + self.proj_cmd, self.cluster_cmd, self.rule_specific_params, self.jobscript, From 1599ca43eccbf5e9fea257ae1bcd54e811def7a0 Mon Sep 17 00:00:00 2001 From: Brian Fulton-Howard Date: Mon, 22 Nov 2021 16:49:23 -0500 Subject: [PATCH 04/33] remove useless default threads from json --- cookiecutter.json | 1 - 1 file changed, 1 deletion(-) diff --git a/cookiecutter.json b/cookiecutter.json index 8a254e7..978a5f0 100644 --- a/cookiecutter.json +++ b/cookiecutter.json @@ -9,7 +9,6 @@ "print_shell_commands": false, "jobs": 500, "default_mem_mb": 1024, - "default_threads": 1, "default_cluster_logdir": "logs/cluster", "default_queue": "", "default_project": "", From dc8bba2d4eb5772db47b4a8ce46719800da978f9 Mon Sep 17 00:00:00 2001 From: Brian Fulton-Howard Date: Mon, 22 Nov 2021 17:17:08 -0500 Subject: [PATCH 05/33] project is not an int --- {{cookiecutter.profile_name}}/CookieCutter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/{{cookiecutter.profile_name}}/CookieCutter.py b/{{cookiecutter.profile_name}}/CookieCutter.py index 2be08d1..58d2729 100644 --- a/{{cookiecutter.profile_name}}/CookieCutter.py +++ b/{{cookiecutter.profile_name}}/CookieCutter.py @@ -17,7 +17,7 @@ def get_default_queue() -> str: @staticmethod def get_default_project() -> str: - return int("{{cookiecutter.default_project}}") + return "{{cookiecutter.default_project}}" @staticmethod def get_lsf_unit_for_limits() -> str: From fac4e93d17ab3b39c90c1740e38301d8d3f7c46c Mon Sep 17 00:00:00 2001 From: Brian Fulton-Howard Date: Mon, 22 Nov 2021 17:27:00 -0500 Subject: [PATCH 06/33] single to double quotes from running black --- {{cookiecutter.profile_name}}/lsf_submit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/{{cookiecutter.profile_name}}/lsf_submit.py b/{{cookiecutter.profile_name}}/lsf_submit.py index 4929885..537a2bc 100755 --- a/{{cookiecutter.profile_name}}/lsf_submit.py +++ b/{{cookiecutter.profile_name}}/lsf_submit.py @@ -177,7 +177,7 @@ def rule_specific_params(self) -> str: @property def proj(self) -> str: - if re.match(r'-P ', self.rule_specific_params): + if re.match(r"-P ", self.rule_specific_params): return "" return self.cluster.get("project", CookieCutter.get_default_project()) From e5077f5b6a83abddb62f10bab7a0549f99f72ad7 Mon Sep 17 00:00:00 2001 From: Brian Fulton-Howard Date: Mon, 22 Nov 2021 17:37:29 -0500 Subject: [PATCH 07/33] update tests and README for removal of thread default --- README.md | 10 ---------- tests/test_lsf_submit.py | 29 +---------------------------- 2 files changed, 1 insertion(+), 38 deletions(-) diff --git a/README.md b/README.md index 556c218..54cdb31 100644 --- a/README.md +++ b/README.md @@ -187,16 +187,6 @@ without `mem_mb` set under `resources`. See [below](#standard-rule-specific-cluster-resource-settings) for how to overwrite this in a `rule`. -#### `default_threads` - -**Default**: `1` - -This sets the default number of threads for a `rule` being submitted to the cluster -without the `threads` variable set. - -See [below](#standard-rule-specific-cluster-resource-settings) for how to overwrite this -in a `rule`. - #### `default_cluster_logdir` **Default**: `"logs/cluster"` diff --git a/tests/test_lsf_submit.py b/tests/test_lsf_submit.py index 4ef093f..544d2f9 100644 --- a/tests/test_lsf_submit.py +++ b/tests/test_lsf_submit.py @@ -24,9 +24,6 @@ class TestSubmitter(unittest.TestCase): @patch.object( CookieCutter, CookieCutter.get_default_mem_mb.__name__, return_value=1000 ) - @patch.object( - CookieCutter, CookieCutter.get_default_threads.__name__, return_value=8 - ) @patch.object(OSLayer, OSLayer.get_uuid4_string.__name__, return_value="random") def test___several_trivial_getter_methods(self, *mocks): argv = [ @@ -96,9 +93,6 @@ def test___several_trivial_getter_methods(self, *mocks): @patch.object( CookieCutter, CookieCutter.get_default_mem_mb.__name__, return_value=1000 ) - @patch.object( - CookieCutter, CookieCutter.get_default_threads.__name__, return_value=8 - ) @patch.object(OSLayer, OSLayer.get_uuid4_string.__name__, return_value="random") def test____submit_cmd_and_get_external_job_id___real_output_stream_from_submission( self, *mocks @@ -124,9 +118,6 @@ def test____submit_cmd_and_get_external_job_id___real_output_stream_from_submiss @patch.object( CookieCutter, CookieCutter.get_default_mem_mb.__name__, return_value=1000 ) - @patch.object( - CookieCutter, CookieCutter.get_default_threads.__name__, return_value=8 - ) @patch.object(OSLayer, OSLayer.get_uuid4_string.__name__, return_value="random") def test____submit_cmd_and_get_external_job_id___output_stream_has_no_jobid( self, *mocks @@ -149,9 +140,6 @@ def test____submit_cmd_and_get_external_job_id___output_stream_has_no_jobid( @patch.object( CookieCutter, CookieCutter.get_default_mem_mb.__name__, return_value=1000 ) - @patch.object( - CookieCutter, CookieCutter.get_default_threads.__name__, return_value=8 - ) @patch.object(OSLayer, OSLayer.get_uuid4_string.__name__, return_value="random") @patch.object(OSLayer, OSLayer.mkdir.__name__) @patch.object(OSLayer, OSLayer.remove_file.__name__) @@ -216,9 +204,6 @@ def test___submit___successfull_submit( @patch.object( CookieCutter, CookieCutter.get_default_mem_mb.__name__, return_value=1000 ) - @patch.object( - CookieCutter, CookieCutter.get_default_threads.__name__, return_value=8 - ) @patch.object(OSLayer, OSLayer.get_uuid4_string.__name__, return_value="random") @patch.object(OSLayer, OSLayer.mkdir.__name__) @patch.object(OSLayer, OSLayer.remove_file.__name__) @@ -297,9 +282,6 @@ def test_get_queue_cmd_returns_cookiecutter_default_if_no_cluster_config( @patch.object( CookieCutter, CookieCutter.get_default_mem_mb.__name__, return_value=1000 ) - @patch.object( - CookieCutter, CookieCutter.get_default_threads.__name__, return_value=8 - ) @patch.object(OSLayer, OSLayer.get_uuid4_string.__name__, return_value="random") def test_rule_specific_params_are_submitted(self, *mocks): argv = [ @@ -353,9 +335,6 @@ def test_rule_specific_params_are_submitted(self, *mocks): @patch.object( CookieCutter, CookieCutter.get_default_mem_mb.__name__, return_value=1000 ) - @patch.object( - CookieCutter, CookieCutter.get_default_threads.__name__, return_value=8 - ) @patch.object(OSLayer, OSLayer.get_uuid4_string.__name__, return_value="random") def test_lsf_mem_unit_is_kb_and_mem_mb_is_converted_accordingly(self, *mocks): argv = [ @@ -407,9 +386,6 @@ def test_lsf_mem_unit_is_kb_and_mem_mb_is_converted_accordingly(self, *mocks): @patch.object( CookieCutter, CookieCutter.get_default_mem_mb.__name__, return_value=1000 ) - @patch.object( - CookieCutter, CookieCutter.get_default_threads.__name__, return_value=8 - ) @patch.object(OSLayer, OSLayer.get_uuid4_string.__name__, return_value="random") def test_lsf_mem_unit_is_tb_and_mem_mb_is_converted_and_rounded_up_to_int( self, *mocks @@ -585,9 +561,6 @@ def test_jobname_for_group(self): @patch.object( CookieCutter, CookieCutter.get_default_mem_mb.__name__, return_value=1000 ) - @patch.object( - CookieCutter, CookieCutter.get_default_threads.__name__, return_value=8 - ) def test_time_resource_for_group(self, *mocks): for time_str in ("time", "runtime", "walltime", "time_min"): jobscript = Path( @@ -602,7 +575,7 @@ def test_time_resource_for_group(self, *mocks): actual = lsf_submit.resources_cmd expected = ( - "-M 1000 -n 8 -R 'select[mem>1000] rusage[mem=1000] " + "-M 1000 -n 1 -R 'select[mem>1000] rusage[mem=1000] " "span[hosts=1]' -W 1" ) From 15c311bb9e69f06b06d44721ce03468fedbfa7aa Mon Sep 17 00:00:00 2001 From: Brian Fulton-Howard Date: Mon, 22 Nov 2021 18:17:12 -0500 Subject: [PATCH 08/33] Add default_project to README --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index 54cdb31..9d2c8f5 100644 --- a/README.md +++ b/README.md @@ -219,6 +219,15 @@ The default queue on the cluster to submit jobs to. If left unset, then the defa your cluster will be used. The `bsub` parameter that this controls is [`-q`][bsub-q]. +#### `default_project` + +**Default**: None + +The default project on the cluster to submit jobs with. If left unset, then the default on +your cluster will be used. + +The `bsub` parameter that this controls is [`-P`][bsub-P]. + #### `max_status_checks_per_second` **Default**: `10` From 1812d6c5d7b8aad5ac42510ebe241b53ef0b343a Mon Sep 17 00:00:00 2001 From: Brian Fulton-Howard Date: Tue, 23 Nov 2021 13:04:31 -0500 Subject: [PATCH 09/33] don't include default queue if specified in lsf.yaml --- {{cookiecutter.profile_name}}/lsf_submit.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/{{cookiecutter.profile_name}}/lsf_submit.py b/{{cookiecutter.profile_name}}/lsf_submit.py index 537a2bc..8771aa3 100755 --- a/{{cookiecutter.profile_name}}/lsf_submit.py +++ b/{{cookiecutter.profile_name}}/lsf_submit.py @@ -165,6 +165,8 @@ def jobinfo_cmd(self) -> str: @property def queue(self) -> str: + if re.match(r"-q ", self.rule_specific_params): + return "" return self.cluster.get("queue", CookieCutter.get_default_queue()) @property From 37d4f3cad4bdbc971cd8fa95f16854a7f984c5cd Mon Sep 17 00:00:00 2001 From: BEFH Date: Tue, 23 Nov 2021 16:05:05 -0500 Subject: [PATCH 10/33] change defaults for Mount Sinai --- README.md | 74 +++++++++++++++++++++++++++++------------------ cookiecutter.json | 18 ++++++------ 2 files changed, 55 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index 9d2c8f5..e5a5817 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,17 @@ ## Install +To install the preconfigured Goate Lab settings, run the following: + +```bash +mkdir -p ~/.config/snakemake && \ + git clone -b preconfigured_goatelab \ + https://github.com/BEFH/lsf.git \ + ~/.config/snakemake/lsf +``` + +You can modify the presets in `~/.config/snakemake/lsf/config.yaml` and `~/.config/snakemake/lsf/CookieCutter.py` + ### Dependencies This profile is deployed using [Cookiecutter][cookiecutter-repo]. If you do not have @@ -53,9 +64,11 @@ You will then be prompted to set some default parameters. #### `LSF_UNIT_FOR_LIMITS` -**Default**: `KB` +**Default**: `KB` **Valid options:** `KB`, `MB`, `GB`, `TB`, `PB`, `EB`, `ZB` +***On Minerva, this is in MB.*** + **⚠️IMPORTANT⚠️**: This **must** be set to the same as [`LSF_UNIT_FOR_LIMITS`][limits] on your cluster. This value is stored in your cluster's [`lsf.conf`][lsf-conf] file. In general, this file is located at `${LSF_ENVDIR}/lsf.conf`. So the easiest way to get @@ -77,7 +90,7 @@ submitting jobs. See [here][18] for further information. #### `UNKWN_behaviour` -**Default**: `wait` +**Default**: `wait` **Valid options**: `wait`, `kill` When LSF returns a job status of `UNKWN` do you want to wait for the host the job is @@ -86,7 +99,7 @@ outlined [here][job_kill]? #### `ZOMBI_behaviour` -**Default**: `ignore` +**Default**: `ignore` **Valid options**: `ignore`, `kill` When LSF returns a job status of `ZOMBI` do you want to ignore this (not clean it up) or @@ -95,9 +108,9 @@ considered failed. #### `latency_wait` -**Default:** `5` +**Default:** `10` -This sets the default `--latency-wait/--output-wait/-w` parameter in `snakemake`. +This sets the default `--latency-wait/--output-wait/-w` parameter in `snakemake`. From the `snakemake --help` menu ```text @@ -109,10 +122,10 @@ From the `snakemake --help` menu #### `use_conda` -**Default**: `False` +**Default**: `True` **Valid options:** `False`, `True` -This sets the default `--use-conda` parameter in `snakemake`. +This sets the default `--use-conda` parameter in `snakemake`. From the `snakemake --help` menu ```text @@ -123,10 +136,10 @@ From the `snakemake --help` menu #### `use_singularity` -**Default**: `False` +**Default**: `True` **Valid options:** `False`, `True` -This sets the default `--use-singularity` parameter in `snakemake`. +This sets the default `--use-singularity` parameter in `snakemake`. From the `snakemake --help` menu ```text @@ -139,7 +152,7 @@ From the `snakemake --help` menu **Default**: `0` -This sets the default `--restart-times` parameter in `snakemake`. +This sets the default `--restart-times` parameter in `snakemake`. From the `snakemake --help` menu ```text @@ -150,10 +163,10 @@ From the `snakemake --help` menu #### `print_shell_commands` -**Default**: `False` +**Default**: `True` **Valid options:** `False`, `True` -This sets the default ` --printshellcmds/-p` parameter in `snakemake`. +This sets the default ` --printshellcmds/-p` parameter in `snakemake`. From the `snakemake --help` menu ```text @@ -162,9 +175,15 @@ From the `snakemake --help` menu #### `jobs` -**Default**: `500` +Determine how many jobs are supported by running -This sets the default `--cores/--jobs/-j` parameter in `snakemake`. +``` +ulimit -u +``` + +**Default**: `2000` + +This sets the default `--cores/--jobs/-j` parameter in `snakemake`. From the `snakemake --help` menu ```text @@ -179,7 +198,7 @@ the same time[1][1]. #### `default_mem_mb` -**Default**: `1024` +**Default**: `4096` This sets the default memory, in megabytes, for a `rule` being submitted to the cluster without `mem_mb` set under `resources`. @@ -196,13 +215,13 @@ to the working directory of the pipeline. If it does not exist, it will be creat The log files for a given rule are organised into sub-directories. This is to avoid having potentially thousands of files in one directory, as this can cause file system -issues. +issues. If you want to find the log files for a rule called `foo`, with wildcards `sample=a,ext=fq` then this would be located at `logs/cluster/foo/sample=a,ext=fq/jobid-.out` for the [standard -output][bsub-o] and with extension `.err` for the [standard error][bsub-e]. +output][bsub-o] and with extension `.err` for the [standard error][bsub-e]. `` is the internal jobid used by `snakemake` and is the same across multiple -attempts at running the same rule. +attempts at running the same rule. [``][uuid] is a random 28-digit, separated by `-`, and is specific to each attempt at running a rule. So if a rule fails, and is restarted, the uuid will be different. @@ -216,7 +235,7 @@ specific rule by following the instructions **Default**: None The default queue on the cluster to submit jobs to. If left unset, then the default on -your cluster will be used. +your cluster will be used. The `bsub` parameter that this controls is [`-q`][bsub-q]. #### `default_project` @@ -230,9 +249,9 @@ The `bsub` parameter that this controls is [`-P`][bsub-P]. #### `max_status_checks_per_second` -**Default**: `10` +**Default**: `5` -This sets the default `--max-status-checks-per-second` parameter in `snakemake`. +This sets the default `--max-status-checks-per-second` parameter in `snakemake`. From the `snakemake --help` menu ```text @@ -243,9 +262,9 @@ From the `snakemake --help` menu #### `max_jobs_per_second` -**Default**: `10` +**Default**: `5` -This sets the default `--max-jobs-per-second` parameter in `snakemake`. +This sets the default `--max-jobs-per-second` parameter in `snakemake`. From the `snakemake --help` menu ```text @@ -259,7 +278,7 @@ From the `snakemake --help` menu **Default**: `lsf` The name to use for this profile. The directory for the profile is created as this name -i.e. `$HOME/.config/snakemake/`. +i.e. `$HOME/.config/snakemake/`. This is also the value you pass to `snakemake --profile `. ## Usage @@ -313,7 +332,7 @@ rule foo: output: "bar.txt" shell: "grep 'bar' {input} > {output}" - + rule bar: input: "bar.txt" output: "file.out" @@ -334,7 +353,7 @@ foo: ``` In this example, we specify a default (`__default__`) [project][bsub-P] (`-P`) and -[runtime limit][bsub-W] (`-W`) that will apply to all rules. +[runtime limit][bsub-W] (`-W`) that will apply to all rules. We then override the project and, additionally, specify [GPU resources][bsub-gpu] for the rule `foo`. @@ -380,7 +399,7 @@ complex log file naming scheme. As the status-checker uses `tail` to get the sta the standard output log file of the job is very large, then status checking will be slowed down as a result. If you run into these problems and the `tail` solution is no feasible, the first suggestion would be to reduce `--max_status_checks_per_second` and -see if this helps. +see if this helps. Please raise an issue if you experience this, and the log file check doesn't seem to work. @@ -411,4 +430,3 @@ Please refer to [`CONTRIBUTING.md`](CONTRIBUTING.md). [yaml-collections]: https://yaml.org/spec/1.2/spec.html#id2759963 [leandro]: https://github.com/leoisl [snakemake_params]: https://snakemake.readthedocs.io/en/stable/executable.html#all-options - diff --git a/cookiecutter.json b/cookiecutter.json index 978a5f0..8fb686a 100644 --- a/cookiecutter.json +++ b/cookiecutter.json @@ -1,18 +1,18 @@ { - "LSF_UNIT_FOR_LIMITS": ["KB", "MB", "GB", "TB", "PB", "EB", "ZB"], + "LSF_UNIT_FOR_LIMITS": "MB", "UNKWN_behaviour": ["wait", "kill"], "ZOMBI_behaviour": ["ignore", "kill"], - "latency_wait": 5, - "use_conda": false, - "use_singularity": false, + "latency_wait": 10, + "use_conda": true, + "use_singularity": true, "restart_times": 0, - "print_shell_commands": false, - "jobs": 500, - "default_mem_mb": 1024, + "print_shell_commands": true, + "jobs": 2000, + "default_mem_mb": 4096, "default_cluster_logdir": "logs/cluster", "default_queue": "", "default_project": "", - "max_status_checks_per_second": 10, - "max_jobs_per_second": 10, + "max_status_checks_per_second": 5, + "max_jobs_per_second": 5, "profile_name": "lsf" } From ce5d5f610cd0acff6d83d1c77c858ed4f24ee626 Mon Sep 17 00:00:00 2001 From: BEFH Date: Tue, 23 Nov 2021 16:06:21 -0500 Subject: [PATCH 11/33] point at the correct repo for cookiecutter --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e5a5817..3af4a05 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ Download and set up the profile on your cluster profile_dir="${HOME}/.config/snakemake" mkdir -p "$profile_dir" # use cookiecutter to create the profile in the config directory -template="gh:Snakemake-Profiles/lsf" +template="gh:BEFH/lsf" cookiecutter --output-dir "$profile_dir" "$template" ``` From d849c0442b3733d0927c35a42a06522d402387e7 Mon Sep 17 00:00:00 2001 From: Michael Hall Date: Fri, 4 Mar 2022 10:08:22 +1000 Subject: [PATCH 12/33] remove BEFH README settings --- README.md | 78 ++++++++++++++++++++----------------------------------- 1 file changed, 28 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index 3af4a05..36af607 100644 --- a/README.md +++ b/README.md @@ -10,20 +10,6 @@ [TOC]: # -## Table of Contents -- [Install](#install) - - [Dependencies](#dependencies) - - [Profile](#profile) -- [Usage](#usage) - - [Standard rule-specific cluster resource settings](#standard-rule-specific-cluster-resource-settings) - - [Non-standard rule-specific cluster resource settings](#non-standard-rule-specific-cluster-resource-settings) -- [Known Issues](#known-issues) -- [Contributing](#contributing) - -## Install - -To install the preconfigured Goate Lab settings, run the following: - ```bash mkdir -p ~/.config/snakemake && \ git clone -b preconfigured_goatelab \ @@ -56,7 +42,7 @@ Download and set up the profile on your cluster profile_dir="${HOME}/.config/snakemake" mkdir -p "$profile_dir" # use cookiecutter to create the profile in the config directory -template="gh:BEFH/lsf" +template="gh:Snakemake-Profiles/lsf" cookiecutter --output-dir "$profile_dir" "$template" ``` @@ -64,11 +50,9 @@ You will then be prompted to set some default parameters. #### `LSF_UNIT_FOR_LIMITS` -**Default**: `KB` +**Default**: `KB` **Valid options:** `KB`, `MB`, `GB`, `TB`, `PB`, `EB`, `ZB` -***On Minerva, this is in MB.*** - **⚠️IMPORTANT⚠️**: This **must** be set to the same as [`LSF_UNIT_FOR_LIMITS`][limits] on your cluster. This value is stored in your cluster's [`lsf.conf`][lsf-conf] file. In general, this file is located at `${LSF_ENVDIR}/lsf.conf`. So the easiest way to get @@ -90,7 +74,7 @@ submitting jobs. See [here][18] for further information. #### `UNKWN_behaviour` -**Default**: `wait` +**Default**: `wait` **Valid options**: `wait`, `kill` When LSF returns a job status of `UNKWN` do you want to wait for the host the job is @@ -99,7 +83,7 @@ outlined [here][job_kill]? #### `ZOMBI_behaviour` -**Default**: `ignore` +**Default**: `ignore` **Valid options**: `ignore`, `kill` When LSF returns a job status of `ZOMBI` do you want to ignore this (not clean it up) or @@ -108,9 +92,9 @@ considered failed. #### `latency_wait` -**Default:** `10` +**Default:** `5` -This sets the default `--latency-wait/--output-wait/-w` parameter in `snakemake`. +This sets the default `--latency-wait/--output-wait/-w` parameter in `snakemake`. From the `snakemake --help` menu ```text @@ -122,10 +106,10 @@ From the `snakemake --help` menu #### `use_conda` -**Default**: `True` +**Default**: `False` **Valid options:** `False`, `True` -This sets the default `--use-conda` parameter in `snakemake`. +This sets the default `--use-conda` parameter in `snakemake`. From the `snakemake --help` menu ```text @@ -136,10 +120,10 @@ From the `snakemake --help` menu #### `use_singularity` -**Default**: `True` +**Default**: `False` **Valid options:** `False`, `True` -This sets the default `--use-singularity` parameter in `snakemake`. +This sets the default `--use-singularity` parameter in `snakemake`. From the `snakemake --help` menu ```text @@ -152,7 +136,7 @@ From the `snakemake --help` menu **Default**: `0` -This sets the default `--restart-times` parameter in `snakemake`. +This sets the default `--restart-times` parameter in `snakemake`. From the `snakemake --help` menu ```text @@ -163,10 +147,10 @@ From the `snakemake --help` menu #### `print_shell_commands` -**Default**: `True` +**Default**: `False` **Valid options:** `False`, `True` -This sets the default ` --printshellcmds/-p` parameter in `snakemake`. +This sets the default ` --printshellcmds/-p` parameter in `snakemake`. From the `snakemake --help` menu ```text @@ -175,15 +159,9 @@ From the `snakemake --help` menu #### `jobs` -Determine how many jobs are supported by running +**Default**: `500` -``` -ulimit -u -``` - -**Default**: `2000` - -This sets the default `--cores/--jobs/-j` parameter in `snakemake`. +This sets the default `--cores/--jobs/-j` parameter in `snakemake`. From the `snakemake --help` menu ```text @@ -198,7 +176,7 @@ the same time[1][1]. #### `default_mem_mb` -**Default**: `4096` +**Default**: `1024` This sets the default memory, in megabytes, for a `rule` being submitted to the cluster without `mem_mb` set under `resources`. @@ -215,13 +193,13 @@ to the working directory of the pipeline. If it does not exist, it will be creat The log files for a given rule are organised into sub-directories. This is to avoid having potentially thousands of files in one directory, as this can cause file system -issues. +issues. If you want to find the log files for a rule called `foo`, with wildcards `sample=a,ext=fq` then this would be located at `logs/cluster/foo/sample=a,ext=fq/jobid-.out` for the [standard -output][bsub-o] and with extension `.err` for the [standard error][bsub-e]. +output][bsub-o] and with extension `.err` for the [standard error][bsub-e]. `` is the internal jobid used by `snakemake` and is the same across multiple -attempts at running the same rule. +attempts at running the same rule. [``][uuid] is a random 28-digit, separated by `-`, and is specific to each attempt at running a rule. So if a rule fails, and is restarted, the uuid will be different. @@ -235,7 +213,7 @@ specific rule by following the instructions **Default**: None The default queue on the cluster to submit jobs to. If left unset, then the default on -your cluster will be used. +your cluster will be used. The `bsub` parameter that this controls is [`-q`][bsub-q]. #### `default_project` @@ -249,9 +227,9 @@ The `bsub` parameter that this controls is [`-P`][bsub-P]. #### `max_status_checks_per_second` -**Default**: `5` +**Default**: `10` -This sets the default `--max-status-checks-per-second` parameter in `snakemake`. +This sets the default `--max-status-checks-per-second` parameter in `snakemake`. From the `snakemake --help` menu ```text @@ -262,9 +240,9 @@ From the `snakemake --help` menu #### `max_jobs_per_second` -**Default**: `5` +**Default**: `10` -This sets the default `--max-jobs-per-second` parameter in `snakemake`. +This sets the default `--max-jobs-per-second` parameter in `snakemake`. From the `snakemake --help` menu ```text @@ -278,7 +256,7 @@ From the `snakemake --help` menu **Default**: `lsf` The name to use for this profile. The directory for the profile is created as this name -i.e. `$HOME/.config/snakemake/`. +i.e. `$HOME/.config/snakemake/`. This is also the value you pass to `snakemake --profile `. ## Usage @@ -353,7 +331,7 @@ foo: ``` In this example, we specify a default (`__default__`) [project][bsub-P] (`-P`) and -[runtime limit][bsub-W] (`-W`) that will apply to all rules. +[runtime limit][bsub-W] (`-W`) that will apply to all rules. We then override the project and, additionally, specify [GPU resources][bsub-gpu] for the rule `foo`. @@ -376,7 +354,7 @@ The above is also a valid form of the previous example but **not recommended**. #### Quote-escaping -Some LSF commands require multiple levels of quote-escaping. +Some LSF commands require multiple levels of quote-escaping. For example, to exclude a node from job submission which has non-alphabetic characters in its name ([docs](https://www.ibm.com/support/knowledgecenter/SSWRJV_10.1.0/lsf_command_ref/bsub.__r.1.html?view=embed)): `bsub -R "select[hname!='node-name']"`. @@ -399,7 +377,7 @@ complex log file naming scheme. As the status-checker uses `tail` to get the sta the standard output log file of the job is very large, then status checking will be slowed down as a result. If you run into these problems and the `tail` solution is no feasible, the first suggestion would be to reduce `--max_status_checks_per_second` and -see if this helps. +see if this helps. Please raise an issue if you experience this, and the log file check doesn't seem to work. From bb4fcc2045a191d6299e303ebe1648369ac8afb3 Mon Sep 17 00:00:00 2001 From: Michael Hall Date: Fri, 4 Mar 2022 10:10:41 +1000 Subject: [PATCH 13/33] revert cookiecutter json --- cookiecutter.json | 38 ++++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/cookiecutter.json b/cookiecutter.json index 8fb686a..8c38871 100644 --- a/cookiecutter.json +++ b/cookiecutter.json @@ -1,18 +1,32 @@ { - "LSF_UNIT_FOR_LIMITS": "MB", - "UNKWN_behaviour": ["wait", "kill"], - "ZOMBI_behaviour": ["ignore", "kill"], - "latency_wait": 10, - "use_conda": true, - "use_singularity": true, + "LSF_UNIT_FOR_LIMITS": [ + "KB", + "MB", + "GB", + "TB", + "PB", + "EB", + "ZB" + ], + "UNKWN_behaviour": [ + "wait", + "kill" + ], + "ZOMBI_behaviour": [ + "ignore", + "kill" + ], + "latency_wait": 5, + "use_conda": false, + "use_singularity": false, "restart_times": 0, - "print_shell_commands": true, - "jobs": 2000, - "default_mem_mb": 4096, + "print_shell_commands": false, + "jobs": 500, + "default_mem_mb": 1024, "default_cluster_logdir": "logs/cluster", "default_queue": "", "default_project": "", - "max_status_checks_per_second": 5, - "max_jobs_per_second": 5, + "max_status_checks_per_second": 10, + "max_jobs_per_second": 10, "profile_name": "lsf" -} +} \ No newline at end of file From b0a5d73b1dfc924537e691565ed9de645f208e88 Mon Sep 17 00:00:00 2001 From: Michael Hall Date: Fri, 4 Mar 2022 10:54:06 +1000 Subject: [PATCH 14/33] fix tests and re method --- tests/test_lsf_submit.py | 25 ++++++++++++++++----- {{cookiecutter.profile_name}}/lsf_submit.py | 4 ++-- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/tests/test_lsf_submit.py b/tests/test_lsf_submit.py index 544d2f9..94d79a5 100644 --- a/tests/test_lsf_submit.py +++ b/tests/test_lsf_submit.py @@ -24,6 +24,9 @@ class TestSubmitter(unittest.TestCase): @patch.object( CookieCutter, CookieCutter.get_default_mem_mb.__name__, return_value=1000 ) + @patch.object( + CookieCutter, CookieCutter.get_default_project.__name__, return_value="proj" + ) @patch.object(OSLayer, OSLayer.get_uuid4_string.__name__, return_value="random") def test___several_trivial_getter_methods(self, *mocks): argv = [ @@ -72,7 +75,7 @@ def test___several_trivial_getter_methods(self, *mocks): self.assertEqual( lsf_submit.submit_cmd, "bsub -M {mem} -n 1 -R 'select[mem>{mem}] rusage[mem={mem}] span[hosts=1]' " - "{jobinfo} -q q1 cluster_opt_1 cluster_opt_2 cluster_opt_3 " + "{jobinfo} -q q1 -P proj cluster_opt_1 cluster_opt_2 cluster_opt_3 " "real_jobscript.sh".format(mem=expected_mem, jobinfo=expected_jobinfo_cmd), ) @@ -140,6 +143,9 @@ def test____submit_cmd_and_get_external_job_id___output_stream_has_no_jobid( @patch.object( CookieCutter, CookieCutter.get_default_mem_mb.__name__, return_value=1000 ) + @patch.object( + CookieCutter, CookieCutter.get_default_project.__name__, return_value="proj" + ) @patch.object(OSLayer, OSLayer.get_uuid4_string.__name__, return_value="random") @patch.object(OSLayer, OSLayer.mkdir.__name__) @patch.object(OSLayer, OSLayer.remove_file.__name__) @@ -191,7 +197,7 @@ def test___submit___successfull_submit( expected_mem = "2662" run_process_mock.assert_called_once_with( "bsub -M {mem} -n 1 -R 'select[mem>{mem}] rusage[mem={mem}] span[hosts=1]' " - "{jobinfo} -q q1 cluster_opt_1 cluster_opt_2 cluster_opt_3 " + "{jobinfo} -q q1 -P proj cluster_opt_1 cluster_opt_2 cluster_opt_3 " "real_jobscript.sh".format(mem=expected_mem, jobinfo=expected_jobinfo_cmd) ) print_mock.assert_called_once_with( @@ -204,6 +210,9 @@ def test___submit___successfull_submit( @patch.object( CookieCutter, CookieCutter.get_default_mem_mb.__name__, return_value=1000 ) + @patch.object( + CookieCutter, CookieCutter.get_default_project.__name__, return_value="proj" + ) @patch.object(OSLayer, OSLayer.get_uuid4_string.__name__, return_value="random") @patch.object(OSLayer, OSLayer.mkdir.__name__) @patch.object(OSLayer, OSLayer.remove_file.__name__) @@ -247,7 +256,7 @@ def test___submit___failed_submit_bsub_invocation_error( expected_mem = "2662" run_process_mock.assert_called_once_with( "bsub -M {mem} -n 1 -R 'select[mem>{mem}] rusage[mem={mem}] span[hosts=1]' " - "{jobinfo} -q q1 cluster_opt_1 cluster_opt_2 cluster_opt_3 " + "{jobinfo} -q q1 -P proj cluster_opt_1 cluster_opt_2 cluster_opt_3 " "real_jobscript.sh".format(mem=expected_mem, jobinfo=expected_jobinfo_cmd) ) print_mock.assert_not_called() @@ -335,6 +344,9 @@ def test_rule_specific_params_are_submitted(self, *mocks): @patch.object( CookieCutter, CookieCutter.get_default_mem_mb.__name__, return_value=1000 ) + @patch.object( + CookieCutter, CookieCutter.get_default_project.__name__, return_value="proj" + ) @patch.object(OSLayer, OSLayer.get_uuid4_string.__name__, return_value="random") def test_lsf_mem_unit_is_kb_and_mem_mb_is_converted_accordingly(self, *mocks): argv = [ @@ -373,7 +385,7 @@ def test_lsf_mem_unit_is_kb_and_mem_mb_is_converted_accordingly(self, *mocks): ).format(outlog=expected_outlog, errlog=expected_errlog) expected = ( "bsub -M {mem} -n 1 -R 'select[mem>{mem}] rusage[mem={mem}] span[hosts=1]' " - "{jobinfo} -q q1 cluster_opt_1 cluster_opt_2 cluster_opt_3 " + "{jobinfo} cluster_opt_1 cluster_opt_2 cluster_opt_3 " "-q queue -gpu - -P project " "real_jobscript.sh".format(mem=expected_mem, jobinfo=expected_jobinfo_cmd) ) @@ -383,6 +395,9 @@ def test_lsf_mem_unit_is_kb_and_mem_mb_is_converted_accordingly(self, *mocks): @patch.object( CookieCutter, CookieCutter.get_log_dir.__name__, return_value="logdir" ) + @patch.object( + CookieCutter, CookieCutter.get_default_project.__name__, return_value="proj" + ) @patch.object( CookieCutter, CookieCutter.get_default_mem_mb.__name__, return_value=1000 ) @@ -426,7 +441,7 @@ def test_lsf_mem_unit_is_tb_and_mem_mb_is_converted_and_rounded_up_to_int( ).format(outlog=expected_outlog, errlog=expected_errlog) expected = ( "bsub -M {mem} -n 1 -R 'select[mem>{mem}] rusage[mem={mem}] span[hosts=1]' " - "{jobinfo} -q q1 cluster_opt_1 cluster_opt_2 cluster_opt_3 " + "{jobinfo} cluster_opt_1 cluster_opt_2 cluster_opt_3 " "-q queue -gpu - -P project " "real_jobscript.sh".format(mem=expected_mem, jobinfo=expected_jobinfo_cmd) ) diff --git a/{{cookiecutter.profile_name}}/lsf_submit.py b/{{cookiecutter.profile_name}}/lsf_submit.py index 8771aa3..9b01450 100755 --- a/{{cookiecutter.profile_name}}/lsf_submit.py +++ b/{{cookiecutter.profile_name}}/lsf_submit.py @@ -165,7 +165,7 @@ def jobinfo_cmd(self) -> str: @property def queue(self) -> str: - if re.match(r"-q ", self.rule_specific_params): + if re.search(r"-q ", self.rule_specific_params): return "" return self.cluster.get("queue", CookieCutter.get_default_queue()) @@ -179,7 +179,7 @@ def rule_specific_params(self) -> str: @property def proj(self) -> str: - if re.match(r"-P ", self.rule_specific_params): + if re.search(r"-P ", self.rule_specific_params): return "" return self.cluster.get("project", CookieCutter.get_default_project()) From 1ce3e120fbf0e05a6ac95c358c332b5f78b79d98 Mon Sep 17 00:00:00 2001 From: Michael Hall Date: Fri, 4 Mar 2022 10:56:18 +1000 Subject: [PATCH 15/33] fmt --- {{cookiecutter.profile_name}}/memory_units.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/{{cookiecutter.profile_name}}/memory_units.py b/{{cookiecutter.profile_name}}/memory_units.py index d79bd97..12e7310 100644 --- a/{{cookiecutter.profile_name}}/memory_units.py +++ b/{{cookiecutter.profile_name}}/memory_units.py @@ -112,7 +112,7 @@ def _scaling_factor(self, decimal: bool = True) -> int: def bytes(self, decimal_multiples: bool = True) -> float: scaling_factor = self._scaling_factor(decimal_multiples) - return float(self.value * (scaling_factor ** self.power)) + return float(self.value * (scaling_factor**self.power)) def to(self, unit: Unit, decimal_multiples: bool = True) -> "Memory": scaling_factor = self._scaling_factor(decimal_multiples) ** unit.power From 1a3354f1e13dfd4c98c4c4dd6830595a396c8b55 Mon Sep 17 00:00:00 2001 From: Michael Hall Date: Fri, 4 Mar 2022 10:57:36 +1000 Subject: [PATCH 16/33] remove 3.5 from linting --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c50d680..02153e0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -32,7 +32,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [ 3.5, 3.6, 3.7, 3.8 ] + python-version: [ 3.6, 3.7, 3.8 ] steps: - uses: actions/checkout@v2 From 87c9a4d2b4176e06e458da953401d20096f802b8 Mon Sep 17 00:00:00 2001 From: Michael Hall Date: Fri, 4 Mar 2022 11:02:10 +1000 Subject: [PATCH 17/33] update changelog --- .github/workflows/ci.yaml | 2 +- CHANGELOG.md | 12 ++++++++++++ README.md | 19 +++++++++++-------- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 02153e0..37cd124 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -52,7 +52,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [ 3.5, 3.6, 3.7, 3.8 ] + python-version: [ 3.6, 3.7, 3.8 ] steps: - uses: actions/checkout@v2 diff --git a/CHANGELOG.md b/CHANGELOG.md index 07f036a..f50702d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,18 @@ This document tracks changes to the `master` branch of the profile. ## [Unreleased] +### Added + +- Default project in cookiecutter + +### Removed + +- Default threads in cookiecutter + +### Changed + +- Default project and queue will be removed from the submission command if they are present in the `lsf.yaml` + ## [0.1.2] - 01/04/2021 ### Added diff --git a/README.md b/README.md index 36af607..7f6a26e 100644 --- a/README.md +++ b/README.md @@ -10,14 +10,17 @@ [TOC]: # -```bash -mkdir -p ~/.config/snakemake && \ - git clone -b preconfigured_goatelab \ - https://github.com/BEFH/lsf.git \ - ~/.config/snakemake/lsf -``` - -You can modify the presets in `~/.config/snakemake/lsf/config.yaml` and `~/.config/snakemake/lsf/CookieCutter.py` +## Table of Contents +- [Install](#install) + - [Dependencies](#dependencies) + - [Profile](#profile) +- [Usage](#usage) + - [Standard rule-specific cluster resource settings](#standard-rule-specific-cluster-resource-settings) + - [Non-standard rule-specific cluster resource settings](#non-standard-rule-specific-cluster-resource-settings) +- [Known Issues](#known-issues) +- [Contributing](#contributing) + +## Install ### Dependencies From 461ffbecf1ce20bb23fab92fb2b4fed6e3f59547 Mon Sep 17 00:00:00 2001 From: Brian Fulton-Howard Date: Thu, 31 Mar 2022 11:02:01 -0400 Subject: [PATCH 18/33] fixed status check for new version --- {{cookiecutter.profile_name}}/lsf_status.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/{{cookiecutter.profile_name}}/lsf_status.py b/{{cookiecutter.profile_name}}/lsf_status.py index 5589f44..7051c9a 100755 --- a/{{cookiecutter.profile_name}}/lsf_status.py +++ b/{{cookiecutter.profile_name}}/lsf_status.py @@ -198,8 +198,8 @@ def get_status(self) -> str: if __name__ == "__main__": - jobid = int(sys.argv[1]) - outlog = sys.argv[2] + jobid = int(sys.argv[1].split()[0]) + outlog = sys.argv[1].split()[1] if CookieCutter.get_unknwn_behaviour().lower() == "wait": kill_unknown = False elif CookieCutter.get_unknwn_behaviour().lower() == "kill": From 8c0063423e4a1bf350b8ad2bb30682c6ba59a23f Mon Sep 17 00:00:00 2001 From: Michael Hall Date: Sun, 17 Apr 2022 09:34:17 +1000 Subject: [PATCH 19/33] Revert "fixed status check for new version" This reverts commit 0c9ab5cb31f5e94f1d02c1e640d448121a2f5d57. --- {{cookiecutter.profile_name}}/lsf_status.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/{{cookiecutter.profile_name}}/lsf_status.py b/{{cookiecutter.profile_name}}/lsf_status.py index 7051c9a..5589f44 100755 --- a/{{cookiecutter.profile_name}}/lsf_status.py +++ b/{{cookiecutter.profile_name}}/lsf_status.py @@ -198,8 +198,8 @@ def get_status(self) -> str: if __name__ == "__main__": - jobid = int(sys.argv[1].split()[0]) - outlog = sys.argv[1].split()[1] + jobid = int(sys.argv[1]) + outlog = sys.argv[2] if CookieCutter.get_unknwn_behaviour().lower() == "wait": kill_unknown = False elif CookieCutter.get_unknwn_behaviour().lower() == "kill": From d7c0d8ad7e41a5749e8b366acf465f302d6d7a9e Mon Sep 17 00:00:00 2001 From: Michael Hall Date: Sun, 17 Apr 2022 09:37:43 +1000 Subject: [PATCH 20/33] add suggested tests --- tests/test_lsf_submit.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_lsf_submit.py b/tests/test_lsf_submit.py index 94d79a5..45b6906 100644 --- a/tests/test_lsf_submit.py +++ b/tests/test_lsf_submit.py @@ -72,6 +72,8 @@ def test___several_trivial_getter_methods(self, *mocks): ).format(outlog=expected_outlog, errlog=expected_errlog) self.assertEqual(lsf_submit.jobinfo_cmd, expected_jobinfo_cmd) self.assertEqual(lsf_submit.queue_cmd, "-q q1") + self.assertEqual(lsf_submit.proj, "proj") + self.assertEqual(lsf_submit.proj_cmd, "-P proj") self.assertEqual( lsf_submit.submit_cmd, "bsub -M {mem} -n 1 -R 'select[mem>{mem}] rusage[mem={mem}] span[hosts=1]' " From 6301451f6069352fab00a4364b122c4f9110d655 Mon Sep 17 00:00:00 2001 From: Michael Hall Date: Sat, 28 May 2022 16:47:12 +1000 Subject: [PATCH 21/33] add cluster cancel to config --- CHANGELOG.md | 1 + {{cookiecutter.profile_name}}/config.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c6a87ec..e80a83d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ This document tracks changes to the `master` branch of the profile. ### Added - Default project in cookiecutter +- Cluster cancel (`--cluster-cancel`) command (`bkill`) ### Removed diff --git a/{{cookiecutter.profile_name}}/config.yaml b/{{cookiecutter.profile_name}}/config.yaml index debc6fa..331c17c 100644 --- a/{{cookiecutter.profile_name}}/config.yaml +++ b/{{cookiecutter.profile_name}}/config.yaml @@ -7,5 +7,6 @@ restart-times: "{{cookiecutter.restart_times}}" jobs: "{{cookiecutter.jobs}}" cluster: "lsf_submit.py" cluster-status: "lsf_status.py" +cluster-cancel: "bkill" max-jobs-per-second: "{{cookiecutter.max_jobs_per_second}}" max-status-checks-per-second: "{{cookiecutter.max_status_checks_per_second}}" \ No newline at end of file From 19dfe45a911a0c6b0323d75a33201e0b0898e8d8 Mon Sep 17 00:00:00 2001 From: Michael Hall Date: Sat, 28 May 2022 16:50:33 +1000 Subject: [PATCH 22/33] update changelog after release --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e80a83d..50f9450 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ This document tracks changes to the `master` branch of the profile. ## [Unreleased] +## [0.2.0] - 28/05/2022 + ### Added - Default project in cookiecutter @@ -92,5 +94,6 @@ This document tracks changes to the `master` branch of the profile. [0.1.0]: https://github.com/Snakemake-Profiles/lsf/releases/tag/0.1.0 [0.1.1]: https://github.com/Snakemake-Profiles/lsf/releases/tag/0.1.1 [0.1.2]: https://github.com/Snakemake-Profiles/lsf/releases/tag/0.1.2 +[0.2.0]: https://github.com/Snakemake-Profiles/lsf/releases/tag/0.2.0 [Unreleased]: https://github.com/Snakemake-Profiles/lsf/compare/v0.1.2...HEAD From c0dd02401f61e9d42f1fece35d8a3c0e51c53740 Mon Sep 17 00:00:00 2001 From: Michael Hall Date: Wed, 15 Jun 2022 14:14:10 +1000 Subject: [PATCH 23/33] add cluster cancel script --- tests/test_lsf_cancel.py | 136 ++++++++++++++++++++ {{cookiecutter.profile_name}}/OSLayer.py | 4 + {{cookiecutter.profile_name}}/config.yaml | 2 +- {{cookiecutter.profile_name}}/lsf_cancel.py | 44 +++++++ 4 files changed, 185 insertions(+), 1 deletion(-) create mode 100644 tests/test_lsf_cancel.py create mode 100755 {{cookiecutter.profile_name}}/lsf_cancel.py diff --git a/tests/test_lsf_cancel.py b/tests/test_lsf_cancel.py new file mode 100644 index 0000000..9972a8e --- /dev/null +++ b/tests/test_lsf_cancel.py @@ -0,0 +1,136 @@ +import unittest +from unittest.mock import patch + +from tests.src.OSLayer import OSLayer +from tests.src.lsf_cancel import kill_jobs, parse_input, KILL + + +class TestParseInput(unittest.TestCase): + script = "lsf_cancel.py" + + def test_parse_input_no_args(self): + fake_args = [self.script] + with patch("sys.argv", fake_args): + actual = parse_input() + + assert not actual + + def test_parse_input_one_job_no_log(self): + fake_args = [self.script, "1234"] + with patch("sys.argv", fake_args): + actual = parse_input() + + expected = fake_args[1:] + assert actual == expected + + def test_parse_input_one_job_and_log(self): + fake_args = [self.script, "1234", "log/file.out"] + with patch("sys.argv", fake_args): + actual = parse_input() + + expected = [fake_args[1]] + assert actual == expected + + def test_parse_input_two_jobs_and_log(self): + fake_args = [self.script, "1234", "log/file.out", "9090", "log/other.out"] + with patch("sys.argv", fake_args): + actual = parse_input() + + expected = [fake_args[1], fake_args[3]] + assert actual == expected + + def test_parse_input_two_jobs_and_digits_in_log(self): + fake_args = [self.script, "1234", "log/file.out", "9090", "log/123"] + with patch("sys.argv", fake_args): + actual = parse_input() + + expected = [fake_args[1], fake_args[3]] + assert actual == expected + + def test_parse_input_multiple_args_but_no_jobs(self): + fake_args = [self.script, "log/file.out", "log/123"] + with patch("sys.argv", fake_args): + actual = parse_input() + + assert not actual + + +class TestKillJobs(unittest.TestCase): + @patch.object( + OSLayer, + OSLayer.run_process.__name__, + return_value=( + "Job <123> is being terminated", + "", + ), + ) + def test_kill_jobs_one_job( + self, + run_process_mock, + ): + jobids = ["123"] + expected_kill_cmd = "{} {}".format(KILL, " ".join(jobids)) + + kill_jobs(jobids) + + run_process_mock.assert_called_once_with(expected_kill_cmd) + + @patch.object( + OSLayer, + OSLayer.run_process.__name__, + return_value=( + "Job <123> is being terminated\nJob <456> is being terminated", + "", + ), + ) + def test_kill_jobs_two_jobs( + self, + run_process_mock, + ): + jobids = ["123", "456"] + expected_kill_cmd = "{} {}".format(KILL, " ".join(jobids)) + + kill_jobs(jobids) + + run_process_mock.assert_called_once_with(expected_kill_cmd) + + @patch.object( + OSLayer, + OSLayer.run_process.__name__, + return_value=("", ""), + ) + def test_kill_jobs_no_jobs( + self, + run_process_mock, + ): + jobids = [] + + kill_jobs(jobids) + + run_process_mock.assert_not_called() + + @patch.object( + OSLayer, + OSLayer.run_process.__name__, + return_value=("", ""), + ) + def test_kill_jobs_empty_jobs(self, run_process_mock): + jobids = ["", ""] + + kill_jobs(jobids) + + run_process_mock.assert_not_called() + + @patch.object( + OSLayer, + OSLayer.run_process.__name__, + return_value=("", ""), + ) + def test_kill_jobs_empty_job_and_non_empty_job(self, run_process_mock): + jobids = ["", "123"] + + expected_kill_cmd = "{} {}".format(KILL, " ".join(jobids)) + + kill_jobs(jobids) + + run_process_mock.assert_called_once_with(expected_kill_cmd) diff --git a/{{cookiecutter.profile_name}}/OSLayer.py b/{{cookiecutter.profile_name}}/OSLayer.py index 187af60..349b081 100644 --- a/{{cookiecutter.profile_name}}/OSLayer.py +++ b/{{cookiecutter.profile_name}}/OSLayer.py @@ -49,6 +49,10 @@ def run_process(cmd: str) -> Tuple[stdout, stderr]: def print(string: str): print(string) + @staticmethod + def eprint(string: str): + print(string, file=sys.stderr) + @staticmethod def get_uuid4_string() -> str: return str(uuid.uuid4()) diff --git a/{{cookiecutter.profile_name}}/config.yaml b/{{cookiecutter.profile_name}}/config.yaml index 331c17c..0fb590d 100644 --- a/{{cookiecutter.profile_name}}/config.yaml +++ b/{{cookiecutter.profile_name}}/config.yaml @@ -7,6 +7,6 @@ restart-times: "{{cookiecutter.restart_times}}" jobs: "{{cookiecutter.jobs}}" cluster: "lsf_submit.py" cluster-status: "lsf_status.py" -cluster-cancel: "bkill" +cluster-cancel: "lsf_cancel.py" max-jobs-per-second: "{{cookiecutter.max_jobs_per_second}}" max-status-checks-per-second: "{{cookiecutter.max_status_checks_per_second}}" \ No newline at end of file diff --git a/{{cookiecutter.profile_name}}/lsf_cancel.py b/{{cookiecutter.profile_name}}/lsf_cancel.py new file mode 100755 index 0000000..409deec --- /dev/null +++ b/{{cookiecutter.profile_name}}/lsf_cancel.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +import re +import shlex +import sys +from pathlib import Path +from typing import List + +if not __name__.startswith("tests.src."): + sys.path.append(str(Path(__file__).parent.absolute())) + from OSLayer import OSLayer +else: + from .OSLayer import OSLayer + +KILL = "bkill" + + +def kill_jobs(ids_to_kill: List[str]): + # we don't want to run bkill with no argument as this will kill the last job + if any(ids_to_kill): + cmd = "{} {}".format(KILL, " ".join(ids_to_kill)) + _ = OSLayer.run_process(cmd) + + +def parse_input() -> List[str]: + # need to support quoted and unquoted jobid + # see https://github.com/Snakemake-Profiles/lsf/issues/45 + split_args = shlex.split(" ".join(sys.argv[1:])) + valid_ids = [] + for arg in map(str.strip, split_args): + if re.fullmatch(r"\d+", arg): + valid_ids.append(arg) + + return valid_ids + + +if __name__ == "__main__": + jobids = parse_input() + + if jobids: + kill_jobs(jobids) + else: + OSLayer.eprint( + "[cluster-cancel error] Did not get any valid jobids to cancel..." + ) From 1e0df9a59e23e968974c446ccfbb100377df2501 Mon Sep 17 00:00:00 2001 From: Michael Hall Date: Wed, 15 Jun 2022 14:17:26 +1000 Subject: [PATCH 24/33] Update CHANGELOG.md --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 50f9450..9899b3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ This document tracks changes to the `master` branch of the profile. ## [Unreleased] +### Changed +- Cluster cancel is now a script instead of the `bkill` command in order to handle the log file paths that come with the job ID [[#55][55]] + ## [0.2.0] - 28/05/2022 ### Added @@ -91,6 +94,7 @@ This document tracks changes to the `master` branch of the profile. [36]: https://github.com/Snakemake-Profiles/lsf/issues/36 [39]: https://github.com/Snakemake-Profiles/lsf/issues/39 [45]: https://github.com/Snakemake-Profiles/lsf/issues/45 +[55]: https://github.com/Snakemake-Profiles/lsf/issues/55 [0.1.0]: https://github.com/Snakemake-Profiles/lsf/releases/tag/0.1.0 [0.1.1]: https://github.com/Snakemake-Profiles/lsf/releases/tag/0.1.1 [0.1.2]: https://github.com/Snakemake-Profiles/lsf/releases/tag/0.1.2 From b74cabfee02f1019312915017c5fd86d9c4956b8 Mon Sep 17 00:00:00 2001 From: Michael Hall Date: Wed, 15 Jun 2022 15:01:51 +1000 Subject: [PATCH 25/33] Allow bkill to silently fail --- tests/test_lsf_cancel.py | 6 +++--- {{cookiecutter.profile_name}}/OSLayer.py | 4 ++-- {{cookiecutter.profile_name}}/lsf_cancel.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_lsf_cancel.py b/tests/test_lsf_cancel.py index 9972a8e..71919d2 100644 --- a/tests/test_lsf_cancel.py +++ b/tests/test_lsf_cancel.py @@ -73,7 +73,7 @@ def test_kill_jobs_one_job( kill_jobs(jobids) - run_process_mock.assert_called_once_with(expected_kill_cmd) + run_process_mock.assert_called_once_with(expected_kill_cmd, check=False) @patch.object( OSLayer, @@ -92,7 +92,7 @@ def test_kill_jobs_two_jobs( kill_jobs(jobids) - run_process_mock.assert_called_once_with(expected_kill_cmd) + run_process_mock.assert_called_once_with(expected_kill_cmd, check=False) @patch.object( OSLayer, @@ -133,4 +133,4 @@ def test_kill_jobs_empty_job_and_non_empty_job(self, run_process_mock): kill_jobs(jobids) - run_process_mock.assert_called_once_with(expected_kill_cmd) + run_process_mock.assert_called_once_with(expected_kill_cmd, check=False) diff --git a/{{cookiecutter.profile_name}}/OSLayer.py b/{{cookiecutter.profile_name}}/OSLayer.py index 349b081..9e5b8c5 100644 --- a/{{cookiecutter.profile_name}}/OSLayer.py +++ b/{{cookiecutter.profile_name}}/OSLayer.py @@ -36,9 +36,9 @@ def remove_file(file: Path): file.unlink() @staticmethod - def run_process(cmd: str) -> Tuple[stdout, stderr]: + def run_process(cmd: str, check: bool = True) -> Tuple[stdout, stderr]: completed_process = subprocess.run( - cmd, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE + cmd, check=check, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) return ( completed_process.stdout.decode().strip(), diff --git a/{{cookiecutter.profile_name}}/lsf_cancel.py b/{{cookiecutter.profile_name}}/lsf_cancel.py index 409deec..48399fa 100755 --- a/{{cookiecutter.profile_name}}/lsf_cancel.py +++ b/{{cookiecutter.profile_name}}/lsf_cancel.py @@ -18,7 +18,7 @@ def kill_jobs(ids_to_kill: List[str]): # we don't want to run bkill with no argument as this will kill the last job if any(ids_to_kill): cmd = "{} {}".format(KILL, " ".join(ids_to_kill)) - _ = OSLayer.run_process(cmd) + _ = OSLayer.run_process(cmd, check=False) def parse_input() -> List[str]: From 3198c662d4756bc7210a2ad116af7db2240dddf8 Mon Sep 17 00:00:00 2001 From: Michael Hall Date: Wed, 15 Jun 2022 15:22:05 +1000 Subject: [PATCH 26/33] expose max status checks and wait between tries --- CHANGELOG.md | 4 ++++ README.md | 12 ++++++++++++ cookiecutter.json | 2 ++ {{cookiecutter.profile_name}}/CookieCutter.py | 8 ++++++++ {{cookiecutter.profile_name}}/lsf_status.py | 7 ++++++- 5 files changed, 32 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9899b3c..fb7602d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ This document tracks changes to the `master` branch of the profile. ## [Unreleased] +### Added +- Exposed `max_status_check` and `wait_between_tries` for status checker [[#48][48]] + ### Changed - Cluster cancel is now a script instead of the `bkill` command in order to handle the log file paths that come with the job ID [[#55][55]] @@ -94,6 +97,7 @@ This document tracks changes to the `master` branch of the profile. [36]: https://github.com/Snakemake-Profiles/lsf/issues/36 [39]: https://github.com/Snakemake-Profiles/lsf/issues/39 [45]: https://github.com/Snakemake-Profiles/lsf/issues/45 +[48]: https://github.com/Snakemake-Profiles/lsf/issues/48 [55]: https://github.com/Snakemake-Profiles/lsf/issues/55 [0.1.0]: https://github.com/Snakemake-Profiles/lsf/releases/tag/0.1.0 [0.1.1]: https://github.com/Snakemake-Profiles/lsf/releases/tag/0.1.1 diff --git a/README.md b/README.md index 7f6a26e..54ea7bf 100644 --- a/README.md +++ b/README.md @@ -254,6 +254,18 @@ From the `snakemake --help` menu default is 10, fractions allowed. ``` +#### `max_status_checks` + +**Default**: `1` + +How many times to check the status of a job. + +#### `wait_between_tries` + +**Default**: `0.001` + +How many seconds to wait until checking the status of a job again (if `max_status_checks` is greater than 1). + #### `profile_name` **Default**: `lsf` diff --git a/cookiecutter.json b/cookiecutter.json index 8c38871..14b4b71 100644 --- a/cookiecutter.json +++ b/cookiecutter.json @@ -28,5 +28,7 @@ "default_project": "", "max_status_checks_per_second": 10, "max_jobs_per_second": 10, + "max_status_checks": 1, + "wait_between_tries": 0.001, "profile_name": "lsf" } \ No newline at end of file diff --git a/{{cookiecutter.profile_name}}/CookieCutter.py b/{{cookiecutter.profile_name}}/CookieCutter.py index 58d2729..5acf8f7 100644 --- a/{{cookiecutter.profile_name}}/CookieCutter.py +++ b/{{cookiecutter.profile_name}}/CookieCutter.py @@ -34,3 +34,11 @@ def get_zombi_behaviour() -> str: @staticmethod def get_latency_wait() -> float: return float("{{cookiecutter.latency_wait}}") + + @staticmethod + def get_wait_between_tries() -> float: + return float("{{cookiecutter.wait_between_tries}}") + + @staticmethod + def get_max_status_checks() -> int: + return int("{{cookiecutter.max_status_checks}}") diff --git a/{{cookiecutter.profile_name}}/lsf_status.py b/{{cookiecutter.profile_name}}/lsf_status.py index a0d04e9..b96699a 100755 --- a/{{cookiecutter.profile_name}}/lsf_status.py +++ b/{{cookiecutter.profile_name}}/lsf_status.py @@ -227,6 +227,11 @@ def get_status(self) -> str: ) lsf_status_checker = StatusChecker( - jobid, outlog, kill_unknown=kill_unknown, kill_zombie=kill_zombie + jobid, + outlog, + kill_unknown=kill_unknown, + kill_zombie=kill_zombie, + wait_between_tries=CookieCutter.get_wait_between_tries(), + max_status_checks=CookieCutter.get_max_status_checks(), ) print(lsf_status_checker.get_status()) From 4a681b0ed42737e311d11209b2626c4c965f4bd2 Mon Sep 17 00:00:00 2001 From: Michael Hall Date: Tue, 12 Jul 2022 10:00:03 +1000 Subject: [PATCH 27/33] add maintenance notice --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 7f6a26e..b4eb4ee 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,8 @@ ![Python versions](https://img.shields.io/badge/Python%20versions->=3.5-blue) ![License](https://img.shields.io/github/license/Snakemake-Profiles/lsf) +> 📢**NOTICE: We are seeking volunteers to maintain this repository as the current maintainers no longer use LSF. See [this issue](https://github.com/Snakemake-Profiles/lsf/issues/57).**📢 + [Snakemake profile][profile] for running jobs on an [LSF][lsf] cluster. [TOC]: # From 9abd00ed08f2484352ab496b75fe86218df76475 Mon Sep 17 00:00:00 2001 From: Michael Hall Date: Tue, 12 Jul 2022 10:00:28 +1000 Subject: [PATCH 28/33] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b4eb4ee..ee7e208 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ ![Python versions](https://img.shields.io/badge/Python%20versions->=3.5-blue) ![License](https://img.shields.io/github/license/Snakemake-Profiles/lsf) -> 📢**NOTICE: We are seeking volunteers to maintain this repository as the current maintainers no longer use LSF. See [this issue](https://github.com/Snakemake-Profiles/lsf/issues/57).**📢 +> 📢 **NOTICE: We are seeking volunteers to maintain this repository as the current maintainers no longer use LSF. See [this issue](https://github.com/Snakemake-Profiles/lsf/issues/57).** 📢 [Snakemake profile][profile] for running jobs on an [LSF][lsf] cluster. From 1bdd36a7041ae6952ec0278cc0200e5a78842bbe Mon Sep 17 00:00:00 2001 From: Michael Hall Date: Wed, 13 Jul 2022 09:06:24 +1000 Subject: [PATCH 29/33] Update CHANGELOG.md --- CHANGELOG.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fb7602d..0e5333f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ This document tracks changes to the `master` branch of the profile. ## [Unreleased] +## [0.3.0] - 13/07/2022 + ### Added - Exposed `max_status_check` and `wait_between_tries` for status checker [[#48][48]] @@ -103,5 +105,6 @@ This document tracks changes to the `master` branch of the profile. [0.1.1]: https://github.com/Snakemake-Profiles/lsf/releases/tag/0.1.1 [0.1.2]: https://github.com/Snakemake-Profiles/lsf/releases/tag/0.1.2 [0.2.0]: https://github.com/Snakemake-Profiles/lsf/releases/tag/0.2.0 -[Unreleased]: https://github.com/Snakemake-Profiles/lsf/compare/v0.1.2...HEAD +[0.3.0]: https://github.com/Snakemake-Profiles/lsf/compare/0.2.0...0.3.0 +[Unreleased]: https://github.com/Snakemake-Profiles/lsf/compare/0.3.0...HEAD From b32c7a9ba98a1688b9a8096c7bb68d4a4528c459 Mon Sep 17 00:00:00 2001 From: Michael Hall Date: Wed, 14 Dec 2022 08:40:15 +1000 Subject: [PATCH 30/33] add jobscript timeout [#58] --- cookiecutter.json | 1 + {{cookiecutter.profile_name}}/CookieCutter.py | 4 ++++ {{cookiecutter.profile_name}}/lsf_submit.py | 12 ++++++++++++ 3 files changed, 17 insertions(+) diff --git a/cookiecutter.json b/cookiecutter.json index 14b4b71..32208d1 100644 --- a/cookiecutter.json +++ b/cookiecutter.json @@ -30,5 +30,6 @@ "max_jobs_per_second": 10, "max_status_checks": 1, "wait_between_tries": 0.001, + "jobscript_timeout": 10, "profile_name": "lsf" } \ No newline at end of file diff --git a/{{cookiecutter.profile_name}}/CookieCutter.py b/{{cookiecutter.profile_name}}/CookieCutter.py index 5acf8f7..53af9f1 100644 --- a/{{cookiecutter.profile_name}}/CookieCutter.py +++ b/{{cookiecutter.profile_name}}/CookieCutter.py @@ -42,3 +42,7 @@ def get_wait_between_tries() -> float: @staticmethod def get_max_status_checks() -> int: return int("{{cookiecutter.max_status_checks}}") + + @staticmethod + def jobscript_timeout() -> int: + return int("{{cookiecutter.jobscript_timeout}}") diff --git a/{{cookiecutter.profile_name}}/lsf_submit.py b/{{cookiecutter.profile_name}}/lsf_submit.py index 9b01450..61ee9d1 100755 --- a/{{cookiecutter.profile_name}}/lsf_submit.py +++ b/{{cookiecutter.profile_name}}/lsf_submit.py @@ -1,8 +1,10 @@ #!/usr/bin/env python3 import math +import os import re import subprocess import sys +import time from pathlib import Path from typing import List, Union, Optional @@ -213,6 +215,16 @@ def _remove_previous_logs(self): OSLayer.remove_file(self.errlog) def _submit_cmd_and_get_external_job_id(self) -> int: + start_time = time.time() + # Wait for the jobscript to be created + while not os.path.exists(self.jobscript): + if time.time() - start_time >= CookieCutter.jobscript_timeout(): + # The file was not created within the specified timeout period + raise TimeoutError("Timed out waiting for jobscript to be created") + + # Sleep for a short amount of time before checking again + time.sleep(0.1) + output_stream, error_stream = OSLayer.run_process(self.submit_cmd) match = re.search(r"Job <(\d+)> is submitted", output_stream) jobid = match.group(1) From 8c4436ba777c6954670998c5e5db4255b9825fef Mon Sep 17 00:00:00 2001 From: Michael Hall Date: Wed, 14 Dec 2022 08:41:32 +1000 Subject: [PATCH 31/33] remove python3.6 from actions --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 37cd124..4a4b885 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -32,7 +32,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [ 3.6, 3.7, 3.8 ] + python-version: [ 3.7, 3.8, 3.9, '3.10' ] steps: - uses: actions/checkout@v2 From ffc9494e8773876ec4772935cc54956e62eb94fc Mon Sep 17 00:00:00 2001 From: Michael Hall Date: Wed, 14 Dec 2022 08:42:28 +1000 Subject: [PATCH 32/33] remove python3.6 from actions --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4a4b885..bb2a6bd 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [ 3.6, 3.7, 3.8 ] + python-version: [ 3.7, 3.8, 3.9, '3.10' ] steps: - uses: actions/checkout@v2 @@ -52,7 +52,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [ 3.6, 3.7, 3.8 ] + python-version: [ 3.7, 3.8, 3.9, '3.10' ] steps: - uses: actions/checkout@v2 From d9116a228c77258da81c55d8072ab5534e6ba86b Mon Sep 17 00:00:00 2001 From: Michael Hall Date: Wed, 14 Dec 2022 08:46:08 +1000 Subject: [PATCH 33/33] update changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e5333f..c1bc525 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ This document tracks changes to the `master` branch of the profile. ## [Unreleased] +### Added + +- `jobscript_timeout` cookiecutter variable that sets the number of seconds to wait for the jobscript to exist before submiting the job + ## [0.3.0] - 13/07/2022 ### Added