From 994b00db38393649ab2cad021339ebb0c2a740d7 Mon Sep 17 00:00:00 2001 From: Christine Simpson Date: Wed, 10 May 2023 14:00:29 -0500 Subject: [PATCH 1/5] added all option to balsam app rm --- balsam/cmdline/app.py | 76 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 64 insertions(+), 12 deletions(-) diff --git a/balsam/cmdline/app.py b/balsam/cmdline/app.py index 1ed8bd21..957655ad 100644 --- a/balsam/cmdline/app.py +++ b/balsam/cmdline/app.py @@ -45,20 +45,72 @@ def ls(site_selector: str, verbose: bool) -> None: @app.command() -@click.option("-n", "--name", required=True) +@click.option("-n", "--name", "name_selector", default=None) @click.option("-s", "--site", "site_selector", default="") -def rm(site_selector: str, name: str) -> None: +@click.option("-a", "--all", is_flag=True, default=False) +def rm(site_selector: str, name_selector: str, all: bool) -> None: + """ + Remove Apps + + 1) Remove named app + + balsam app rm -n hello_world + + 1) Remove all apps across a site + + balsam app rm --all --site=123,my_site_folder + + 2) Filter apps by specific site IDs or Path fragments + + balsam app rm -n hello_world --site=123,my_site_folder + + """ client = ClientSettings.load_from_file().build_client() qs = client.App.objects.all() qs = filter_by_sites(qs, site_selector) - resolved_app = qs.get(name=name) - resolved_id = resolved_app.id - appstr = f"App(id={resolved_id}, name={resolved_app.name})" - job_count = client.Job.objects.filter(app_id=resolved_id).count() - if job_count == 0: - resolved_app.delete() - click.echo(f"Deleted {appstr}: there were no associated jobs.") - elif click.confirm(f"Really Delete {appstr}?? There are {job_count} Jobs that will be ERASED!"): - resolved_app.delete() - click.echo(f"Deleted App {resolved_id} ({name})") + if all and name_selector is not None: + raise click.BadParameter("Specify app name or --all, but not both") + elif not all and name_selector is None: + raise click.BadParameter("Specify app name with -n or specify --all") + else: + app_list = [] + + if all and site_selector == "": + raise click.BadParameter("balsam app rm --all requires that you specify --site to remove jobs") + elif all and site_selector != "": + click.echo("THIS WILL DELETE ALL APPS IN SITE! CAUTION!") + app_list = [a.name for a in list(qs)] + num_apps = 0 + num_jobs = 0 + elif name_selector is not None: + app_list = [name_selector] + + if len(app_list) > 0: + for name in app_list: + resolved_app = qs.get(name=name) + resolved_id = resolved_app.id + job_count = client.Job.objects.filter(app_id=resolved_id).count() + + if name_selector is not None: + appstr = f"App(id={resolved_id}, name={resolved_app.name}, site={resolved_app.site_id})" + if job_count == 0: + resolved_app.delete() + click.echo(f"Deleted {appstr}: there were no associated jobs.") + elif click.confirm(f"Really Delete {appstr}?? There are {job_count} Jobs that will be ERASED!"): + resolved_app.delete() + click.echo(f"Deleted App {resolved_id} ({name})") + else: + num_apps += 1 + num_jobs += job_count + + if all: + if click.confirm( + f"Really DELETE {num_apps} apps and {num_jobs} jobs from site {site_selector}?? They will be ERASED!" + ): + for name in app_list: + resolved_app = qs.get(name=name) + resolved_app.delete() + click.echo(f"Deleted {num_apps} apps and {num_jobs} jobs from site {site_selector}") + else: + click.echo("Found no apps to Delete") From 3ca0ec28b59efc5c0dfa0ec3e838d4dc1ad54dd4 Mon Sep 17 00:00:00 2001 From: Christine Simpson Date: Wed, 10 May 2023 15:36:10 -0500 Subject: [PATCH 2/5] updates to balsam queue ls --- balsam/cmdline/scheduler.py | 102 +++++++++++++++++++++++------------- 1 file changed, 65 insertions(+), 37 deletions(-) diff --git a/balsam/cmdline/scheduler.py b/balsam/cmdline/scheduler.py index 4bf14296..6e9ea80e 100644 --- a/balsam/cmdline/scheduler.py +++ b/balsam/cmdline/scheduler.py @@ -84,9 +84,12 @@ def submit( @queue.command() +@click.option("-n", "--num", default=3, type=int) @click.option("-h", "--history", is_flag=True, default=False) +@click.option("-v", "--verbose", is_flag=True, default=False) @click.option("--site", "site_selector", default="") -def ls(history: bool, site_selector: str) -> None: +@click.option("--id", "scheduler_id", type=int, default=None) +def ls(history: bool, verbose: bool, num: int, site_selector: str, scheduler_id: int) -> None: """ List BatchJobs @@ -97,49 +100,74 @@ def ls(history: bool, site_selector: str) -> None: 2) View historical BatchJobs at all sites balsam queue ls --history --site all + + 3) View verbose record for BatchJob with scheduler id + + balsam queue ls --id 12345 -v + + 4) View the last n BatchJobs + + balsam queue ls --num n + """ client = load_client() BatchJob = client.BatchJob qs = filter_by_sites(BatchJob.objects.all(), site_selector) + + active_only = False if not history: - qs = qs.filter(state=["pending_submission", "queued", "running", "pending_deletion"]) + active_only = True + qs_filter = qs.filter(state=["pending_submission", "queued", "running", "pending_deletion"]) + if len(qs_filter) > 0 or num == 0: + qs = qs_filter + + if scheduler_id is not None: + qs = qs.filter(scheduler_id=scheduler_id) jobs = [j.display_dict() for j in qs] - sites = {site.id: site for site in client.Site.objects.all()} - for job in jobs: - site = sites[job["site_id"]] - path_str = site.path.as_posix() - if len(path_str) > 27: - path_str = "..." + path_str[-27:] - job["site"] = f"{site.name}" - - fields = [ - "id", - "site", - "scheduler_id", - "state", - "filter_tags", - "project", - "queue", - "num_nodes", - "wall_time_min", - "job_mode", - ] - rows = [[str(j[field]) for field in fields] for j in jobs] - - col_widths = [len(f) for f in fields] - for row in rows: - for col_idx, width in enumerate(col_widths): - col_widths[col_idx] = max(width, len(row[col_idx])) - - for i, field in enumerate(fields): - fields[i] = field.rjust(col_widths[i] + 1) - - print(*fields) - for row in rows: - for i, col in enumerate(row): - row[i] = col.rjust(col_widths[i] + 1) - print(*row) + if active_only and num > 0: + click.echo(f"No active Batch Jobs. Displaying records for last {num} Batch Jobs") + jobs = jobs[-num:] + + if verbose: + for j in jobs: + click.echo(j) + else: + sites = {site.id: site for site in client.Site.objects.all()} + for job in jobs: + site = sites[job["site_id"]] + path_str = site.path.as_posix() + if len(path_str) > 27: + path_str = "..." + path_str[-27:] + job["site"] = f"{site.name}" + + fields = [ + "id", + "site", + "scheduler_id", + "state", + "filter_tags", + "project", + "queue", + "num_nodes", + "wall_time_min", + "job_mode", + ] + rows = [[str(j[field]) for field in fields] for j in jobs] + + col_widths = [len(f) for f in fields] + for row in rows: + for col_idx, width in enumerate(col_widths): + col_widths[col_idx] = max(width, len(row[col_idx])) + + for i, field in enumerate(fields): + fields[i] = field.rjust(col_widths[i] + 1) + + print(*fields) + for row in rows: + for i, col in enumerate(row): + row[i] = col.rjust(col_widths[i] + 1) + print(*row) @queue.command() From 62f0e69f28b2c1fe9f6104a04a6f084883a9617e Mon Sep 17 00:00:00 2001 From: Christine Simpson Date: Wed, 10 May 2023 15:52:55 -0500 Subject: [PATCH 3/5] more updates to balsam queue ls --- balsam/cmdline/scheduler.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/balsam/cmdline/scheduler.py b/balsam/cmdline/scheduler.py index 6e9ea80e..6aca8e48 100644 --- a/balsam/cmdline/scheduler.py +++ b/balsam/cmdline/scheduler.py @@ -114,18 +114,16 @@ def ls(history: bool, verbose: bool, num: int, site_selector: str, scheduler_id: BatchJob = client.BatchJob qs = filter_by_sites(BatchJob.objects.all(), site_selector) - active_only = False if not history: - active_only = True qs_filter = qs.filter(state=["pending_submission", "queued", "running", "pending_deletion"]) - if len(qs_filter) > 0 or num == 0: + if (len(qs_filter) > 0 and scheduler_id is None) or num == 0: qs = qs_filter if scheduler_id is not None: qs = qs.filter(scheduler_id=scheduler_id) jobs = [j.display_dict() for j in qs] - if active_only and num > 0: + if not history and num > 0 and scheduler_id is None: click.echo(f"No active Batch Jobs. Displaying records for last {num} Batch Jobs") jobs = jobs[-num:] From 2016f51ceeb21668b0da2f44466fe3f7e6a23ad6 Mon Sep 17 00:00:00 2001 From: Christine Simpson Date: Wed, 10 May 2023 16:15:05 -0500 Subject: [PATCH 4/5] updates to balsam job modify --- balsam/cmdline/job.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/balsam/cmdline/job.py b/balsam/cmdline/job.py index eb2943d5..ac8bffd1 100644 --- a/balsam/cmdline/job.py +++ b/balsam/cmdline/job.py @@ -315,8 +315,9 @@ def ls( @job.command() @click.option("-i", "--id", "job_ids", multiple=True, type=int) +@click.option("-t", "--tag", "tags", multiple=True, type=str, callback=validate_tags) @click.option("-s", "--state", "state", type=str) -def modify(job_ids: List[int], state: JobState) -> None: +def modify(job_ids: List[int], tags: List[str], state: JobState) -> None: """ Modify Jobs @@ -328,6 +329,8 @@ def modify(job_ids: List[int], state: JobState) -> None: jobs = client.Job.objects.all() if job_ids: jobs = jobs.filter(id=job_ids) + elif tags: + jobs = jobs.filter(tags=tags) else: raise click.BadParameter("Provide either list of Job ids or tags to delete") count = jobs.count() From 6112efe9a48a705daf0c7c0d91e84248679da830 Mon Sep 17 00:00:00 2001 From: Christine Simpson Date: Wed, 10 May 2023 19:16:03 -0500 Subject: [PATCH 5/5] added forced site deletion --- balsam/cmdline/site.py | 42 ++++++++++++++++++++++++++++++------------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/balsam/cmdline/site.py b/balsam/cmdline/site.py index f7ffbffe..4ba39ffe 100644 --- a/balsam/cmdline/site.py +++ b/balsam/cmdline/site.py @@ -1,3 +1,4 @@ +import os import shutil import socket import sys @@ -128,23 +129,40 @@ def mv(src: Union[Path, str], dest: Union[Path, str]) -> None: @site.command() -@click.argument("path", type=click.Path(exists=True, file_okay=False)) -def rm(path: Union[str, Path]) -> None: +# @click.argument("path", type=click.Path(exists=True, file_okay=False)) +@click.argument("path", type=click.Path()) +@click.option("-f", "--force", is_flag=True, default=False) +def rm(path: str, force: bool) -> None: """ Remove a balsam site balsam site rm /path/to/site """ - cf = SiteConfig(path) - client = cf.client - site = client.Site.objects.get(id=cf.site_id) - jobcount = client.Job.objects.filter(site_id=site.id).count() - warning = f"This will wipe out {jobcount} jobs inside!" if jobcount else "" - - if click.confirm(f"Do you really want to destroy {Path(path).name}? {warning}"): - site.delete() - shutil.rmtree(path) - click.echo(f"Deleted site {path}") + if not force: + if os.path.exists(path): + cf = SiteConfig(path) + client = cf.client + site = client.Site.objects.get(id=cf.site_id) + jobcount = client.Job.objects.filter(site_id=site.id).count() + warning = f"This will wipe out {jobcount} jobs inside!" if jobcount else "" + + if click.confirm(f"Do you really want to destroy {Path(path).name}? {warning}"): + site.delete() + shutil.rmtree(path) + click.echo(f"Deleted site {path}") + else: + raise click.BadParameter("Path doesn't exist") + else: + client = ClientSettings.load_from_file().build_client() + qs = client.Site.objects.all() + qs = qs.filter(path=path) + if len(qs) > 1: + raise click.BadParameter(f"Path found in {len(qs)} sites") + else: + site_id = qs[0].id + site = client.Site.objects.get(id=site_id) + site.delete() + click.echo("Forced site deletion; check for path to clean up") @site.command()