From 2ea81c031662fb94390c8f91ccd0577e8d76ec87 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Thu, 26 Oct 2017 03:57:02 +0100 Subject: [PATCH] added file regex option for retrieving runs --- atomic_hpc/__init__.py | 2 +- atomic_hpc/deploy_runs.py | 6 ++++-- bin/retrieve_config | 8 ++++++-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/atomic_hpc/__init__.py b/atomic_hpc/__init__.py index da0d9ac..4fc550a 100644 --- a/atomic_hpc/__init__.py +++ b/atomic_hpc/__init__.py @@ -1,4 +1,4 @@ from atomic_hpc import config_yaml, deploy_runs, mockssh, context_folder, utils -__version__ = "0.2.0" +__version__ = "0.2.1" diff --git a/atomic_hpc/deploy_runs.py b/atomic_hpc/deploy_runs.py index 4d8efd9..3f3f9d7 100644 --- a/atomic_hpc/deploy_runs.py +++ b/atomic_hpc/deploy_runs.py @@ -648,7 +648,7 @@ def deploy_run_qsub(run, inputs, root_path, if_exists="abort", exec_errors=False # TODO add tests for retrieve_outputs -def retrieve_outputs(runs, local_path, root_path, if_exists="abort"): +def retrieve_outputs(runs, local_path, root_path, if_exists="abort", path_regex="*"): """ Parameters @@ -661,6 +661,8 @@ def retrieve_outputs(runs, local_path, root_path, if_exists="abort"): the path of the config file if_exists: ["abort", "remove", "use"] either; raise an IOError if the output path already exists, remove the output path or use it without change + path_regex: str + regex to search for files Returns ------- @@ -723,7 +725,7 @@ def retrieve_outputs(runs, local_path, root_path, if_exists="abort"): continue logger.info("copying {0} to {1}".format(outname, local_path)) - for pname in folder.glob(os.path.join(outname, "*")): + for pname in folder.glob(os.path.join(outname, path_regex)): folder.copy_to(pname, local_path.joinpath(outname)) logger.info("finished copying {0} to {1}".format(outname, local_path)) diff --git a/bin/retrieve_config b/bin/retrieve_config index 47c3fe9..cdd4880 100644 --- a/bin/retrieve_config +++ b/bin/retrieve_config @@ -12,7 +12,7 @@ from jsonschema import ValidationError logger = logging.getLogger('atomic_hpc.retrieve_config') -def run(fpath, runs=None, outpath="", basepath="", log_level='INFO', if_exists="abort"): +def run(fpath, runs=None, outpath="", basepath="", log_level='INFO', if_exists="abort", path_regex="*"): """ Parameters @@ -24,6 +24,8 @@ def run(fpath, runs=None, outpath="", basepath="", log_level='INFO', if_exists=" log_level: str if_exists: ["abort", "remove", "use"] either; raise an IOError if the output path already exists, remove the output path or use it without change + path_regex: str + regex to search for files Returns ------- @@ -68,7 +70,7 @@ def run(fpath, runs=None, outpath="", basepath="", log_level='INFO', if_exists=" runs_to_deploy = [r for r in runs_to_deploy if r["id"] in runs] try: - retrieve_outputs(runs_to_deploy, outpath, basepath, if_exists=if_exists) + retrieve_outputs(runs_to_deploy, outpath, basepath, if_exists=if_exists, path_regex=path_regex) except RuntimeError as err: logger.critical(err) @@ -105,6 +107,8 @@ if __name__ == "__main__": choices=['debug_full', 'debug', 'info', 'exec', 'warning', 'error'], help='the logging level to output to screen/file ' '(NB: debug_full allows logging from external packages)') + parser.add_argument("-rx", "--path-regex", type=str, metavar='str', + help='file/folder regex to retrieve', default="*") # parser.add_argument("--test-run", action="store_true", # help='do not run any executables (only create directories and create/copy files)')