added file regex option for retrieving runs

atomic_hpc/__init__.py

@@ -1,4 +1,4 @@
 from atomic_hpc import config_yaml, deploy_runs, mockssh, context_folder, utils
 
-__version__ = "0.2.0"
+__version__ = "0.2.1"
 

atomic_hpc/deploy_runs.py

@@ -648,7 +648,7 @@ def deploy_run_qsub(run, inputs, root_path, if_exists="abort", exec_errors=False
 
 
 # TODO add tests for retrieve_outputs
-def retrieve_outputs(runs, local_path, root_path, if_exists="abort"):
+def retrieve_outputs(runs, local_path, root_path, if_exists="abort", path_regex="*"):
     """
 
     Parameters
@@ -661,6 +661,8 @@ def retrieve_outputs(runs, local_path, root_path, if_exists="abort"):
         the path of the config file
     if_exists: ["abort", "remove", "use"]
         either; raise an IOError if the output path already exists, remove the output path or use it without change
+    path_regex: str
+        regex to search for files
 
     Returns
     -------
@@ -723,7 +725,7 @@ def retrieve_outputs(runs, local_path, root_path, if_exists="abort"):
                 continue
 
             logger.info("copying {0} to {1}".format(outname, local_path))
-            for pname in folder.glob(os.path.join(outname, "*")):
+            for pname in folder.glob(os.path.join(outname, path_regex)):
                 folder.copy_to(pname, local_path.joinpath(outname))
 
             logger.info("finished copying {0} to {1}".format(outname, local_path))

bin/retrieve_config

@@ -12,7 +12,7 @@ from jsonschema import ValidationError
 logger = logging.getLogger('atomic_hpc.retrieve_config')
 
 
-def run(fpath, runs=None, outpath="", basepath="", log_level='INFO', if_exists="abort"):
+def run(fpath, runs=None, outpath="", basepath="", log_level='INFO', if_exists="abort", path_regex="*"):
     """
 
     Parameters
@@ -24,6 +24,8 @@ def run(fpath, runs=None, outpath="", basepath="", log_level='INFO', if_exists="
     log_level: str
     if_exists: ["abort", "remove", "use"]
         either; raise an IOError if the output path already exists, remove the output path or use it without change
+   path_regex: str
+        regex to search for files
 
     Returns
     -------
@@ -68,7 +70,7 @@ def run(fpath, runs=None, outpath="", basepath="", log_level='INFO', if_exists="
         runs_to_deploy = [r for r in runs_to_deploy if r["id"] in runs]
 
     try:
-        retrieve_outputs(runs_to_deploy, outpath, basepath, if_exists=if_exists)
+        retrieve_outputs(runs_to_deploy, outpath, basepath, if_exists=if_exists, path_regex=path_regex)
     except RuntimeError as err:
         logger.critical(err)
 
@@ -105,6 +107,8 @@ if __name__ == "__main__":
                         choices=['debug_full', 'debug', 'info', 'exec', 'warning', 'error'],
                         help='the logging level to output to screen/file '
                              '(NB: debug_full allows logging from external packages)')
+    parser.add_argument("-rx", "--path-regex", type=str, metavar='str',
+                        help='file/folder regex to retrieve', default="*")
     # parser.add_argument("--test-run", action="store_true",
     #                     help='do not run any executables (only create directories and create/copy files)')