Merge branch 'develop' into zep_user

lib/ramble/docs/configuration_files.rst

@@ -166,6 +166,7 @@ The current default configuration is as follows:
         type: 'BigQuery'
         uri: ''
 
+
 .. _upload-config-option:
 
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

lib/ramble/docs/tutorials/10_using_modifiers.rst

@@ -20,6 +20,10 @@ application for atmospheric research and operational forecasting applications.
 This tutorial builds off of concepts introduced in previous tutorials. Please
 make sure you review those before starting with this tutorial's content.
 
+**NOTE:** In this tutorial, you will encounter expected errors when copying and
+pasting the commands. This is to help show situations you might run into when
+trying to use Ramble on your own, and illustrate how you might fix them.
+
 Create a Workspace
 ------------------
 

lib/ramble/docs/tutorials/4_using_vectors_and_matrices.rst

@@ -19,6 +19,10 @@ workspaces. Ramble's vector and matrix variable logic is defined in more detail
 This tutorial builds off of concepts introduced in previous tutorials. Please
 make sure you review those before starting with this tutorial's content.
 
+**NOTE:** In this tutorial, you will encounter expected errors when copying and
+pasting the commands. This is to help show situations you might run into when
+trying to use Ramble on your own, and illustrate how you might fix them.
+
 .. include:: shared/gromacs_workspace.rst
 
 Activate the Workspace

lib/ramble/docs/tutorials/6_configuring_a_scaling_study.rst

@@ -19,6 +19,10 @@ application for atmospheric research and operational forecasting applications.
 This tutorial builds off of concepts introduced in previous tutorials. Please
 make sure you review those before starting with this tutorial's content.
 
+**NOTE:** In this tutorial, you will encounter expected errors when copying and
+pasting the commands. This is to help show situations you might run into when
+trying to use Ramble on your own, and illustrate how you might fix them.
+
 Create a Workspace
 ------------------
 

lib/ramble/docs/tutorials/7_using_zips_and_matrices.rst

@@ -20,6 +20,10 @@ application for atmospheric research and operational forecasting applications.
 This tutorial builds off of concepts introduced in previous tutorials. Please
 make sure you review those before starting with this tutorial's content.
 
+**NOTE:** In this tutorial, you will encounter expected errors when copying and
+pasting the commands. This is to help show situations you might run into when
+trying to use Ramble on your own, and illustrate how you might fix them.
+
 Create a Workspace
 ------------------
 

lib/ramble/docs/tutorials/8_var_expansion_indirection_and_stack_parameterization.rst

@@ -21,6 +21,10 @@ application for atmospheric research and operational forecasting applications.
 This tutorial builds off of concepts introduced in previous tutorials. Please
 make sure you review those before starting with this tutorial's content.
 
+**NOTE:** In this tutorial, you will encounter expected errors when copying and
+pasting the commands. This is to help show situations you might run into when
+trying to use Ramble on your own, and illustrate how you might fix them.
+
 Create a Workspace
 ------------------
 

lib/ramble/ramble/application.py

@@ -71,6 +71,8 @@ class ApplicationBase(object, metaclass=ApplicationMeta):
     maintainers: List[str] = []
     tags: List[str] = []
 
+    license_inc_name = 'license.inc'
+
     def __init__(self, file_path):
         super().__init__()
 
@@ -111,7 +113,6 @@ def __init__(self, file_path):
 
         self.license_path = ''
         self.license_file = ''
-        self.license_inc_name = 'license.inc'
 
         self.build_phase_order()
 

lib/ramble/ramble/cmd/workspace.py

@@ -664,6 +664,10 @@ def workspace_archive_setup_parser(subparser):
         default=None,
         help='URL to upload tar archive into. Does nothing if `-t` is not specified.')
 
+    subparser.add_argument(
+        '--include-secrets', action='store_true',
+        help='If set, secrets are included in the archive. Default is false')
+
     arguments.add_common_arguments(subparser, ['phases', 'include_phase_dependencies',
                                                'where', 'exclude_where'])
 
@@ -683,7 +687,8 @@ def workspace_archive(args):
                             filters,
                             create_tar=args.tar_archive,
                             archive_prefix=args.archive_prefix,
-                            upload_url=args.upload_url)
+                            upload_url=args.upload_url,
+                            include_secrets=args.include_secrets)
 
     workspace_run_pipeline(args, pipeline)
 

lib/ramble/ramble/pipeline.py

@@ -17,6 +17,7 @@
 import llnl.util.tty as tty
 from llnl.util.tty.color import cprint
 
+import ramble.application
 import ramble.config
 import ramble.experiment_set
 import ramble.software_environments
@@ -249,11 +250,14 @@ class ArchivePipeline(Pipeline):
 
     name = 'archive'
 
-    def __init__(self, workspace, filters, create_tar=False, archive_prefix=None, upload_url=None):
+    def __init__(self, workspace, filters, create_tar=False,
+                 archive_prefix=None, upload_url=None,
+                 include_secrets=False):
         super().__init__(workspace, filters)
         self.action_string = 'Archiving'
         self.create_tar = create_tar
         self.upload_url = upload_url
+        self.include_secrets = include_secrets
         self.archive_prefix = archive_prefix
         self.archive_name = None
 
@@ -304,6 +308,24 @@ def _prepare(self):
                 fs.mkdirp(os.path.dirname(dest))
                 shutil.copyfile(file, dest)
 
+        # Copy shared files
+        archive_shared = os.path.join(self.workspace.latest_archive_path,
+                                      ramble.workspace.workspace_shared_path)
+
+        excluded_secrets = set()
+        if not self.include_secrets:
+            excluded_secrets.add(ramble.application.ApplicationBase.license_inc_name)
+
+        fs.mkdirp(archive_shared)
+        for root, dirs, files in os.walk(self.workspace.shared_dir):
+            for name in files:
+                if name not in excluded_secrets:
+                    src_dir = os.path.join(self.workspace.shared_dir, root)
+                    src = os.path.join(src_dir, name)
+                    dest = src.replace(self.workspace.shared_dir, archive_shared)
+                    fs.mkdirp(os.path.dirname(dest))
+                    shutil.copy(src, dest)
+
         # Copy logs, but omit all symlinks (i.e. "latest")
         archive_logs = os.path.join(self.workspace.latest_archive_path,
                                     ramble.workspace.workspace_log_path)

lib/ramble/ramble/test/cmd/workspace.py

@@ -13,6 +13,7 @@
 
 import llnl.util.filesystem as fs
 
+import ramble.application
 import ramble.workspace
 from ramble.software_environments import RambleSoftwareEnvironmentError
 from ramble.main import RambleCommand, RambleCommandError
@@ -1076,15 +1077,26 @@ def test_workspace_archive():
     environments: {}
 """
 
+    test_licenses = """
+licenses:
+  basic:
+    set:
+      TEST_LIC: 'value'
+"""
+
     workspace_name = 'test_basic_archive'
     ws1 = ramble.workspace.create(workspace_name)
     ws1.write()
 
     config_path = os.path.join(ws1.config_dir, ramble.workspace.config_file_name)
+    lic_path = os.path.join(ws1.config_dir, 'licenses.yaml')
 
     with open(config_path, 'w+') as f:
         f.write(test_config)
 
+    with open(lic_path, 'w+') as f:
+        f.write(test_licenses)
+
     # Create more templates
     new_templates = []
     for i in range(0, 5):
@@ -1129,6 +1141,69 @@ def test_workspace_archive():
         archived_path = file.replace(ws1.root, ws1.latest_archive_path)
         assert os.path.exists(archived_path)
 
+    assert not os.path.exists(os.path.join(ws1.latest_archive_path, 'shared', 'licenses', 'basic',
+                                           ramble.application.ApplicationBase.license_inc_name))
+
+
+def test_workspace_archive_include_secrets():
+    test_config = """
+ramble:
+  variables:
+    mpi_command: 'mpirun -n {n_ranks} -ppn {processes_per_node}'
+    batch_submit: 'batch_submit {execute_experiment}'
+    processes_per_node: '5'
+    n_ranks: '{processes_per_node}*{n_nodes}'
+  applications:
+    basic:
+      workloads:
+        test_wl:
+          experiments:
+            test_experiment:
+              variables:
+                n_nodes: '2'
+  spack:
+    concretized: true
+    packages: {}
+    environments: {}
+"""
+
+    test_licenses = """
+licenses:
+  basic:
+    set:
+      TEST_LIC: 'value'
+"""
+
+    workspace_name = 'test_basic_archive'
+    ws1 = ramble.workspace.create(workspace_name)
+    ws1.write()
+
+    config_path = os.path.join(ws1.config_dir, ramble.workspace.config_file_name)
+    lic_path = os.path.join(ws1.config_dir, 'licenses.yaml')
+
+    with open(config_path, 'w+') as f:
+        f.write(test_config)
+
+    with open(lic_path, 'w+') as f:
+        f.write(test_licenses)
+
+    # Create more templates
+    new_templates = []
+    for i in range(0, 5):
+        new_template = os.path.join(ws1.config_dir, 'test_template.%s' % i)
+        new_templates.append(new_template)
+        f = open(new_template, 'w+')
+        f.close()
+
+    ws1._re_read()
+
+    workspace('setup', '--dry-run', global_args=['-w', workspace_name])
+
+    workspace('archive', '--include-secrets', global_args=['-w', workspace_name])
+
+    assert os.path.exists(os.path.join(ws1.latest_archive_path, 'shared', 'licenses', 'basic',
+                                       ramble.application.ApplicationBase.license_inc_name))
+
 
 def test_workspace_tar_archive():
     test_config = """

lib/ramble/ramble/workspace/__init__.py

@@ -42,7 +42,8 @@
     ramble_workspace_var,
     namespace,
     workspace_config_path,
-    workspace_log_path
+    workspace_log_path,
+    workspace_shared_path
 )
 
 __all__ = [
@@ -79,4 +80,5 @@
     'namespace',
     'workspace_config_path',
     'workspace_log_path',
+    'workspace_shared_path',
 ]

share/ramble/ramble-completion.bash

@@ -654,7 +654,7 @@ _ramble_workspace_activate() {
 }
 
 _ramble_workspace_archive() {
-    RAMBLE_COMPREPLY="-h --help --tar-archive -t --prefix -p --upload-url -u --phases --include-phase-dependencies --where --exclude-where"
+    RAMBLE_COMPREPLY="-h --help --tar-archive -t --prefix -p --upload-url -u --include-secrets --phases --include-phase-dependencies --where --exclude-where"
 }
 
 _ramble_workspace_deactivate() {