Add simple prototype of functionality to push to spack caches

lib/ramble/ramble/application.py

@@ -58,7 +58,7 @@ class ApplicationBase(object, metaclass=ApplicationMeta):
     _workload_exec_key = 'executables'
     _inventory_file_name = 'ramble_inventory.json'
     _status_file_name = 'ramble_status.json'
-    _pipelines = ['analyze', 'archive', 'mirror', 'setup']
+    _pipelines = ['analyze', 'archive', 'mirror', 'setup', 'pushtocache'] # TODO: why do we repeat this in so many places
 
     #: Lists of strings which contains GitHub usernames of attributes.
     #: Do not include @ here in order not to unnecessarily ping the users.
@@ -1490,10 +1490,11 @@ def env_vars(self):
             license_conf = ramble.config.config.get_config('licenses',
                                                            scope=scope)
             if license_conf:
-                app_licenses = license_conf[self.name]
-                if app_licenses:
-                    # Append logic to source file which contains the exports
-                    command.append(f". {{license_input_dir}}/{self.license_inc_name}")
+                if self.name in license_conf:
+                    app_licenses = license_conf[self.name]
+                    if app_licenses:
+                        # Append logic to source file which contains the exports
+                        command.append(f". {{license_input_dir}}/{self.license_inc_name}")
 
         # Process environment variable actions
         for env_var_set in self._env_variable_sets:

lib/ramble/ramble/application_types/spack.py

@@ -365,6 +365,21 @@ def _mirror_software(self, workspace):
             with self.logger.force_echo():
                 tty.die(e)
 
+    register_phase('push_to_spack_cache', pipeline='pushtocache', depends_on=[])
+
+    def _push_to_spack_cache(self, workspace):
+        try:
+            # TODO: make sure this is called as infrequently as possible
+            # TODO: this is a lot of repeated stuff, can we DRY with out pipelines?
+            self.spack_runner.set_dry_run(workspace.dry_run)
+            env_path = self.expander.env_path
+            self.spack_runner.set_env(env_path)
+            self.spack_runner.activate()
+            self.spack_runner.push_to_spack_cache(workspace.spack_cache_path)
+        except ramble.spack_runner.RunnerError as e:
+            with self.logger.force_echo():
+                tty.die(e)
+
     def populate_inventory(self, workspace, force_compute=False, require_exist=False):
         """Add software environment information to hash inventory"""
 

lib/ramble/ramble/cmd/workspace.py

@@ -51,6 +51,7 @@
     'concretize',
     'setup',
     'analyze',
+    'pushtocache',
     'info',
     'edit',
     'mirror',
@@ -404,6 +405,37 @@ def workspace_analyze(args):
         workspace_run_pipeline(args, pipeline)
 
 
+def workspace_pushtocache(args):
+    current_pipeline = ramble.pipeline.pipelines.pushtocache
+    ws = ramble.cmd.require_active_workspace(cmd_name='workspace pushtocache')
+
+    # TODO: this is repeated for pipelines, can we DRY it?
+    filters = ramble.filters.Filters(
+        phase_filters=args.phases,
+        include_where_filters=args.where,
+        exclude_where_filters=args.exclude_where
+    )
+    pipeline_cls = ramble.pipeline.pipeline_class(current_pipeline)
+
+    pipeline = pipeline_cls(ws, filters, spack_cache_path=args.cache_path)
+
+    workspace_run_pipeline(args, pipeline)
+    pipeline.run()
+
+
+def workspace_pushtocache_setup_parser(subparser):
+    """push workspace envs to a given spack buildcache"""
+
+    subparser.add_argument(
+        '-d', dest='cache_path',
+        default=None,
+        required=True,
+        help='Path to spack cache.')
+
+    arguments.add_common_arguments(subparser, ['phases', 'include_phase_dependencies',
+                                               'where', 'exclude_where'])
+
+
 def workspace_info_setup_parser(subparser):
     """Information about a workspace"""
     subparser.add_argument('-v', '--verbose', action='count', default=0,

lib/ramble/ramble/language/modifier_language.py

@@ -35,6 +35,21 @@ def _execute_mode(mod):
     return _execute_mode
 
 
+@modifier_directive('default_modes')
+def default_mode(name, **kwargs):
+    """Define a default mode for this modifier.
+
+    The default mode will be used if modifier mode is not specified in an experiment."""
+
+    def _execute_default_mode(mod):
+        if name not in mod.modes:
+            raise DirectiveError(f'default_mode directive given an invalid mode for modifier '
+                                 f'{mod.name}. Valid modes are {str(list(mod.modes.keys()))}')
+        mod.default_mode = name
+
+    return _execute_default_mode
+
+
 @modifier_directive('variable_modifications')
 def variable_modification(name, modification, method='set', mode=None, modes=None, **kwargs):
     """Define a new variable modification for a mode in this modifier.

lib/ramble/ramble/modifier.py

@@ -66,6 +66,9 @@ def set_usage_mode(self, mode):
         """
         if mode:
             self._usage_mode = mode
+        elif hasattr(self, 'default_mode'):
+            self._usage_mode = self.default_mode
+            tty.msg(f'    Using default usage mode {self._usage_mode} on modifier {self.name}')
         else:
             if len(self.modes) > 1 or len(self.modes) == 0:
                 raise InvalidModeError('Cannot auto determine usage '

lib/ramble/ramble/pipeline.py

@@ -319,13 +319,33 @@ def _complete(self):
                  | stat.S_IROTH | stat.S_IXOTH)
 
 
-pipelines = Enum('pipelines', ['analyze', 'archive', 'mirror', 'setup'])
+class PushToCachePipeline(Pipeline):
+    """Class for the pushtocache pipeline"""
+
+    name = 'pushtocache'
+
+    def __init__(self, workspace, filters, spack_cache_path=None):
+        super().__init__(workspace, filters)
+        self.action_string = 'Pushing to Spack Cache'
+        self.spack_cache_path = spack_cache_path
+
+    def _prepare(self):
+        super()._prepare()
+        self.workspace.spack_cache_path = self.spack_cache_path
+
+    def _complete(self):
+        tty.msg('Pushed envs to spack cache %s' % self.spack_cache_path)
+
+
+# TODO: should this use the classes `.name`?
+pipelines = Enum('pipelines', ['analyze', 'archive', 'mirror', 'setup', 'pushtocache'])
 
 _pipeline_map = {
     pipelines.analyze: AnalyzePipeline,
     pipelines.archive: ArchivePipeline,
     pipelines.mirror: MirrorPipeline,
-    pipelines.setup: SetupPipeline
+    pipelines.setup: SetupPipeline,
+    pipelines.pushtocache: PushToCachePipeline
 }
 
 

lib/ramble/ramble/spack_runner.py

@@ -676,6 +676,36 @@ def mirror_environment(self, mirror_path):
   %-4d added
   %-4d failed to fetch.""" % (0, 0, 0)
 
+    def get_env_hash_list(self):
+        self._check_active()
+        args = [
+            'find',
+            '--format',
+            '/{hash}'
+        ]
+        output = self.spack(*args, output=str).strip()
+        return output
+
+    def push_to_spack_cache(self, spack_cache_path):
+        """Push packages for a given env to the spack cache"""
+        self._check_active()
+
+        hash_list = self.get_env_hash_list()
+
+        args = [
+            "buildcache",
+            "push",
+            "-a",  # TODO: remove this, it's just here for testing
+            spack_cache_path,
+            hash_list
+        ]
+
+        if not self.dry_run:
+            return self.spack(*args, output=str).strip()
+        else:
+            self._dry_run_print(self.spack, args)
+            return
+
     def _dry_run_print(self, executable, args):
         tty.msg('DRY-RUN: would run %s' % executable)
         tty.msg('         with args: %s' % args)