Merge pull request #695 from douglasjacobsen/additive-concretize

Allow `workspace concretize` to be additive

lib/ramble/ramble/cmd/workspace.py

@@ -414,6 +414,14 @@ def workspace_concretize_setup_parser(subparser):
         help="Remove unused software and experiment templates from workspace config",
         required=False,
     )
+    subparser.add_argument(
+        "--quiet",
+        "-q",
+        dest="quiet",
+        action="store_true",
+        help="Silently ignore conflicting package definitions",
+        required=False,
+    )
 
 
 def workspace_concretize(args):
@@ -423,11 +431,8 @@ def workspace_concretize(args):
         logger.debug("Simplifying workspace config")
         ws.simplify()
     else:
-        if args.force_concretize:
-            ws.force_concretize = True
-
         logger.debug("Concretizing workspace")
-        ws.concretize()
+        ws.concretize(force=args.force_concretize, quiet=args.quiet)
 
 
 def workspace_run_pipeline(args, pipeline):

lib/ramble/ramble/test/cmd/workspace.py

@@ -598,7 +598,7 @@ def test_concretize_concrete_config():
   software:
     packages:
       zlib:
-        pkg_spec: 'zlib'
+        pkg_spec: 'zlib@1.3'
     environments:
       zlib:
         packages:
@@ -618,7 +618,7 @@ def test_concretize_concrete_config():
 
     with pytest.raises(ramble.workspace.RambleWorkspaceError) as e:
         workspace("concretize", global_args=["-w", workspace_name])
-        assert "Cannot concretize an already concretized workspace." in e
+        assert "Package zlib would be defined in multiple conflicting ways" in e
 
 
 def test_force_concretize():
@@ -672,12 +672,13 @@ def test_force_concretize():
 
     ws1._re_read()
 
-    with pytest.raises(ramble.workspace.RambleWorkspaceError) as e:
-        workspace("concretize", global_args=["-w", workspace_name])
-        assert "Cannot concretize an already concretized workspace." in e
-
     workspace("concretize", "-f", global_args=["-w", workspace_name])
 
+    assert search_files_for_string([config_path], "zlib:") is True
+    assert search_files_for_string([config_path], "zlib-test") is True
+
+    workspace("concretize", "--simplify", global_args=["-w", workspace_name])
+
     assert search_files_for_string([config_path], "zlib:") is True
     assert search_files_for_string([config_path], "zlib-test") is False
 

lib/ramble/ramble/test/cmd/workspace_concretize.py

@@ -0,0 +1,64 @@
+# Copyright 2022-2024 The Ramble Authors
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+import os
+import pytest
+
+import ramble.workspace
+from ramble.main import RambleCommand
+
+# everything here uses the mock_workspace_path
+pytestmark = pytest.mark.usefixtures("mutable_config", "mutable_mock_workspace_path")
+
+workspace = RambleCommand("workspace")
+
+
+def test_workspace_concretize_additive(request):
+    workspace_name = request.node.name
+
+    ws = ramble.workspace.create(workspace_name)
+    global_args = ["-w", workspace_name]
+
+    workspace(
+        "generate-config", "gromacs", "-p", "spack", "--wf", "water_*", global_args=global_args
+    )
+    workspace("concretize", "-q", global_args=global_args)
+
+    with open(ws.config_file_path) as f:
+        content = f.read()
+        assert "spack_gromacs" in content
+        assert "gcc9" in content
+        assert "wrfv4" not in content
+        assert "intel-oneapi-vtune" not in content
+
+    workspace("generate-config", "wrfv4", "-p", "spack", global_args=global_args)
+    workspace("concretize", "-q", global_args=global_args)
+
+    with open(ws.config_file_path) as f:
+        content = f.read()
+        assert "spack_gromacs" in content
+        assert "gcc9" in content
+        assert "wrfv4" in content
+        assert "intel-oneapi-vtune" not in content
+
+    modifiers_path = os.path.join(ws.config_dir, "modifiers.yaml")
+
+    with open(modifiers_path, "w+") as f:
+        f.write(
+            """modifiers:
+- name: intel-aps"""
+        )
+
+    workspace("concretize", "-q", global_args=global_args)
+
+    with open(ws.config_file_path) as f:
+        content = f.read()
+        assert "spack_gromacs" in content
+        assert "gcc9" in content
+        assert "wrfv4" in content
+        assert "intel-oneapi-vtune" in content

lib/ramble/ramble/workspace/workspace.py

@@ -469,7 +469,6 @@ def __init__(self, root, dry_run=False, read_default_template=True):
         self.txlock = lk.Lock(self._transaction_lock_path)
         self.dry_run = dry_run
         self.repeat_success_strict = True
-        self.force_concretize = False
 
         self.read_default_template = read_default_template
         self.configs = ramble.config.ConfigScope("workspace", self.config_dir)
@@ -1106,25 +1105,19 @@ def yaml_add_comment_before_key(
             workspace_dict[namespace.ramble][namespace.application] = apps_dict
             print(f"\n{syaml.dump(workspace_dict)}")
 
-    def concretize(self):
-        full_software_dict = self.get_software_dict()
+    def concretize(self, force=False, quiet=False):
+        """Concretize software definitions for defined experiments
 
-        if not self.force_concretize:
-            try:
-                if (
-                    full_software_dict[namespace.packages]
-                    or full_software_dict[namespace.environments]
-                ):
-                    raise RambleWorkspaceError(
-                        "Cannot concretize an already concretized "
-                        "workspace. To overwrite the current configuration "
-                        "with the default software configuration, use "
-                        "'ramble workspace concretize -f'."
-                    )
-            except KeyError:
-                pass
+        Extract suggested software for experiments defined in a workspace, and
+        ensure the software environments are defined properly.
+
+        Args:
+            force (bool): Whether to overwrite conflicting definitions of named packages or not
+            quiet (bool): Whether to silently ignore conflicts or not
 
-        full_software_dict = syaml.syaml_dict()
+
+        """
+        full_software_dict = self.get_software_dict()
 
         if (
             namespace.packages not in full_software_dict
@@ -1159,7 +1152,7 @@ def concretize(self):
             for compiler_dict in compiler_dicts:
                 for comp, info in compiler_dict.items():
                     if fnmatch.fnmatch(app_inst.package_manager.name, info["package_manager"]):
-                        if comp not in packages_dict:
+                        if comp not in packages_dict or force:
                             packages_dict[comp] = syaml.syaml_dict()
                             packages_dict[comp]["pkg_spec"] = info["pkg_spec"]
                             ramble.config.add(
@@ -1183,20 +1176,18 @@ def concretize(self):
                                 ramble.config.add(
                                     config_path, scope=self.ws_file_config_scope_name()
                                 )
-                        elif not specs_equiv(info, packages_dict[comp]):
+                        elif not quiet and not specs_equiv(info, packages_dict[comp]):
                             logger.debug(f"  Spec 1: {str(info)}")
                             logger.debug(f"  Spec 2: {str(packages_dict[comp])}")
                             raise RambleConflictingDefinitionError(
-                                f"Compiler {comp} defined in multiple conflicting ways"
+                                f"Compiler {comp} would be defined " "in multiple conflicting ways"
                             )
 
-            add_env = False
-            if env_name not in environments_dict:
-                new_env = syaml.syaml_dict()
-                new_env[namespace.packages] = []
-
             logger.debug(f"Trying to define packages for {env_name}")
-            app_packages = new_env[namespace.packages]
+            app_packages = []
+            if env_name in environments_dict:
+                if namespace.packages in environments_dict[env_name]:
+                    app_packages = environments_dict[env_name][namespace.packages].copy()
 
             software_dicts = [app_inst.software_specs]
             for mod_inst in app_inst._modifier_instances:
@@ -1206,27 +1197,30 @@ def concretize(self):
                 for spec_name, info in software_dict.items():
                     if fnmatch.fnmatch(app_inst.package_manager.name, info["package_manager"]):
                         logger.debug(f"    Found spec: {spec_name}")
-                        if spec_name not in packages_dict:
-                            add_env = True
+                        if spec_name not in packages_dict or force:
                             packages_dict[spec_name] = syaml.syaml_dict()
                             packages_dict[spec_name]["pkg_spec"] = info["pkg_spec"]
                             if "compiler_spec" in info and info["compiler_spec"]:
                                 packages_dict[spec_name]["compiler_spec"] = info["compiler_spec"]
                             if "compiler" in info and info["compiler"]:
                                 packages_dict[spec_name]["compiler"] = info["compiler"]
 
-                        elif not specs_equiv(info, packages_dict[spec_name]):
+                        elif not quiet and not specs_equiv(info, packages_dict[spec_name]):
                             logger.debug(f"  Spec 1: {str(info)}")
                             logger.debug(f"  Spec 2: {str(packages_dict[spec_name])}")
                             raise RambleConflictingDefinitionError(
-                                f"Package {spec_name} defined in multiple conflicting ways"
+                                f"Package {spec_name} would be defined in multiple "
+                                "conflicting ways"
                             )
 
                         if spec_name not in app_packages:
                             app_packages.append(spec_name)
 
-            if add_env:
-                environments_dict[env_name] = new_env.copy()
+            if app_packages:
+                if env_name not in environments_dict:
+                    environments_dict[env_name] = syaml.syaml_dict()
+
+                environments_dict[env_name][namespace.packages] = app_packages.copy()
 
         ramble.config.config.update_config(
             "software", full_software_dict, scope=self.ws_file_config_scope_name()

share/ramble/ramble-completion.bash

@@ -669,7 +669,7 @@ _ramble_workspace_create() {
 }
 
 _ramble_workspace_concretize() {
-    RAMBLE_COMPREPLY="-h --help -f --force-concretize --simplify"
+    RAMBLE_COMPREPLY="-h --help -f --force-concretize --simplify --quiet -q"
 }
 
 _ramble_workspace_setup() {