Merge pull request #691 from douglasjacobsen/software-generate

Front-end commands for managing software

lib/ramble/ramble/cmd/workspace.py

@@ -54,8 +54,11 @@
     ["list", "ls"],
     ["remove", "rm"],
     "generate-config",
+    "manage",
 ]
 
+manage_commands = ["experiments", "software"]
+
 
 def workspace_activate_setup_parser(subparser):
     """Set the current workspace"""
@@ -984,9 +987,8 @@ def workspace_mirror(args):
     pipeline.run()
 
 
-def workspace_generate_config_setup_parser(subparser):
-    """generate current workspace config"""
-
+def workspace_manage_experiments_setup_parser(subparser):
+    """manage experiment definitions"""
     arguments.add_common_arguments(subparser, ["application"])
 
     subparser.add_argument(
@@ -1080,8 +1082,8 @@ def workspace_generate_config_setup_parser(subparser):
     )
 
 
-def workspace_generate_config(args):
-    """Generate a configuration file for this ramble workspace"""
+def workspace_manage_experiments(args):
+    """Perform experiment management"""
     ws = ramble.cmd.find_workspace(args)
 
     if ws is None:
@@ -1129,6 +1131,153 @@ def workspace_generate_config(args):
         args.overwrite,
     )
 
+    if ws.dry_run:
+        ws.print_config()
+
+
+def workspace_manage_software_setup_parser(subparser):
+    """manage workspace software definitions"""
+
+    subparser.add_argument(
+        "--environment-name",
+        "--env",
+        dest="environment_name",
+        metavar="ENV",
+        help="Name of environment to define",
+    )
+
+    env_types = subparser.add_mutually_exclusive_group()
+    env_types.add_argument(
+        "--environment-packages",
+        dest="environment_packages",
+        help="Comma separated list of packages to add into environment",
+        metavar="PKG1,PKG2,PKG2",
+    )
+
+    env_types.add_argument(
+        "--external-env",
+        dest="external_env_path",
+        help="Path to external environment description",
+        metavar="PATH",
+    )
+
+    subparser.add_argument(
+        "--package-name",
+        "--pkg",
+        dest="package_name",
+        metavar="NAME",
+        help="Name of package to define",
+    )
+
+    subparser.add_argument(
+        "--package-spec",
+        "--pkg-spec",
+        "--spec",
+        dest="package_spec",
+        metavar="SPEC",
+        help="Value for the pkg_spec attribute in the defined package",
+    )
+
+    subparser.add_argument(
+        "--compiler-package",
+        "--compiler-pkg",
+        "--compiler",
+        dest="compiler_package",
+        metavar="PKG",
+        help="Value for the compiler attribute in the defined package",
+    )
+
+    subparser.add_argument(
+        "--compiler-spec",
+        dest="compiler_spec",
+        metavar="SPEC",
+        help="Value for the compiler_spec attribute in the defined package",
+    )
+
+    subparser.add_argument(
+        "--package-manager-prefix",
+        "--prefix",
+        dest="package_manager_prefix",
+        metavar="PREFIX",
+        help="Prefix for defined package attributes. "
+        "Resulting attributes will be {prefix}_pkg_spec.",
+    )
+
+    modify_types = subparser.add_mutually_exclusive_group()
+    modify_types.add_argument(
+        "--remove",
+        "--delete",
+        action="store_true",
+        help="Whether to remove named package and environment definitions if they exist.",
+    )
+
+    modify_types.add_argument(
+        "--overwrite",
+        "-o",
+        action="store_true",
+        help="Whether to overwrite existing definitions or not.",
+    )
+
+    subparser.add_argument(
+        "--dry-run",
+        "--print",
+        dest="dry_run",
+        action="store_true",
+        help="perform a dry run. Print resulting config to screen and not "
+        + "to the workspace configuration file",
+    )
+
+
+def workspace_manage_software(args):
+    """Execute workspace manage software command"""
+
+    ws = ramble.cmd.find_workspace(args)
+
+    if ws is None:
+        import tempfile
+
+        logger.warn("No active workspace found. Defaulting to `--dry-run`")
+
+        root = tempfile.TemporaryDirectory()
+        ws = ramble.workspace.Workspace(str(root))
+        ws.dry_run = True
+    else:
+        ws.dry_run = args.dry_run
+
+    if args.package_name:
+        ws.manage_packages(
+            args.package_name,
+            args.package_spec,
+            args.compiler_package,
+            args.compiler_spec,
+            args.package_manager_prefix,
+            args.remove,
+            args.overwrite,
+        )
+        logger.all_msg("Need to manipulate package definitions")
+
+    if args.environment_name:
+        ws.manage_environments(
+            args.environment_name,
+            args.environment_packages,
+            args.external_env_path,
+            args.remove,
+            args.overwrite,
+        )
+
+    if ws.dry_run:
+        ws.print_config()
+
+
+def workspace_generate_config_setup_parser(subparser):
+    """generate current workspace config"""
+    workspace_manage_experiments_setup_parser(subparser)
+
+
+def workspace_generate_config(args):
+    """Generate a configuration file for this ramble workspace"""
+    workspace_manage_experiments(args)
+
 
 #: Dictionary mapping subcommand names and aliases to functions
 subcommand_functions = {}
@@ -1173,3 +1322,42 @@ def workspace(parser, args):
     """Look for a function called workspace_<name> and call it."""
     action = subcommand_functions[args.workspace_command]
     action(args)
+
+
+manage_subcommand_functions = {}
+
+
+def workspace_manage(args):
+    """Look for a function for the manage subcommand, and execute it."""
+    action = manage_subcommand_functions[args.manage_command]
+    action(args)
+
+
+def workspace_manage_setup_parser(subparser):
+    """manage workspace definitions"""
+    sp = subparser.add_subparsers(metavar="SUBCOMMAND", dest="manage_command")
+
+    for name in manage_commands:
+        if isinstance(name, (list, tuple)):
+            name, aliases = name[0], name[1:]
+        else:
+            aliases = []
+
+        # add commands to subcommands dict
+        function_name = sanitize_arg_name("workspace_manage_%s" % name)
+
+        function = globals()[function_name]
+        for alias in [name] + aliases:
+            manage_subcommand_functions[alias] = function
+
+        # make a subparser and run the command's setup function on it
+        setup_parser_cmd_name = sanitize_arg_name("workspace_manage_%s_setup_parser" % name)
+        setup_parser_cmd = globals()[setup_parser_cmd_name]
+
+        subsubparser = sp.add_parser(
+            name,
+            aliases=aliases,
+            help=setup_parser_cmd.__doc__,
+            description=setup_parser_cmd.__doc__,
+        )
+        setup_parser_cmd(subsubparser)

lib/ramble/ramble/test/end_to_end/manage_software.py

@@ -0,0 +1,120 @@
+# Copyright 2022-2024 The Ramble Authors
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+import pytest
+
+import ramble.workspace
+import ramble.config
+import ramble.software_environments
+from ramble.main import RambleCommand
+
+
+# everything here uses the mock_workspace_path
+pytestmark = pytest.mark.usefixtures("mutable_config", "mutable_mock_workspace_path")
+
+workspace = RambleCommand("workspace")
+
+
+def test_manage_software(mutable_config, mutable_mock_workspace_path):
+    workspace_name = "test_manage_software"
+    with ramble.workspace.create(workspace_name) as ws1:
+        ws1.write()
+
+        config_path = ws1.config_file_path
+
+        workspace(
+            "manage",
+            "experiments",
+            "wrfv4",
+            "-v",
+            "n_ranks=1",
+            "-v",
+            "n_nodes=1",
+            "-p",
+            "spack",
+            global_args=["-w", workspace_name],
+        )
+        workspace("concretize", global_args=["-w", workspace_name])
+
+        ws1._re_read()
+
+        with open(config_path) as f:
+            content = f.read()
+            # Check that wrf has a package, and the package is in an environment
+            assert "pkg_spec: wrf" in content
+            assert "- wrfv4" in content
+
+            # Check that intel-mpi was defined
+            assert "intel-mpi" in content
+
+            # Check that gcc was defined
+            assert "[email protected]" in content
+
+            # Check that the (soon to be new) definition of gcc is not defined
+            assert "[email protected]" not in content
+
+        # Change the GCC package definition
+        workspace(
+            "manage",
+            "software",
+            "--pkg",
+            "gcc9",
+            "--overwrite",
+            "--package-spec",
+            "[email protected]",
+            global_args=["-w", workspace_name],
+        )
+
+        with open(config_path) as f:
+            content = f.read()
+            assert "[email protected]" in content
+
+        # Delete configs for wrf
+        workspace(
+            "manage",
+            "software",
+            "--remove",
+            "--env",
+            "wrfv4",
+            "--pkg",
+            "wrfv4",
+            global_args=["-w", workspace_name],
+        )
+        workspace(
+            "manage",
+            "software",
+            "--remove",
+            "--pkg",
+            "intel-mpi",
+            global_args=["-w", workspace_name],
+        )
+        workspace(
+            "manage", "software", "--remove", "--pkg", "gcc9", global_args=["-w", workspace_name]
+        )
+        workspace(
+            "manage",
+            "software",
+            "--env",
+            "foo",
+            "--environment-packages",
+            "bar,baz",
+            global_args=["-w", workspace_name],
+        )
+
+        with open(config_path) as f:
+            content = f.read()
+
+            # Check that new env definitions are found
+            assert "foo" in content
+            assert "bar" in content
+            assert "baz" in content
+
+            # Check that removed definitions no longer exist
+            assert "intel-mpi" not in content
+            assert "gcc" not in content
+            assert "- wrf" not in content

lib/ramble/ramble/util/conversions.py

@@ -0,0 +1,30 @@
+# Copyright 2022-2024 The Ramble Authors
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+
+def list_str_to_list(in_str):
+    """Convert a comma delimited list as a string into a python list
+
+    Args:
+        in_str (str): Input string, comma delimited list of values
+
+    Returns:
+        (list) Each value from input string is a separate entry in the list.
+
+    """
+    if "[" not in in_str and "]" not in in_str:
+        return in_str
+
+    temp = in_str.replace("[", "").replace("]", "")
+    out_value = []
+    for part in temp.split(","):
+        if part[0] == " ":
+            out_value.append(part[1:])
+        else:
+            out_value.append(part)
+    return out_value