diff --git a/lib/ramble/ramble/language/application_language.py b/lib/ramble/ramble/language/application_language.py index aa2d32c10..34ebfd168 100644 --- a/lib/ramble/ramble/language/application_language.py +++ b/lib/ramble/ramble/language/application_language.py @@ -63,12 +63,12 @@ def workload( its application. Args: - executable: The name of an executable to be used - executables: A list of executable names to be used - input (Optional): The name of an input be used - inputs (Optional): A list of input names that will be used + executable (str): The name of an executable to be used + executables (str): A list of executable names to be used + input (str): Optional, name of an input be used + inputs (str): Optional, A list of input names that will be used - Either executable, or executables is a required input argument. + One of executable, or executables is required as an input argument. """ def _execute_workload(app): @@ -93,8 +93,8 @@ def workload_group(name, workloads=[], mode=None, **kwargs): application. Args: - name: The name of the group - workloads: A list of workloads to be grouped + name (str): The name of the group + workloads (list(str)): A list of workloads to be grouped """ def _execute_workload_groups(app): @@ -126,16 +126,17 @@ def executable(name, template, **kwargs): template (list[str] or str): The template command this executable should generate from Optional Args: - use_mpi or mpi: (Boolean) determines if this executable should be + use_mpi or mpi (boolean): determines if this executable should be wrapped with an `mpirun` like command or not. - variables (dict): dictionary of variable definitions to use for this executable only - redirect (Optional): Sets the path for outputs to be written to. + variables (dict): Dictionary of variable definitions to use for this + executable only + redirect (str): Optional, sets the path for outputs to be written to. defaults to {log_file} - output_capture (Optional): Declare which ouptu (stdout, stderr, both) to - capture. Defaults to stdout - run_in_background (Optional): Declare if the command should run in the background. - Defaults to False + output_capture (str): Optional, Declare which output (stdout, stderr, + both) to capture. Defaults to stdout + run_in_background (boolean): Optional, Declare if the command should + run in the background. Defaults to False """ def _execute_executable(app): @@ -164,15 +165,15 @@ def input_file( fetched from. Args: - url: Path to the input file / archive - description: Description of this input file - target_dir (Optional): The directory where the archive will be + url (str): Path to the input file / archive + description (str): Description of this input file + target_dir (str): Optional, the directory where the archive will be expanded. Defaults to the '{workload_input_dir}' + os.sep + '{input_name}' - sha256 (Optional): The expected sha256 checksum for the input file - extension (Optional): The extension to use for the input, if it isn't part of the + sha256 (str): Optional, the expected sha256 checksum for the input file + extension (str): Optiona, the extension to use for the input, if it isn't part of the file name. - expand (Optional): Whether the input should be expanded or not. Defaults to True + expand (boolean): Optional. Whether the input should be expanded or not. Defaults to True """ def _execute_input_file(app): @@ -264,7 +265,13 @@ def _execute_workload_variable(app): def environment_variable(name, value, description, workload=None, workloads=None, **kwargs): """Define an environment variable to be used in experiments - These can be specific to workloads. + Args: + name (str): Name of environment variable to define + value (str): Value to set env-var to + description (str): Description of the env-var + workload (str): Name of workload this env-var should be added to + workloads (list(str)): List of workload names this env-var should be + added to """ def _execute_environment_variable(app): @@ -280,76 +287,3 @@ def _execute_environment_variable(app): ) return _execute_environment_variable - - -@application_directive("phase_definitions") -def register_phase(name, pipeline=None, run_before=[], run_after=[]): - """Register a phase - - Phases are portions of a pipeline that will execute when - executing a full pipeline. - - Registering a phase allows an application to know what the phases - dependencies are, to ensure the execution order is correct. - - If called multiple times, the dependencies are combined together. Only one - instance of a phase will show up in the resulting dependency list for a phase. - - Args: - - name: The name of the phase. Phases are functions named '_'. - - pipeline: The name of the pipeline this phase should be registered into. - - run_before: A list of phase names this phase should run before - - run_after: A list of phase names this phase should run after - """ - - def _execute_register_phase(app): - import ramble.util.graph - - if pipeline not in app._pipelines: - raise DirectiveError( - "Directive register_phase was " - f'given an invalid pipeline "{pipeline}"\n' - "Available pipelines are: " - f" {app._pipelines}" - ) - - if not isinstance(run_before, list): - raise DirectiveError( - "Directive register_phase was " - "given an invalid type for " - "the run_before attribute in application " - f"{app.name}" - ) - - if not isinstance(run_after, list): - raise DirectiveError( - "Directive register_phase was " - "given an invalid type for " - "the run_after attribute in application " - f"{app.name}" - ) - - if not hasattr(app, f"_{name}"): - raise DirectiveError( - "Directive register_phase was " - f"given an undefined phase {name} " - f"in application {app.name}" - ) - - if pipeline not in app.phase_definitions: - app.phase_definitions[pipeline] = {} - - if name in app.phase_definitions[pipeline]: - phase_node = app.phase_definitions[pipeline][name] - else: - phase_node = ramble.util.graph.GraphNode(name, attribute=pipeline) - - for before in run_before: - phase_node.order_before(before) - - for after in run_after: - phase_node.order_after(after) - - app.phase_definitions[pipeline][name] = phase_node - - return _execute_register_phase diff --git a/lib/ramble/ramble/language/modifier_language.py b/lib/ramble/ramble/language/modifier_language.py index 5067e8fad..9b2204390 100644 --- a/lib/ramble/ramble/language/modifier_language.py +++ b/lib/ramble/ramble/language/modifier_language.py @@ -27,6 +27,10 @@ def mode(name, description, **kwargs): """Define a new mode for this modifier. Modes allow a modifier to bundle a set of modifications together. + + Args: + name (str): Name of the mode to define + description (str): Description of the new mode """ def _execute_mode(mod): @@ -39,7 +43,11 @@ def _execute_mode(mod): def default_mode(name, **kwargs): """Define a default mode for this modifier. - The default mode will be used if modifier mode is not specified in an experiment.""" + The default mode will be used if modifier mode is not specified in an experiment. + + Args: + name (str): Name of mode to be used as default + """ def _execute_default_mode(mod): if name not in mod.modes: @@ -59,20 +67,16 @@ def variable_modification(name, modification, method="set", mode=None, modes=Non A variable modification will apply a change to a defined variable within an experiment. Args: - name: The variable to modify - modification: The value to modify 'name' with - method: How the modification should be applied - mode: Single mode to group this modification into - modes: List of modes to group this modification into + name (str): The variable to modify + modification (str): The value to modify 'name' with + method (str): How the modification should be applied + mode (str): Single mode to group this modification into + modes (str): List of modes to group this modification into Supported values are 'append', 'prepend', and 'set': - - 'append' will add the modification to the end of 'name' - - 'prepend' will add the modification to the beginning of 'name' - - 'set' (Default) will overwrite 'name' with the modification - + 'append' will add the modification to the end of 'name' + 'prepend' will add the modification to the beginning of 'name' + 'set' (Default) will overwrite 'name' with the modification """ def _execute_variable_modification(mod): @@ -132,9 +136,15 @@ def write_exec_name(self, executable_name, executable, app_inst=None): Executable modifiers are allowed to modify the input executable in place. Executable modifiers must return two lists of executables. - Returns: - prepend_execs: List of executables to inject before the base executable - append_execs: List of executables to inject after the base executable + Args: + name (str): Name of executable modifier to use. Should be the name of a + class method. + + Each executable modifier needs to return: + prepend_execs (list(CommandExecutable)): List of executables to inject + before the base executable + append_execs (list(CommandExecutable)): List of executables to inject + after the base executable """ def _executable_modifier(mod): @@ -151,22 +161,20 @@ def env_var_modification(name, modification=None, method="set", mode=None, modes variables within the application instance. Args: - name: The name of the environment variable that will be modified - modification: The value of the modification - method: The method of the modification. - mode: Name of mode this env_var_modification should apply in - modes: List of mode names this env_var_modification should apply in + name (str): The name of the environment variable that will be modified + modification (str): The value of the modification + method (str): The method of the modification. + mode (str): Name of mode this env_var_modification should apply in + modes (list(str)): List of mode names this env_var_modification should apply in Supported values for method are: - set: Defines the variable to equal the modification value - - unset: Removes any definition of the variable from the environment - - prepend: Prepends the modification to the beginning of the variable. - Always uses the separator ':' - - append: Appends the modification value to the end of the value. Allows a - keyword argument of 'separator' to define the delimiter between values. + set: Defines the variable to equal the modification value + unset: Removes any definition of the variable from the environment + prepend: Prepends the modification to the beginning of the variable. + Always uses the separator ':' + append: Appends the modification value to the end of the value. Allows a + keyword argument of 'separator' to define the delimiter between + values. """ @@ -231,11 +239,11 @@ def _env_var_modification(mod): @modifier_directive("required_vars") def required_variable(var: str, results_level="variable"): - """Mark a var as being required + """Mark a variable as being required by this modifier Args: - var: Value to mark as required - results_level (str): 'variable' or 'key'. If 'key' variable is promoted to + var (str): Variable name to mark as required + results_level (str): 'variable' or 'key'. If 'key', variable is promoted to a key within JSON or YAML formatted results. """ @@ -310,15 +318,19 @@ def package_manager_requirement( """Define a requirement when this modifier is used in an experiment with a package manager. + This allows modifiers to inject additional requirements on packages + managers. These can be used to ensure package manager commands return + specific values. + Args: - command: Package manager command to execute, when evaluating the requirement - validation_type: Type of validation to perform on output of command. + command (str): Package manager command to execute, when evaluating the requirement + validation_type (str): Type of validation to perform on output of command. Valid types are: 'empty', 'not_empty', 'contains_regex', 'does_not_contain_regex' - modes: List of usage modes this requirement should apply to - regex: Regular expression to use when validation_type is either 'contains_regex' + modes (list(str)): List of usage modes this requirement should apply to + regex (str): Regular expression to use when validation_type is either 'contains_regex' or 'does_no_contain_regex' - package_manager: Name of the package manager this requirement applies to + package_manager (str): Name of the package manager this requirement applies to """ def _new_package_manager_requirement(mod): diff --git a/lib/ramble/ramble/language/shared_language.py b/lib/ramble/ramble/language/shared_language.py index ba6e0d5ad..3a5af1cac 100644 --- a/lib/ramble/ramble/language/shared_language.py +++ b/lib/ramble/ramble/language/shared_language.py @@ -21,7 +21,7 @@ class Gromacs(ExecutableApplication): # Required package directive - required_package("zlib", package_manager="spack") + required_package("gromacs", package_manager="spack") In the above example, "required_package" is a ramble directive @@ -48,7 +48,7 @@ def archive_pattern(pattern): is being performed. Args: - pattern: Pattern that refers to files to archive + pattern (str): Pattern that refers to files to archive """ def _execute_archive_pattern(obj): @@ -64,11 +64,11 @@ def figure_of_merit_context(name, regex, output_format): Defines a new context to contain figures of merit. Args: - name: High level name of the context. Can be referred to in - the figure of merit - regex: Regular expression, using group names, to match a context. - output_format: String, using python keywords {group_name} to extract - group names from context regular expression. + name (str): High level name of the context. Can be referred to in + the figure of merit + regex (str): Regular expression, using group names, to match a context. + output_format (str): String, using python keywords {group_name} to extract + group names from context regular expression. """ def _execute_figure_of_merit_context(obj): @@ -84,11 +84,14 @@ def figure_of_merit(name, fom_regex, group_name, log_file="{log_file}", units="" Defines a new figure of merit. Args: - name: High level name of the figure of merit - log_file: File the figure of merit can be extracted from - fom_regex: A regular expression using named groups to extract the FOM - group_name: The name of the group that the FOM should be pulled from - units: The units associated with the FOM + name (str): High level name of the figure of merit + log_file (str): File the figure of merit can be extracted from + fom_regex (str): A regular expression using named groups to extract the FOM + group_name (str): The name of the group that the FOM should be pulled from + units (str): The units associated with the FOM + contexts (list(str)): List of contexts (defined by + figure_of_merit_context) this figure of merit + should exist in. """ def _execute_figure_of_merit(obj): @@ -223,21 +226,21 @@ def success_criteria( These will be checked during the analyze step to see if a job exited properly. - Arguments: - name: The name of this success criteria - mode: The type of success criteria that will be validated - Valid values are: 'string', 'application_function', and 'fom_comparison' - match: For mode='string'. Value to check indicate success (if found, it - would mark success) - file: For mode='string'. File success criteria should be located in - fom_name: For mode='fom_comparison'. Name of fom for a criteria. - Accepts globbing. - fom_context: For mode='fom_comparison'. Context the fom is contained - in. Accepts globbing. - formula: For mode='fom_comparison'. Formula to use to evaluate success. - '{value}' keyword is set as the value of the FOM. - anti_match: For mode='string'. Value to check indicate failure. - This setting and `match` are mutually exclusive. + Args: + name (str): The name of this success criteria + mode (str): The type of success criteria that will be validated + Valid values are: 'string', 'application_function', and 'fom_comparison' + match (str): For mode='string'. Value to check indicate success (if found, it + would mark success) + file (str): For mode='string'. File success criteria should be located in + fom_name (str): For mode='fom_comparison'. Name of fom for a criteria. + Accepts globbing. + fom_context (str): For mode='fom_comparison'. Context the fom is contained + in. Accepts globbing. + formula (str): For mode='fom_comparison'. Formula to use to evaluate success. + '{value}' keyword is set as the value of the FOM. + anti_match (str): For mode='string'. Value to check indicate failure. + This setting and `match` are mutually exclusive. """ def _execute_success_criteria(obj): @@ -302,6 +305,14 @@ def example_builtin(self): Options are: - 'prepend' -- This builtin will be injected at the beginning of the executable list - 'append' -- This builtin will be injected at the end of the executable list + + Args: + name (str): Name of builtin (should be the name of a class method) to register + required (boolean): Whether the builtin will be auto-injected or not + injection_method (str): The method of injecting the builtin. Can be + 'prepend' or 'append' + depends_on (list(str)): The names of builtins this builtin depends on + (and must execute after). """ supported_injection_methods = ["prepend", "append"] @@ -339,10 +350,10 @@ def register_phase(name, pipeline=None, run_before=[], run_after=[]): instance of a phase will show up in the resulting dependency list for a phase. Args: - - name: The name of the phase. Phases are functions named '_'. - - pipeline: The name of the pipeline this phase should be registered into. - - run_before: A list of phase names this phase should run before - - run_after: A list of phase names this phase should run after + name (str): The name of the phase. Phases are functions named '_'. + pipeline (str): The name of the pipeline this phase should be registered into. + run_before (list(str)): A list of phase names this phase should run before + run_after (list(str)): A list of phase names this phase should run after """ def _execute_register_phase(obj): @@ -403,7 +414,8 @@ def maintainers(*names: str): """Add a new maintainer directive, to specify maintainers in a declarative way. Args: - names: GitHub username for the maintainer + names (str(s)): GitHub username(s) for the maintainer. Can provide + multiple names as separate arguments. """ def _execute_maintainer(obj): @@ -419,7 +431,8 @@ def tags(*values: str): """Add a new tag directive, to specify tags in a declarative way. Args: - values: Value to mark as a tag + values (str(s)): Values to mark as a tag. Can provide multiple values + as separate arguments. """ def _execute_tag(obj): @@ -438,7 +451,8 @@ def target_shells(shell_support_pattern=None): then it assumes all shells are supported. Args: - shell_support_pattern: The glob pattern that is used to match with the configured shell + shell_support_pattern (str): The glob pattern that is used to match + with the configured shell """ def _execute_target_shells(obj):