From a1ca9dca59d2e4a49b4f44ea5e7790a550f87a92 Mon Sep 17 00:00:00 2001 From: Michael Franklin Date: Fri, 29 May 2020 15:57:27 +1000 Subject: [PATCH 1/2] Add intial bash translator --- janis_core/tests/test_translation_bash.py | 9 + janis_core/tool/tool.py | 2 +- .../translationdeps/supportedtranslations.py | 11 +- janis_core/translations/bash.py | 269 ++++++++++++++++++ 4 files changed, 289 insertions(+), 2 deletions(-) create mode 100644 janis_core/tests/test_translation_bash.py create mode 100644 janis_core/translations/bash.py diff --git a/janis_core/tests/test_translation_bash.py b/janis_core/tests/test_translation_bash.py new file mode 100644 index 000000000..eff690c74 --- /dev/null +++ b/janis_core/tests/test_translation_bash.py @@ -0,0 +1,9 @@ +import unittest + +from janis_core.tests.testtools import SingleTestTool +from janis_core.translations.bash import BashTranslator + + +class TestCwlTypesConversion(unittest.TestCase): + bash = BashTranslator().translate_tool_internal(SingleTestTool()) + print(bash) diff --git a/janis_core/tool/tool.py b/janis_core/tool/tool.py index 92c4f1e5c..54381c58c 100644 --- a/janis_core/tool/tool.py +++ b/janis_core/tool/tool.py @@ -167,7 +167,7 @@ def translate( ): raise Exception("Subclass must provide implementation for 'translate()' method") - def bind_metadata(self): + def bind_metadata(self) -> Metadata: """ A convenient place to add metadata about the tool. You are guaranteed that self.metadata will exist. It's possible to return a new instance of the ToolMetadata / WorkflowMetadata which will be rebound. diff --git a/janis_core/translationdeps/supportedtranslations.py b/janis_core/translationdeps/supportedtranslations.py index 0d42bb7e6..b14aa3389 100644 --- a/janis_core/translationdeps/supportedtranslations.py +++ b/janis_core/translationdeps/supportedtranslations.py @@ -4,6 +4,7 @@ class SupportedTranslation(Enum): CWL = "cwl" WDL = "wdl" + SHELL = "shell" def __str__(self): return self.value @@ -20,7 +21,15 @@ def get_translator(self): from ..translations.wdl import WdlTranslator return WdlTranslator() + elif self == SupportedTranslation.SHELL: + from ..translations.bash import BashTranslator + + return BashTranslator() @staticmethod def all(): - return [SupportedTranslation.CWL, SupportedTranslation.WDL] + return [ + SupportedTranslation.CWL, + SupportedTranslation.WDL, + SupportedTranslation.SHELL, + ] diff --git a/janis_core/translations/bash.py b/janis_core/translations/bash.py new file mode 100644 index 000000000..ce7d35b02 --- /dev/null +++ b/janis_core/translations/bash.py @@ -0,0 +1,269 @@ +from typing import Dict, Tuple, List + +from janis_core import Logger +from janis_core.tool.commandtool import ToolArgument, ToolInput +from janis_core.translations import TranslatorBase + +from janis_core.types import ( + InputSelector, + WildcardSelector, + CpuSelector, + StringFormatter, + String, + Selector, + Directory, + Stdout, + Stderr, + Array, + Boolean, + Filename, + File, +) + + +class BashTranslator(TranslatorBase): + def __init__(self): + super().__init__(name="bash") + + @classmethod + def translate_workflow( + cls, + workflow, + with_container=True, + with_resource_overrides=False, + allow_empty_container=False, + container_override: dict = None, + ) -> Tuple[any, Dict[str, any]]: + raise Exception("Not supported for bash translation") + + @classmethod + def translate_tool_internal( + cls, + tool, + with_container=True, + with_resource_overrides=False, + allow_empty_container=False, + container_override: dict = None, + ): + args: List[ToolArgument] = sorted( + [*(tool.arguments() or []), *(tool.inputs() or [])], + key=lambda a: (a.position or 0), + ) + + params_to_include = None + if tool.connections: + params_to_include = set(tool.connections.keys()) + + bc = tool.base_command() + if bc is None: + bc = [] + elif not isinstance(bc, list): + bc = [bc] + + output_args = [] + for a in args: + if isinstance(a, ToolInput): + if params_to_include and a.id() not in params_to_include: + # skip if we're limiting to specific commands + continue + arg = translate_command_input(tool_input=a, inputsdict={}) + if not arg: + Logger.warn(f"Parameter {a.id()} was skipped") + continue + output_args.append(arg) + else: + output_args.append( + translate_command_argument(tool_arg=a, inputsdict={}) + ) + + str_bc = " ".join(f"'{c}'" for c in bc) + command = " \\\n".join([str_bc, *[" " + a for a in output_args]]) + + doc = f"# {tool.id()} bash wrapper" + meta = tool.bind_metadata() or tool.metadata + if params_to_include: + doc += "\n\tNB: this wrapper only contains a subset of the available parameters" + if meta and meta.documentation: + doc += "".join( + "\n# " + l for l in meta.documentation.splitlines(keepends=False) + ) + + return f""" +#!/usr/bin/env sh + +{doc} + +{command} + """ + + @classmethod + def translate_code_tool_internal( + cls, + tool, + with_docker=True, + allow_empty_container=False, + container_override: dict = None, + ): + raise Exception("CodeTool is not currently supported in bash translation") + + @classmethod + def build_inputs_file( + cls, + workflow, + recursive=False, + merge_resources=False, + hints=None, + additional_inputs: Dict = None, + max_cores=None, + max_mem=None, + ) -> Dict[str, any]: + return {} + + @staticmethod + def stringify_translated_workflow(wf): + return wf + + @staticmethod + def stringify_translated_tool(tool): + return tool + + @staticmethod + def stringify_translated_inputs(inputs): + return str(inputs) + + @staticmethod + def workflow_filename(workflow): + return workflow.versioned_id() + ".sh" + + @staticmethod + def tool_filename(tool): + return tool.versioned_id() + ".sh" + + @staticmethod + def inputs_filename(workflow): + return workflow.id() + ".json" + + @staticmethod + def resources_filename(workflow): + return workflow.id() + "-resources.json" + + @staticmethod + def validate_command_for(wfpath, inppath, tools_dir_path, tools_zip_path): + return None + + +def translate_command_argument(tool_arg: ToolArgument, inputsdict=None, **debugkwargs): + # make sure it has some essence of a command line binding, else we'll skip it + if not (tool_arg.position is not None or tool_arg.prefix): + return None + + separate_value_from_prefix = tool_arg.separate_value_from_prefix is not False + prefix = tool_arg.prefix if tool_arg.prefix else "" + tprefix = prefix + + if prefix and separate_value_from_prefix: + tprefix += " " + + name = tool_arg.value + if tool_arg.shell_quote is not False: + return f"{tprefix}'${name}'" if tprefix else f"'${name}'" + else: + return f"{tprefix}${name}" if tprefix else f"${name}" + + +def translate_command_input(tool_input: ToolInput, inputsdict=None, **debugkwargs): + # make sure it has some essence of a command line binding, else we'll skip it + if not (tool_input.position is not None or tool_input.prefix): + return None + + name = tool_input.id() + intype = tool_input.input_type + + optional = (not isinstance(intype, Filename) and intype.optional) or ( + isinstance(tool_input.default, CpuSelector) and tool_input.default is None + ) + position = tool_input.position + + separate_value_from_prefix = tool_input.separate_value_from_prefix is not False + prefix = tool_input.prefix if tool_input.prefix else "" + tprefix = prefix + + intype = tool_input.input_type + + is_flag = isinstance(intype, Boolean) + + if prefix and separate_value_from_prefix and not is_flag: + tprefix += " " + + if isinstance(intype, Boolean): + if tool_input.prefix: + return tool_input.prefix + return "" + elif isinstance(intype, Array): + Logger.critical("Can't bind arrays onto bash yet") + return "" + + # expr = name + # + # separator = tool_input.separator if tool_input.separator is not None else " " + # should_quote = isinstance(intype.subtype(), (String, File, Directory)) + # condition_for_binding = None + # + # if intype.optional: + # expr = f"select_first([{expr}, []])" + # condition_for_binding = ( + # f"(defined({name}) && length(select_first([{name}, []])) > 0)" + # ) + # + # if intype.subtype().optional: + # expr = f"select_all({expr})" + # + # if should_quote: + # if tool_input.prefix_applies_to_all_elements: + # separator = f"'{separator}{tprefix} '" + # else: + # separator = f"'{separator}'" + # + # if tprefix: + # expr = f'"{tprefix}\'" + sep("{separator}", {expr}) + "\'"' + # else: + # expr = f'"\'" + sep("{separator}", {expr}) + "\'"' + # + # else: + # if tprefix: + # expr = f'"{tprefix}" + sep("{separator}", {expr})' + # else: + # expr = f'sep("{separator}", {expr})' + # if condition_for_binding is not None: + # name = f'~{{if {condition_for_binding} then {expr} else ""}}' + # else: + # name = f"~{{{expr}}}" + elif ( + isinstance(intype, (String, File, Directory)) + and tool_input.shell_quote is not False + ): + return f"{tprefix}'${name}'" if tprefix else f"'${name}'" + # if tprefix: + # # if optional: + # # else: + # name = f"{tprefix}'${name}'" + # else: + # # if not optional: + # # else: + # name = f"'${name}'" + + else: + return f"{tprefix}${name}" if tprefix else f"${name}" + # if prefix: + # if optional: + # name = f"~{{if defined({name}) then (\"{tprefix}\" + {name}) else ''}}" + # else: + # name = f"{tprefix}~{{{name}}}" + # else: + # name = f"~{{{name}}}" + + +if __name__ == "__main__": + from janis_unix.tools import Echo + + Echo().translate("shell") From 170ca600c7ad91aad07146af772a896d4ff7277e Mon Sep 17 00:00:00 2001 From: Michael Franklin Date: Tue, 12 Jan 2021 13:21:46 +1100 Subject: [PATCH 2/2] Fix stringformatter import --- janis_core/translations/bash.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/janis_core/translations/bash.py b/janis_core/translations/bash.py index ce7d35b02..65492c53c 100644 --- a/janis_core/translations/bash.py +++ b/janis_core/translations/bash.py @@ -8,7 +8,6 @@ InputSelector, WildcardSelector, CpuSelector, - StringFormatter, String, Selector, Directory, @@ -20,6 +19,8 @@ File, ) +from janis_core.operators import StringFormatter + class BashTranslator(TranslatorBase): def __init__(self): @@ -34,6 +35,11 @@ def translate_workflow( allow_empty_container=False, container_override: dict = None, ) -> Tuple[any, Dict[str, any]]: + workflow_str = "" + + for stp in workflow.steps(): + workflow_str += BashTranslator.translate_tool_internal(stp.tool) + raise Exception("Not supported for bash translation") @classmethod @@ -93,7 +99,9 @@ def translate_tool_internal( {doc} -{command} +{command} \\ + --fastqs $(joinby ' ' $iterable) + """ @classmethod @@ -151,6 +159,9 @@ def resources_filename(workflow): def validate_command_for(wfpath, inppath, tools_dir_path, tools_zip_path): return None + def unwrap_expression(cls, expression): + return str(expression) + def translate_command_argument(tool_arg: ToolArgument, inputsdict=None, **debugkwargs): # make sure it has some essence of a command line binding, else we'll skip it