Merge branch 'master' of github.com:chanzuckerberg/miniwdl into sepfu…

…nction

WDL/CLI.py

@@ -27,7 +27,7 @@
     write_values_json,
     VERBOSE_LEVEL,
     NOTICE_LEVEL,
-    install_coloredlogs,
+    configure_logger,
     parse_byte_size,
     path_really_within,
 )
@@ -389,6 +389,19 @@ def fill_run_subparser(subparsers):
         action="store_true",
         help="upon failure, print error information JSON to standard output (in addition to standard error logging)",
     )
+    group = run_parser.add_argument_group("logging")
+    group.add_argument(
+        "-v",
+        "--verbose",
+        action="store_true",
+        help="increase logging detail & stream tasks' stderr",
+    )
+    group.add_argument(
+        "--no-color",
+        action="store_true",
+        help="disable colored logging and status bar on terminal (also set by NO_COLOR environment variable)",
+    )
+    group.add_argument("--log-json", action="store_true", help="write all logs in JSON")
     group = run_parser.add_argument_group("configuration")
     group.add_argument(
         "--cfg",
@@ -441,17 +454,6 @@ def fill_run_subparser(subparsers):
         action="store_true",
         help="run all containers as the invoking user uid:gid (more secure, but potentially blocks task commands e.g. apt-get)",
     )
-    run_parser.add_argument(
-        "-v",
-        "--verbose",
-        action="store_true",
-        help="increase logging detail & stream tasks' stderr",
-    )
-    run_parser.add_argument(
-        "--no-color",
-        action="store_true",
-        help="disable colored logging on terminal (also set by NO_COLOR environment variable)",
-    )
     # TODO:
     # way to specify None for an optional value (that has a default)
     return run_parser
@@ -476,6 +478,7 @@ def runner(
     as_me=False,
     no_cache=False,
     error_json=False,
+    log_json=False,
     **kwargs,
 ):
     # set up logging
@@ -487,13 +490,19 @@ def runner(
     else:
         logging.raiseExceptions = False
     if kwargs["no_color"]:
-        # picked up by _util.install_coloredlogs()
+        # picked up by _util.configure_logger()
         os.environ["NO_COLOR"] = os.environ.get("NO_COLOR", "")
+    # log_json setting only comes from command line or environment (not cfg file), because we
+    # need it before loading configuration
+    log_json = log_json or (
+        os.environ.get("MINIWDL__LOGGING__JSON", "").lower().strip()
+        in ("t", "y", "1", "true", "yes")
+    )
     logging.basicConfig(level=level)
     logger = logging.getLogger("miniwdl-run")
 
     with ExitStack() as cleanup:
-        set_status = cleanup.enter_context(install_coloredlogs(logger))
+        set_status = cleanup.enter_context(configure_logger(json=log_json))
 
         if os.geteuid() == 0:
             logger.warning(
@@ -511,6 +520,7 @@ def runner(
             "file_io": {},
             "task_runtime": {},
             "download_cache": {},
+            "logging": {"json": log_json},
         }
         if max_tasks is not None:
             cfg_overrides["scheduler"]["call_concurrency"] = max_tasks
@@ -965,6 +975,7 @@ def fill_run_self_test_subparser(subparsers):
         default=None,
         help="configuration file to load (in preference to file named by MINIWDL_CFG environment, or XDG_CONFIG_{HOME,DIRS}/miniwdl.cfg)",
     )
+    run_parser.add_argument("--log-json", action="store_true", help="write all logs in JSON")
     run_parser.add_argument(
         "--as-me", action="store_true", help="run all containers as the current user uid:gid"
     )
@@ -1023,14 +1034,16 @@ def run_self_test(**kwargs):
         os.path.join(dn, "test.wdl"),
         "who=https://raw.githubusercontent.com/chanzuckerberg/miniwdl/master/tests/alyssa_ben.txt",
         "--dir",
-        dn,
+        dn if dn not in [".", "./"] else os.getcwd(),
         "--debug",
     ]
     if kwargs["as_me"]:
         argv.append("--as-me")
     if kwargs["cfg"]:
         argv.append("--cfg")
         argv.append(kwargs["cfg"])
+    if kwargs["log_json"]:
+        argv.append("--log-json")
     try:
         outputs = main(argv)["outputs"]
         assert len(outputs["hello_caller.messages"]) == 2
@@ -1364,7 +1377,7 @@ def localize(
 ):
     logging.basicConfig(level=NOTICE_LEVEL)
     logger = logging.getLogger("miniwdl-localize")
-    with install_coloredlogs(logger) as set_status:
+    with configure_logger() as set_status:
 
         cfg_arg = None
         if cfg:

WDL/Lint.py

@@ -972,3 +972,31 @@ def task(self, obj: Tree.Task) -> Any:
         for k in obj.runtime:
             if k not in self.known_keys:
                 self.add(obj, "unknown entry in task runtime section: " + k, obj.runtime[k].pos)
+
+
+@a_linter
+class MissingVersion(Linter):
+    def document(self, obj: Tree.Document) -> Any:
+        first_sloc = next(
+            (
+                p
+                for p in enumerate(line.lstrip() for line in obj.source_lines)
+                if p[1] and p[1][0] != "#"
+            ),
+            None,
+        )
+        # (don't bother with this warning if the document is effectively empty)
+        if first_sloc and obj.wdl_version is None:
+            line = (first_sloc[0] + 1) if first_sloc else obj.pos.line
+            self.add(
+                obj,
+                "document should declare WDL version; draft-2 assumed",
+                Error.SourcePosition(
+                    uri=obj.pos.uri,
+                    abspath=obj.pos.abspath,
+                    line=line,
+                    end_line=line,
+                    column=1,
+                    end_column=1,
+                ),
+            )

WDL/StdLib.py

@@ -117,13 +117,16 @@ def sep(sep: Value.String, iterable: Value.Array) -> Value.String:
         # infer_type logic
         self.range = _Range()
         self.prefix = _Prefix()
+        self.suffix = _Suffix()
         self.size = _Size(self)
         self.select_first = _SelectFirst()
         self.select_all = _SelectAll()
         self.zip = _Zip()
         self.cross = _Cross()
         self.flatten = _Flatten()
         self.transpose = _Transpose()
+        self.quote = _Quote()
+        self.squote = _Quote(squote=True)
 
     def _read(self, parse: Callable[[str], Value.Base]) -> Callable[[Value.File], Value.Base]:
         "generate read_* function implementation based on parse"
@@ -887,3 +890,57 @@ def _call_eager(self, expr: "Expr.Apply", arguments: List[Value.Base]) -> Value.
             Type.String(),
             [Value.String(pfx + s.coerce(Type.String()).value) for s in arguments[1].value],
         )
+
+
+class _Suffix(EagerFunction):
+    # string -> t array -> string array
+    # if input array is nonempty then so is output
+    # Append a suffix to every element within the array
+
+    def infer_type(self, expr: "Expr.Apply") -> Type.Base:
+        if len(expr.arguments) != 2:
+            raise Error.WrongArity(expr, 2)
+        expr.arguments[0].typecheck(Type.String())
+        expr.arguments[1].typecheck(Type.Array(Type.String()))
+        arg1ty = expr.arguments[1].type
+        return Type.Array(
+            Type.String(), nonempty=(isinstance(arg1ty, Type.Array) and arg1ty.nonempty)
+        )
+
+    def _call_eager(self, expr: "Expr.Apply", arguments: List[Value.Base]) -> Value.Base:
+        sfx = arguments[0].coerce(Type.String()).value
+        return Value.Array(
+            Type.String(),
+            [Value.String(s.coerce(Type.String()).value + sfx) for s in arguments[1].value],
+        )
+
+
+class _Quote(EagerFunction):
+    # t array -> string array
+    # if input array is nonempty then so is output
+    # Append a suffix to every element within the array
+
+    def __init__(self, squote: bool = False) -> None:
+        if squote:
+            self.quote = "'"
+        else:
+            self.quote = '"'
+
+    def infer_type(self, expr: "Expr.Apply") -> Type.Base:
+        if len(expr.arguments) != 1:
+            raise Error.WrongArity(expr, 1)
+        expr.arguments[0].typecheck(Type.Array(Type.String()))
+        arg0ty = expr.arguments[0].type
+        nonempty = isinstance(arg0ty, Type.Array) and arg0ty.nonempty
+        return Type.Array(
+            Type.String(), nonempty=(isinstance(arg0ty, Type.Array) and arg0ty.nonempty)
+        )
+
+    def _call_eager(self, expr: "Expr.Apply", arguments: List[Value.Base]) -> Value.Base:
+        return Value.Array(
+            Type.String(),
+            [
+                Value.String("{0}{1}{0}".format(self.quote, s.coerce(Type.String()).value))
+                for s in arguments[0].value
+            ],
+        )

WDL/Tree.py

@@ -1187,6 +1187,13 @@ class Document(SourceNode):
     corresponding source line, or ``None`` if the line has no comment.
     """
 
+    wdl_version: Optional[str]
+    """
+    :type: Optional[str]
+
+    Declared WDL language version; if absent, then assume draft-2.
+    """
+
     imports: List[DocImport]
     """
     :type: List[DocImport]
@@ -1208,6 +1215,7 @@ def __init__(
         tasks: List[Task],
         workflow: Optional[Workflow],
         comments: List[SourceComment],
+        wdl_version: Optional[str],
     ) -> None:
         super().__init__(pos)
         self.imports = imports
@@ -1219,6 +1227,7 @@ def __init__(
         self.source_text = source_text
         self.source_lines = source_text.split("\n")
         self.source_comments = [None for _ in self.source_lines]
+        self.wdl_version = wdl_version
         for comment in comments:
             assert self.source_comments[comment.pos.line - 1] is None
             assert self.source_lines[comment.pos.line - 1].endswith(comment.text)

WDL/__init__.py

@@ -263,5 +263,5 @@ def values_to_json(values_env: Env.Bindings[Value.Base], namespace: str = "") ->
         else:
             assert isinstance(item.value, Type.Base)
             j = str(item.value)
-        ans[namespace + item.name] = j
+        ans[(namespace if not item.name.startswith("_") else "") + item.name] = j
     return ans

WDL/_parser.py

@@ -241,14 +241,22 @@ class _DocTransformer(_ExprTransformer, _TypeTransformer):
     _keywords: Set[str]
     _source_text: str
     _comments: List[lark.Token]
+    _version: Optional[str]
 
     def __init__(
-        self, source_text: str, keywords: Set[str], comments: List[lark.Token], *args, **kwargs
+        self,
+        source_text: str,
+        keywords: Set[str],
+        comments: List[lark.Token],
+        version: Optional[str],
+        *args,
+        **kwargs,
     ):
         super().__init__(*args, **kwargs)
         self._source_text = source_text
         self._keywords = keywords
         self._comments = comments
+        self._version = version
 
     def _check_keyword(self, pos, name):
         if name in self._keywords:
@@ -534,7 +542,14 @@ def document(self, items, meta):
         ]
 
         return Tree.Document(
-            self._source_text, self._sp(meta), imports, structs, tasks, workflow, comments
+            self._source_text,
+            self._sp(meta),
+            imports,
+            structs,
+            tasks,
+            workflow,
+            comments,
+            self._version,
         )
 
 
@@ -566,9 +581,9 @@ def parse_tasks(txt: str, version: Optional[str] = None) -> List[Tree.Task]:
     try:
         (grammar, keywords) = _grammar.get(version)
         raw_ast, comments = parse(grammar, txt, "tasks")
-        return _DocTransformer(source_text=txt, keywords=keywords, comments=comments).transform(
-            raw_ast
-        )
+        return _DocTransformer(
+            source_text=txt, keywords=keywords, comments=comments, version=version
+        ).transform(raw_ast)
     except lark.exceptions.VisitError as exn:
         raise exn.__context__
 
@@ -578,25 +593,30 @@ def parse_document(
 ) -> Tree.Document:
     npos = SourcePosition(uri=uri, abspath=abspath, line=0, column=0, end_line=0, end_column=0)
     if not txt.strip():
-        return Tree.Document(txt, npos, [], {}, [], None, [])
-    if version is None:
-        # for now assume the version is 1.0 if the first line is "version <number>"
-        # otherwise draft-2
-        version = "draft-2"
-        for line in txt.split("\n"):
-            line = line.strip()
-            if line and line[0] != "#":
-                if line.startswith("version "):
-                    version = line[8:]
-                break
+        return Tree.Document(txt, npos, [], {}, [], None, [], None)
+    declared_version = None
+    for line in txt.split("\n"):
+        line = line.strip()
+        if line and line[0] != "#":
+            if line.startswith("version "):
+                declared_version = line[8:]
+            break
+    version = version or declared_version or "draft-2"
     try:
         (grammar, keywords) = _grammar.get(version)
     except KeyError:
-        raise Error.SyntaxError(npos, "unknown WDL version " + version) from None
+        raise Error.SyntaxError(
+            npos, f"unknown WDL version {version}; choices: " + ", ".join(_grammar.versions.keys())
+        ) from None
     try:
         raw_ast, comments = parse(grammar, txt, "document")
         return _DocTransformer(
-            source_text=txt, uri=uri, abspath=abspath, keywords=keywords, comments=comments
+            source_text=txt,
+            uri=uri,
+            abspath=abspath,
+            keywords=keywords,
+            comments=comments,
+            version=declared_version,
         ).transform(raw_ast)
     except lark.exceptions.UnexpectedInput as exn:
         pos = SourcePosition(