Skip to content

Commit

Permalink
fix secondaryFiles
Browse files Browse the repository at this point in the history
  • Loading branch information
@mr-c
mr-c committed Nov 12, 2020
1 parent 7af5a74 commit a244aa3
Show file tree
Hide file tree
Showing 5 changed files with 192 additions and 104 deletions.
89 changes: 50 additions & 39 deletions cwl_utils/cwl_v1_0_expression_refactor.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1222,37 +1222,21 @@ def process_level_reqs(
expression = get_expression(entry.entry, inputs, None)
if expression:
modified = True
if entry.entryname is None:
raise SourceLine(
req.listing,
listing_index,
raise_type=WorkflowException,
).makeError(
"`entryname` is required: {}".format(entry)
if entry.entryname is not None:
entryname_expr = get_expression(
entry.entryname, inputs, None
)
entryname_expr = get_expression(
entry.entryname, inputs, None
)
entryname = (
entry.entryname
if entryname_expr
else '"{}"'.format(entry.entryname)
)
d_target_type = ["File", "Directory"]
target = cwl.InputParameter(
id=None,
type=d_target_type,
)
etool_id = "_expression_{}_InitialWorkDirRequirement_{}".format(
step_name, listing_index
)

new_expression = (
"${var result; var entryname = "
+ entryname
+ "; var entry = "
+ entry.entry[2:-1]
+ """;
entryname = (
entry.entryname
if entryname_expr
else '"{}"'.format(entry.entryname)
)
new_expression = (
"${var result; var entryname = "
+ entryname
+ "; var entry = "
+ entry.entry[2:-1]
+ """;
if (typeof entry === 'string' || entry instanceof String) {
result = {"class": "File", "basename": entryname, "contents": entry} ;
if (typeof entryname === 'string' || entryname instanceof String) {
Expand All @@ -1262,7 +1246,18 @@ def process_level_reqs(
result = entry ;
}
return result; }"""
)
else:
new_expression = expression
d_target_type = ["File", "Directory"]
target = cwl.InputParameter(
id=None,
type=d_target_type,
)
etool_id = "_expression_{}_InitialWorkDirRequirement_{}".format(
step_name, listing_index
)

replace_clt_hintreq_expr_with_etool(
new_expression,
etool_id,
Expand Down Expand Up @@ -1371,8 +1366,8 @@ def traverse_CommandLineTool(
replace_step_clt_expr_with_etool(
expression, etool_id, parent, target, step, replace_etool
)
target_clt.arguments[index].valueFrom = "$(inputs.{})".format(
inp_id
target_clt.arguments[index] = cwl.CommandLineBinding(
valueFrom="$(inputs.{})".format(inp_id)
)
target_clt.inputs.append(
cwl.CommandInputParameter(
Expand Down Expand Up @@ -1555,13 +1550,29 @@ def traverse_CommandLineTool(
remove_JSReq(new_clt_step.run, skip_command_line1)
for new_outp in new_clt_step.run.outputs:
if new_outp.id.split("#")[-1] == outp_id:
if new_outp.outputBinding:
new_outp.outputBinding.outputEval = None
new_outp.outputBinding.loadContents = None
new_outp.type = cwl.CommandOutputArraySchema(
items="File",
type="array",
)
if isinstance(
new_outp,
(
cwl.WorkflowOutputParameter,
cwl.ExpressionToolOutputParameter,
),
):
new_outp.type = cwl.OutputArraySchema(
items="File", type="array"
)
elif isinstance(new_outp, cwl.CommandOutputParameter):
if new_outp.outputBinding:
new_outp.outputBinding.outputEval = None
new_outp.outputBinding.loadContents = None
new_outp.type = cwl.CommandOutputArraySchema(
items="File",
type="array",
)
else:
raise Exception(
"Unimplemented OutputParamter type: %s",
type(new_outp),
)
new_clt_step.in_ = copy.deepcopy(step.in_)
for inp in new_clt_step.in_:
inp.id = inp.id.split("/")[-1]
Expand Down
113 changes: 66 additions & 47 deletions cwl_utils/cwl_v1_1_expression_refactor.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -579,12 +579,18 @@ def empty_inputs(
result[param.id.split("#")[-1]] = example_input(param.type)
else:
for param in process_or_step.in_:
try:
result[param.id.split("/")[-1]] = example_input(
type_for_source(process_or_step.run, param.source, parent)
)
except WorkflowException:
pass
param_id = param.id.split("/")[-1]
if param.source is None and param.valueFrom:
result[param_id] = example_input("string")
elif param.source is None and param.default:
result[param_id] = param.default
else:
try:
result[param_id] = example_input(
type_for_source(process_or_step.run, param.source, parent)
)
except WorkflowException:
pass
return result


Expand Down Expand Up @@ -726,15 +732,17 @@ def process_workflow_inputs_and_outputs(
raise_type=WorkflowException,
).makeError(TOPLEVEL_FORMAT_EXPR_ERROR.format(param.id.split("#")[-1]))
if param.secondaryFiles:
if get_expression(param.secondaryFiles, inputs, EMPTY_FILE):
if hasattr(param.secondaryFiles, "pattern") and get_expression(
param.secondaryFiles.pattern, inputs, EMPTY_FILE
):
raise SourceLine(
param.loadingOptions.original_doc,
"secondaryFiles",
raise_type=WorkflowException,
).makeError(TOPLEVEL_SF_EXPR_ERROR.format(param.id.split("#")[-1]))
elif isinstance(param.secondaryFiles, MutableSequence):
for index2, entry in enumerate(param.secondaryFiles):
if get_expression(entry, inputs, EMPTY_FILE):
if get_expression(entry.pattern, inputs, EMPTY_FILE):
raise SourceLine(
param.loadingOptions.original_doc,
index2,
Expand Down Expand Up @@ -1222,37 +1230,21 @@ def process_level_reqs(
expression = get_expression(entry.entry, inputs, None)
if expression:
modified = True
if entry.entryname is None:
raise SourceLine(
req.listing,
listing_index,
raise_type=WorkflowException,
).makeError(
"`entryname` is required: {}".format(entry)
if entry.entryname is not None:
entryname_expr = get_expression(
entry.entryname, inputs, None
)
entryname_expr = get_expression(
entry.entryname, inputs, None
)
entryname = (
entry.entryname
if entryname_expr
else '"{}"'.format(entry.entryname)
)
d_target_type = ["File", "Directory"]
target = cwl.WorkflowInputParameter(
id=None,
type=d_target_type,
)
etool_id = "_expression_{}_InitialWorkDirRequirement_{}".format(
step_name, listing_index
)

new_expression = (
"${var result; var entryname = "
+ entryname
+ "; var entry = "
+ entry.entry[2:-1]
+ """;
entryname = (
entry.entryname
if entryname_expr
else '"{}"'.format(entry.entryname)
)
new_expression = (
"${var result; var entryname = "
+ entryname
+ "; var entry = "
+ entry.entry[2:-1]
+ """;
if (typeof entry === 'string' || entry instanceof String) {
result = {"class": "File", "basename": entryname, "contents": entry} ;
if (typeof entryname === 'string' || entryname instanceof String) {
Expand All @@ -1262,7 +1254,18 @@ def process_level_reqs(
result = entry ;
}
return result; }"""
)
else:
new_expression = expression
d_target_type = ["File", "Directory"]
target = cwl.WorkflowInputParameter(
id=None,
type=d_target_type,
)
etool_id = "_expression_{}_InitialWorkDirRequirement_{}".format(
step_name, listing_index
)

replace_clt_hintreq_expr_with_etool(
new_expression,
etool_id,
Expand Down Expand Up @@ -1371,8 +1374,8 @@ def traverse_CommandLineTool(
replace_step_clt_expr_with_etool(
expression, etool_id, parent, target, step, replace_etool
)
target_clt.arguments[index].valueFrom = "$(inputs.{})".format(
inp_id
target_clt.arguments[index] = cwl.CommandLineBinding(
valueFrom="$(inputs.{})".format(inp_id)
)
target_clt.inputs.append(
cwl.CommandInputParameter(
Expand Down Expand Up @@ -1555,13 +1558,29 @@ def traverse_CommandLineTool(
remove_JSReq(new_clt_step.run, skip_command_line1)
for new_outp in new_clt_step.run.outputs:
if new_outp.id.split("#")[-1] == outp_id:
if new_outp.outputBinding:
new_outp.outputBinding.outputEval = None
new_outp.outputBinding.loadContents = None
new_outp.type = cwl.CommandOutputArraySchema(
items="File",
type="array",
)
if isinstance(
new_outp,
(
cwl.WorkflowOutputParameter,
cwl.ExpressionToolOutputParameter,
),
):
new_outp.type = cwl.OutputArraySchema(
items="File", type="array"
)
elif isinstance(new_outp, cwl.CommandOutputParameter):
if new_outp.outputBinding:
new_outp.outputBinding.outputEval = None
new_outp.outputBinding.loadContents = None
new_outp.type = cwl.CommandOutputArraySchema(
items="File",
type="array",
)
else:
raise Exception(
"Unimplemented OutputParamter type: %s",
type(new_outp),
)
new_clt_step.in_ = copy.deepcopy(step.in_)
for inp in new_clt_step.in_:
inp.id = inp.id.split("/")[-1]
Expand Down
6 changes: 4 additions & 2 deletions cwl_utils/cwl_v1_2_expression_refactor.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -777,15 +777,17 @@ def process_workflow_inputs_and_outputs(
raise_type=WorkflowException,
).makeError(TOPLEVEL_FORMAT_EXPR_ERROR.format(param.id.split("#")[-1]))
if param.secondaryFiles:
if get_expression(param.secondaryFiles, inputs, EMPTY_FILE):
if hasattr(param.secondaryFiles, "pattern") and get_expression(
param.secondaryFiles.pattern, inputs, EMPTY_FILE
):
raise SourceLine(
param.loadingOptions.original_doc,
"secondaryFiles",
raise_type=WorkflowException,
).makeError(TOPLEVEL_SF_EXPR_ERROR.format(param.id.split("#")[-1]))
elif isinstance(param.secondaryFiles, MutableSequence):
for index2, entry in enumerate(param.secondaryFiles):
if get_expression(entry, inputs, EMPTY_FILE):
if get_expression(entry.pattern, inputs, EMPTY_FILE):
raise SourceLine(
param.loadingOptions.original_doc,
index2,
Expand Down
44 changes: 36 additions & 8 deletions cwl_utils/parser_v1_1.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -301,25 +301,53 @@ def load(self, doc, baseuri, loadingOptions, docRoot=None):


class _SecondaryDSLLoader(_Loader):
def __init__(self, items):
def __init__(self, inner):
# type: (_Loader) -> None
self.items = items
self.inner = inner

def load(self, doc, baseuri, loadingOptions, docRoot=None):
# type: (Any, str, LoadingOptions, Optional[str]) -> Any
r: List[Dict[str, Any]] = []
if isinstance(doc, MutableSequence):
r = [] # type: List[Any]
for d in doc:
if isinstance(d, str):
r.append(d)
if d.endswith("?"):
r.append({"pattern": d[:-1], "required": False})
else:
r.append({"pattern": d})
elif isinstance(d, dict):
new_dict: Dict[str, Any] = {}
if "pattern" in d:
new_dict["pattern"] = d.pop("pattern")
else:
raise ValidationException(
"Missing pattern in secondaryFiles specification entry: {}".format(
d
)
)
new_dict["required"] = (
d.pop("required") if "required" in d else None
)

if len(d):
raise ValidationException(
"Unallowed values in secondaryFiles specification entry: {}".format(
d
)
)

else:
raise ValidationException("Expected str or sequence of str")
doc = r
raise ValidationException(
"Expected a string or sequence of (strings or mappings)."
)
elif isinstance(doc, str):
pass
if doc.endswith("?"):
r.append({"pattern": doc[:-1], "required": False})
else:
r.append({"pattern": doc})
else:
raise ValidationException("Expected str or sequence of str")
return doc
return self.inner.load(r, baseuri, loadingOptions, docRoot)


class _RecordLoader(_Loader):
Expand Down
Loading

0 comments on commit a244aa3

Please sign in to comment.