From 19d9198bb04448eadd568c2494a6935efd952af0 Mon Sep 17 00:00:00 2001 From: Giulio Frasca Date: Thu, 5 Sep 2024 14:25:50 -0400 Subject: [PATCH] fix(backend): pass correct ParentDagID to iterator DAG - Passthrough ParentDagID rather than DriverExecutionID to iterator such that iteration item correctly detects dependentTasks. - Remove depends from iterator DAG as it is already handled by root-level task - Update Iterator template names/nomenclature for clarity - Update tests accordingly Signed-off-by: Giulio Frasca --- backend/src/v2/compiler/argocompiler/dag.go | 13 +- .../testdata/multiple_parallel_loops.yaml | 147 +++++++----------- .../testdata/multiple_parallel_loops.json | 8 +- 3 files changed, 69 insertions(+), 99 deletions(-) diff --git a/backend/src/v2/compiler/argocompiler/dag.go b/backend/src/v2/compiler/argocompiler/dag.go index 523992c2a4b0..61687bf44811 100644 --- a/backend/src/v2/compiler/argocompiler/dag.go +++ b/backend/src/v2/compiler/argocompiler/dag.go @@ -274,10 +274,9 @@ func (c *workflowCompiler) iteratorTask(name string, task *pipelinespec.Pipeline // Set up Loop Control Template loopDriverArgoName := name + "-loop-driver" loopDriverInputs := dagDriverInputs{ - component: componentSpecPlaceholder, - parentDagID: parentDagID, - task: taskJson, // TODO(Bobgy): avoid duplicating task JSON twice in the template. - iterationIndex: "0", + component: componentSpecPlaceholder, + parentDagID: parentDagID, + task: taskJson, // TODO(Bobgy): avoid duplicating task JSON twice in the template. } loopDriver, loopDriverOutputs, err := c.dagDriverTask(loopDriverArgoName, loopDriverInputs) if err != nil { @@ -305,7 +304,7 @@ func (c *workflowCompiler) iteratorTask(name string, task *pipelinespec.Pipeline loopTmpl.Parallelism = ¶llellism_limit } - loopTmplName, err := c.addTemplate(loopTmpl, componentName+"-loop-"+name) + loopTmplName, err := c.addTemplate(loopTmpl, fmt.Sprintf("%s-loop-iterator", componentName)) if err != nil { return nil, err } @@ -325,7 +324,7 @@ func (c *workflowCompiler) iteratorTask(name string, task *pipelinespec.Pipeline Parameters: []wfapi.Parameter{ { Name: paramParentDagID, - Value: wfapi.AnyStringPtr(loopDriverOutputs.executionID), + Value: wfapi.AnyStringPtr(parentDagID), }, }, }, @@ -382,7 +381,7 @@ func (c *workflowCompiler) iterationItemTask(name string, task *pipelinespec.Pip Tasks: iterationTasks, }, } - iterationsTmplName, err := c.addTemplate(iterationsTmpl, componentName+"-"+name) + iterationsTmplName, err := c.addTemplate(iterationsTmpl, componentName+"-iteration") if err != nil { return nil, err } diff --git a/backend/src/v2/compiler/argocompiler/testdata/multiple_parallel_loops.yaml b/backend/src/v2/compiler/argocompiler/testdata/multiple_parallel_loops.yaml index 2e312fb0ce63..c7191f52bae1 100755 --- a/backend/src/v2/compiler/argocompiler/testdata/multiple_parallel_loops.yaml +++ b/backend/src/v2/compiler/argocompiler/testdata/multiple_parallel_loops.yaml @@ -1,68 +1,39 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - annotations: - pipelines.kubeflow.org/components-comp-for-loop-2: '{"dag":{"tasks":{"print-op":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op"},"inputs":{"parameters":{"s":{"componentInputParameter":"pipelinechannel--loop-item-param-1","parameterExpressionSelector":"parseJson(string_value)[\"A_a\"]"}}},"taskInfo":{"name":"print-op"}},"print-op-2":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-2"},"inputs":{"parameters":{"s":{"componentInputParameter":"pipelinechannel--loop-item-param-1","parameterExpressionSelector":"parseJson(string_value)[\"B_b\"]"}}},"taskInfo":{"name":"print-op-2"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--loop-item-param-1":{"parameterType":"STRUCT"}}}}' - pipelines.kubeflow.org/components-comp-for-loop-4: '{"dag":{"tasks":{"print-op-3":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-3"},"inputs":{"parameters":{"s":{"componentInputParameter":"pipelinechannel--loop-item-param-3","parameterExpressionSelector":"parseJson(string_value)[\"A_a\"]"}}},"taskInfo":{"name":"print-op-3"}},"print-op-4":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-4"},"inputs":{"parameters":{"s":{"componentInputParameter":"pipelinechannel--loop-item-param-3","parameterExpressionSelector":"parseJson(string_value)[\"B_b\"]"}}},"taskInfo":{"name":"print-op-4"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--loop-item-param-3":{"parameterType":"STRUCT"}}}}' - pipelines.kubeflow.org/components-comp-print-op: '{"executorLabel":"exec-print-op","inputDefinitions":{"parameters":{"s":{"parameterType":"STRING"}}}}' - pipelines.kubeflow.org/components-comp-print-op-2: '{"executorLabel":"exec-print-op-2","inputDefinitions":{"parameters":{"s":{"parameterType":"STRING"}}}}' - pipelines.kubeflow.org/components-comp-print-op-3: '{"executorLabel":"exec-print-op-3","inputDefinitions":{"parameters":{"s":{"parameterType":"STRING"}}}}' - pipelines.kubeflow.org/components-comp-print-op-4: '{"executorLabel":"exec-print-op-4","inputDefinitions":{"parameters":{"s":{"parameterType":"STRING"}}}}' - pipelines.kubeflow.org/components-root: '{"dag":{"tasks":{"for-loop-2":{"componentRef":{"name":"comp-for-loop-2"},"iteratorPolicy":{"parallelismLimit":2},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[{\"A_a\": - \"1\", \"B_b\": \"10\"}, {\"A_a\": \"2\", \"B_b\": \"20\"}, {\"A_a\": \"3\", - \"B_b\": \"30\"}, {\"A_a\": \"4\", \"B_b\": \"40\"}, {\"A_a\": \"5\", \"B_b\": - \"50\"}, {\"A_a\": \"6\", \"B_b\": \"60\"}, {\"A_a\": \"7\", \"B_b\": \"70\"}, - {\"A_a\": \"8\", \"B_b\": \"80\"}, {\"A_a\": \"9\", \"B_b\": \"90\"}, {\"A_a\": - \"10\", \"B_b\": \"100\"}]"}},"taskInfo":{"name":"chicken"}},"for-loop-4":{"componentRef":{"name":"comp-for-loop-4"},"iteratorPolicy":{"parallelismLimit":4},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-3","items":{"raw":"[{\"A_a\": - \"1\", \"B_b\": \"10\"}, {\"A_a\": \"2\", \"B_b\": \"20\"}, {\"A_a\": \"3\", - \"B_b\": \"30\"}, {\"A_a\": \"4\", \"B_b\": \"40\"}, {\"A_a\": \"5\", \"B_b\": - \"50\"}, {\"A_a\": \"6\", \"B_b\": \"60\"}, {\"A_a\": \"7\", \"B_b\": \"70\"}, - {\"A_a\": \"8\", \"B_b\": \"80\"}, {\"A_a\": \"9\", \"B_b\": \"90\"}, {\"A_a\": - \"10\", \"B_b\": \"100\"}]"}},"taskInfo":{"name":"muffin"}}}}}' - pipelines.kubeflow.org/implementations-comp-print-op: '{"args":["--executor_input","{{$}}","--function_to_execute","print_op"],"command":["sh","-c","\nif - ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m - ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 - python3 -m pip install --quiet --no-warn-script-location ''kfp==2.7.0'' ''--no-deps'' - ''typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"'' \u0026\u0026 - \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" - \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m - kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport - kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - print_op(s: str):\n print(s)\n\n"],"image":"python:3.7"}' - pipelines.kubeflow.org/implementations-comp-print-op-2: '{"args":["--executor_input","{{$}}","--function_to_execute","print_op"],"command":["sh","-c","\nif - ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m - ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 - python3 -m pip install --quiet --no-warn-script-location ''kfp==2.7.0'' ''--no-deps'' - ''typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"'' \u0026\u0026 - \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" - \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m - kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport - kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - print_op(s: str):\n print(s)\n\n"],"image":"python:3.7"}' - pipelines.kubeflow.org/implementations-comp-print-op-3: '{"args":["--executor_input","{{$}}","--function_to_execute","print_op"],"command":["sh","-c","\nif - ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m - ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 - python3 -m pip install --quiet --no-warn-script-location ''kfp==2.7.0'' ''--no-deps'' - ''typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"'' \u0026\u0026 - \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" - \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m - kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport - kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - print_op(s: str):\n print(s)\n\n"],"image":"python:3.7"}' - pipelines.kubeflow.org/implementations-comp-print-op-4: '{"args":["--executor_input","{{$}}","--function_to_execute","print_op"],"command":["sh","-c","\nif - ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m - ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 - python3 -m pip install --quiet --no-warn-script-location ''kfp==2.7.0'' ''--no-deps'' - ''typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"'' \u0026\u0026 - \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" - \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m - kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport - kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - print_op(s: str):\n print(s)\n\n"],"image":"python:3.7"}' - creationTimestamp: null generateName: my-pipeline- spec: - arguments: {} + arguments: + parameters: + - name: components-e7a1060777c9ef84e36a4d54f25d3102abbcbd62d4c8bbb5883c3a2cbcfb5c6d + value: '{"executorLabel":"exec-print-op","inputDefinitions":{"parameters":{"s":{"parameterType":"STRING"}}}}' + - name: implementations-e7a1060777c9ef84e36a4d54f25d3102abbcbd62d4c8bbb5883c3a2cbcfb5c6d + value: '{"args":["--executor_input","{{$}}","--function_to_execute","print_op"],"command":["sh","-c","\nif + ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m + ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 + python3 -m pip install --quiet --no-warn-script-location ''kfp==2.7.0'' ''--no-deps'' + ''typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"'' \u0026\u0026 + \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" + \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m + kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport + kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef + print_op(s: str):\n print(s)\n\n"],"image":"python:3.7"}' + - name: components-comp-for-loop-2 + value: '{"dag":{"tasks":{"print-op":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op"},"inputs":{"parameters":{"s":{"componentInputParameter":"pipelinechannel--loop-item-param-1","parameterExpressionSelector":"parseJson(string_value)[\"A_a\"]"}}},"taskInfo":{"name":"print-op"}},"print-op-2":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-2"},"inputs":{"parameters":{"s":{"componentInputParameter":"pipelinechannel--loop-item-param-1","parameterExpressionSelector":"parseJson(string_value)[\"B_b\"]"}}},"taskInfo":{"name":"print-op-2"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--loop-item-param-1":{"parameterType":"STRUCT"}}}}' + - name: components-comp-for-loop-4 + value: '{"dag":{"tasks":{"print-op-3":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-3"},"inputs":{"parameters":{"s":{"componentInputParameter":"pipelinechannel--loop-item-param-3","parameterExpressionSelector":"parseJson(string_value)[\"A_a\"]"}}},"taskInfo":{"name":"print-op-3"}},"print-op-4":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-4"},"inputs":{"parameters":{"s":{"componentInputParameter":"pipelinechannel--loop-item-param-3","parameterExpressionSelector":"parseJson(string_value)[\"B_b\"]"}}},"taskInfo":{"name":"print-op-4"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--loop-item-param-3":{"parameterType":"STRUCT"}}}}' + - name: components-root + value: '{"dag":{"tasks":{"for-loop-2":{"componentRef":{"name":"comp-for-loop-2"},"iteratorPolicy":{"parallelismLimit":2},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[{\"A_a\": + \"1\", \"B_b\": \"10\"}, {\"A_a\": \"2\", \"B_b\": \"20\"}, {\"A_a\": \"3\", + \"B_b\": \"30\"}, {\"A_a\": \"4\", \"B_b\": \"40\"}, {\"A_a\": \"5\", \"B_b\": + \"50\"}, {\"A_a\": \"6\", \"B_b\": \"60\"}, {\"A_a\": \"7\", \"B_b\": \"70\"}, + {\"A_a\": \"8\", \"B_b\": \"80\"}, {\"A_a\": \"9\", \"B_b\": \"90\"}, {\"A_a\": + \"10\", \"B_b\": \"100\"}]"}},"taskInfo":{"name":"foo"}},"for-loop-4":{"componentRef":{"name":"comp-for-loop-4"},"iteratorPolicy":{"parallelismLimit":4},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-3","items":{"raw":"[{\"A_a\": + \"1\", \"B_b\": \"10\"}, {\"A_a\": \"2\", \"B_b\": \"20\"}, {\"A_a\": \"3\", + \"B_b\": \"30\"}, {\"A_a\": \"4\", \"B_b\": \"40\"}, {\"A_a\": \"5\", \"B_b\": + \"50\"}, {\"A_a\": \"6\", \"B_b\": \"60\"}, {\"A_a\": \"7\", \"B_b\": \"70\"}, + {\"A_a\": \"8\", \"B_b\": \"80\"}, {\"A_a\": \"9\", \"B_b\": \"90\"}, {\"A_a\": + \"10\", \"B_b\": \"100\"}]"}},"taskInfo":{"name":"bar"}}}}}' entrypoint: entrypoint podMetadata: annotations: @@ -230,11 +201,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.annotations.pipelines.kubeflow.org/components-comp-print-op}}' + value: '{{workflow.parameters.components-e7a1060777c9ef84e36a4d54f25d3102abbcbd62d4c8bbb5883c3a2cbcfb5c6d}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op"},"inputs":{"parameters":{"s":{"componentInputParameter":"pipelinechannel--loop-item-param-1","parameterExpressionSelector":"parseJson(string_value)[\"A_a\"]"}}},"taskInfo":{"name":"print-op"}}' - name: container - value: '{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-print-op}}' + value: '{{workflow.parameters.implementations-e7a1060777c9ef84e36a4d54f25d3102abbcbd62d4c8bbb5883c3a2cbcfb5c6d}}' - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' name: print-op-driver @@ -252,11 +223,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.annotations.pipelines.kubeflow.org/components-comp-print-op-2}}' + value: '{{workflow.parameters.components-e7a1060777c9ef84e36a4d54f25d3102abbcbd62d4c8bbb5883c3a2cbcfb5c6d}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-2"},"inputs":{"parameters":{"s":{"componentInputParameter":"pipelinechannel--loop-item-param-1","parameterExpressionSelector":"parseJson(string_value)[\"B_b\"]"}}},"taskInfo":{"name":"print-op-2"}}' - name: container - value: '{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-print-op-2}}' + value: '{{workflow.parameters.implementations-e7a1060777c9ef84e36a4d54f25d3102abbcbd62d4c8bbb5883c3a2cbcfb5c6d}}' - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' name: print-op-2-driver @@ -282,11 +253,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.annotations.pipelines.kubeflow.org/components-comp-print-op-3}}' + value: '{{workflow.parameters.components-e7a1060777c9ef84e36a4d54f25d3102abbcbd62d4c8bbb5883c3a2cbcfb5c6d}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-3"},"inputs":{"parameters":{"s":{"componentInputParameter":"pipelinechannel--loop-item-param-3","parameterExpressionSelector":"parseJson(string_value)[\"A_a\"]"}}},"taskInfo":{"name":"print-op-3"}}' - name: container - value: '{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-print-op-3}}' + value: '{{workflow.parameters.implementations-e7a1060777c9ef84e36a4d54f25d3102abbcbd62d4c8bbb5883c3a2cbcfb5c6d}}' - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' name: print-op-3-driver @@ -304,11 +275,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.annotations.pipelines.kubeflow.org/components-comp-print-op-4}}' + value: '{{workflow.parameters.components-e7a1060777c9ef84e36a4d54f25d3102abbcbd62d4c8bbb5883c3a2cbcfb5c6d}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-4"},"inputs":{"parameters":{"s":{"componentInputParameter":"pipelinechannel--loop-item-param-3","parameterExpressionSelector":"parseJson(string_value)[\"B_b\"]"}}},"taskInfo":{"name":"print-op-4"}}' - name: container - value: '{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-print-op-4}}' + value: '{{workflow.parameters.implementations-e7a1060777c9ef84e36a4d54f25d3102abbcbd62d4c8bbb5883c3a2cbcfb5c6d}}' - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' name: print-op-4-driver @@ -397,7 +368,7 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.annotations.pipelines.kubeflow.org/components-comp-for-loop-2}}' + value: '{{workflow.parameters.components-comp-for-loop-2}}' - name: iteration-index value: '{{inputs.parameters.iteration-index}}' - name: parent-dag-id @@ -408,7 +379,7 @@ spec: \"3\", \"B_b\": \"30\"}, {\"A_a\": \"4\", \"B_b\": \"40\"}, {\"A_a\": \"5\", \"B_b\": \"50\"}, {\"A_a\": \"6\", \"B_b\": \"60\"}, {\"A_a\": \"7\", \"B_b\": \"70\"}, {\"A_a\": \"8\", \"B_b\": \"80\"}, {\"A_a\": - \"9\", \"B_b\": \"90\"}, {\"A_a\": \"10\", \"B_b\": \"100\"}]"}},"taskInfo":{"name":"chicken"}}' + \"9\", \"B_b\": \"90\"}, {\"A_a\": \"10\", \"B_b\": \"100\"}]"}},"taskInfo":{"name":"foo"}}' name: iteration-item-driver template: system-dag-driver - arguments: @@ -432,7 +403,7 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.annotations.pipelines.kubeflow.org/components-comp-for-loop-2}}' + value: '{{workflow.parameters.components-comp-for-loop-2}}' - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' - name: task @@ -441,7 +412,7 @@ spec: \"3\", \"B_b\": \"30\"}, {\"A_a\": \"4\", \"B_b\": \"40\"}, {\"A_a\": \"5\", \"B_b\": \"50\"}, {\"A_a\": \"6\", \"B_b\": \"60\"}, {\"A_a\": \"7\", \"B_b\": \"70\"}, {\"A_a\": \"8\", \"B_b\": \"80\"}, {\"A_a\": - \"9\", \"B_b\": \"90\"}, {\"A_a\": \"10\", \"B_b\": \"100\"}]"}},"taskInfo":{"name":"chicken"}}' + \"9\", \"B_b\": \"90\"}, {\"A_a\": \"10\", \"B_b\": \"100\"}]"}},"taskInfo":{"name":"foo"}}' name: iteration-driver template: system-dag-driver - arguments: @@ -459,7 +430,7 @@ spec: parameters: - name: parent-dag-id metadata: {} - name: comp-for-loop-2-loop-for-loop-2 + name: comp-for-loop-2-loop-iterator outputs: {} parallelism: 2 - dag: @@ -467,7 +438,7 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.annotations.pipelines.kubeflow.org/components-comp-for-loop-4}}' + value: '{{workflow.parameters.components-comp-for-loop-4}}' - name: iteration-index value: '{{inputs.parameters.iteration-index}}' - name: parent-dag-id @@ -478,7 +449,7 @@ spec: \"3\", \"B_b\": \"30\"}, {\"A_a\": \"4\", \"B_b\": \"40\"}, {\"A_a\": \"5\", \"B_b\": \"50\"}, {\"A_a\": \"6\", \"B_b\": \"60\"}, {\"A_a\": \"7\", \"B_b\": \"70\"}, {\"A_a\": \"8\", \"B_b\": \"80\"}, {\"A_a\": - \"9\", \"B_b\": \"90\"}, {\"A_a\": \"10\", \"B_b\": \"100\"}]"}},"taskInfo":{"name":"muffin"}}' + \"9\", \"B_b\": \"90\"}, {\"A_a\": \"10\", \"B_b\": \"100\"}]"}},"taskInfo":{"name":"bar"}}' name: iteration-item-driver template: system-dag-driver - arguments: @@ -502,7 +473,7 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.annotations.pipelines.kubeflow.org/components-comp-for-loop-4}}' + value: '{{workflow.parameters.components-comp-for-loop-4}}' - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' - name: task @@ -511,7 +482,7 @@ spec: \"3\", \"B_b\": \"30\"}, {\"A_a\": \"4\", \"B_b\": \"40\"}, {\"A_a\": \"5\", \"B_b\": \"50\"}, {\"A_a\": \"6\", \"B_b\": \"60\"}, {\"A_a\": \"7\", \"B_b\": \"70\"}, {\"A_a\": \"8\", \"B_b\": \"80\"}, {\"A_a\": - \"9\", \"B_b\": \"90\"}, {\"A_a\": \"10\", \"B_b\": \"100\"}]"}},"taskInfo":{"name":"muffin"}}' + \"9\", \"B_b\": \"90\"}, {\"A_a\": \"10\", \"B_b\": \"100\"}]"}},"taskInfo":{"name":"bar"}}' name: iteration-driver template: system-dag-driver - arguments: @@ -529,7 +500,7 @@ spec: parameters: - name: parent-dag-id metadata: {} - name: comp-for-loop-4-loop-for-loop-4 + name: comp-for-loop-4-loop-iterator outputs: {} parallelism: 4 - dag: @@ -537,7 +508,7 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.annotations.pipelines.kubeflow.org/components-comp-for-loop-2}}' + value: '{{workflow.parameters.components-comp-for-loop-2}}' - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' - name: task @@ -546,20 +517,20 @@ spec: \"3\", \"B_b\": \"30\"}, {\"A_a\": \"4\", \"B_b\": \"40\"}, {\"A_a\": \"5\", \"B_b\": \"50\"}, {\"A_a\": \"6\", \"B_b\": \"60\"}, {\"A_a\": \"7\", \"B_b\": \"70\"}, {\"A_a\": \"8\", \"B_b\": \"80\"}, {\"A_a\": - \"9\", \"B_b\": \"90\"}, {\"A_a\": \"10\", \"B_b\": \"100\"}]"}},"taskInfo":{"name":"chicken"}}' + \"9\", \"B_b\": \"90\"}, {\"A_a\": \"10\", \"B_b\": \"100\"}]"}},"taskInfo":{"name":"foo"}}' name: for-loop-2-loop-driver template: system-dag-driver - arguments: parameters: - name: parent-dag-id - value: '{{tasks.for-loop-2-loop-driver.outputs.parameters.execution-id}}' + value: '{{inputs.parameters.parent-dag-id}}' depends: for-loop-2-loop-driver.Succeeded name: for-loop-2-loop - template: comp-for-loop-2-loop-for-loop-2 + template: comp-for-loop-2-loop-iterator - arguments: parameters: - name: component - value: '{{workflow.annotations.pipelines.kubeflow.org/components-comp-for-loop-4}}' + value: '{{workflow.parameters.components-comp-for-loop-4}}' - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' - name: task @@ -568,16 +539,16 @@ spec: \"3\", \"B_b\": \"30\"}, {\"A_a\": \"4\", \"B_b\": \"40\"}, {\"A_a\": \"5\", \"B_b\": \"50\"}, {\"A_a\": \"6\", \"B_b\": \"60\"}, {\"A_a\": \"7\", \"B_b\": \"70\"}, {\"A_a\": \"8\", \"B_b\": \"80\"}, {\"A_a\": - \"9\", \"B_b\": \"90\"}, {\"A_a\": \"10\", \"B_b\": \"100\"}]"}},"taskInfo":{"name":"muffin"}}' + \"9\", \"B_b\": \"90\"}, {\"A_a\": \"10\", \"B_b\": \"100\"}]"}},"taskInfo":{"name":"bar"}}' name: for-loop-4-loop-driver template: system-dag-driver - arguments: parameters: - name: parent-dag-id - value: '{{tasks.for-loop-4-loop-driver.outputs.parameters.execution-id}}' + value: '{{inputs.parameters.parent-dag-id}}' depends: for-loop-4-loop-driver.Succeeded name: for-loop-4-loop - template: comp-for-loop-4-loop-for-loop-4 + template: comp-for-loop-4-loop-iterator inputs: parameters: - name: parent-dag-id @@ -589,9 +560,9 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.annotations.pipelines.kubeflow.org/components-root}}' + value: '{{workflow.parameters.components-root}}' - name: runtime-config - value: '{"parameters":{"text":{"stringValue":"hi there"}}}' + value: '{"parameters":{"text":{"stringValue":"hello world"}}}' - name: driver-type value: ROOT_DAG name: root-driver diff --git a/backend/src/v2/compiler/testdata/multiple_parallel_loops.json b/backend/src/v2/compiler/testdata/multiple_parallel_loops.json index 0e59a0af2d39..1208aa20445d 100644 --- a/backend/src/v2/compiler/testdata/multiple_parallel_loops.json +++ b/backend/src/v2/compiler/testdata/multiple_parallel_loops.json @@ -248,7 +248,7 @@ } }, "taskInfo": { - "name": "chicken" + "name": "foo" } }, "for-loop-4": { @@ -265,7 +265,7 @@ } }, "taskInfo": { - "name": "muffin" + "name": "bar" } } } @@ -277,8 +277,8 @@ "runtimeConfig": { "parameters": { "text": { - "stringValue": "hi there" + "stringValue": "hello world" } } } -} \ No newline at end of file +}