forked from kubeflow/pipelines
-
Notifications
You must be signed in to change notification settings - Fork 11
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Added new sample IRs to test ExitHandler Non-Parallel Loops
Signed-off-by: Achyut Madhusudan <amadhusu@redhat.com>
- Loading branch information
Achyut Madhusudan
committed
Feb 19, 2024
1 parent
17c7f1e
commit 8dfd0c5
Showing
6 changed files
with
621 additions
and
2 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,119 @@ | ||
# PIPELINE DEFINITION | ||
# Name: execution-order-pipeline | ||
# Description: A pipeline to demonstrate execution order management. | ||
# Inputs: | ||
# text1: str [Default: 'message 1'] | ||
# text2: str [Default: 'message 2'] | ||
components: | ||
comp-echo1-op: | ||
executorLabel: exec-echo1-op | ||
inputDefinitions: | ||
parameters: | ||
text1: | ||
parameterType: STRING | ||
comp-echo2-op: | ||
executorLabel: exec-echo2-op | ||
inputDefinitions: | ||
parameters: | ||
text2: | ||
parameterType: STRING | ||
deploymentSpec: | ||
executors: | ||
exec-echo1-op: | ||
container: | ||
args: | ||
- --executor_input | ||
- '{{$}}' | ||
- --function_to_execute | ||
- echo1_op | ||
command: | ||
- sh | ||
- -c | ||
- "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ | ||
\ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ | ||
\ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.6.0'\ | ||
\ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ | ||
$0\" \"$@\"\n" | ||
- sh | ||
- -ec | ||
- 'program_path=$(mktemp -d) | ||
printf "%s" "$0" > "$program_path/ephemeral_component.py" | ||
_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" | ||
' | ||
- "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ | ||
\ *\n\ndef echo1_op(text1: str):\n print(text1)\n\n" | ||
image: quay.io/opendatahub/ds-pipelines-ci-executor-image:v1.0 | ||
exec-echo2-op: | ||
container: | ||
args: | ||
- --executor_input | ||
- '{{$}}' | ||
- --function_to_execute | ||
- echo2_op | ||
command: | ||
- sh | ||
- -c | ||
- "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ | ||
\ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ | ||
\ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.6.0'\ | ||
\ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ | ||
$0\" \"$@\"\n" | ||
- sh | ||
- -ec | ||
- 'program_path=$(mktemp -d) | ||
printf "%s" "$0" > "$program_path/ephemeral_component.py" | ||
_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" | ||
' | ||
- "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ | ||
\ *\n\ndef echo2_op(text2: str):\n print(text2)\n\n" | ||
image: quay.io/opendatahub/ds-pipelines-ci-executor-image:v1.0 | ||
pipelineInfo: | ||
description: A pipeline to demonstrate execution order management. | ||
name: execution-order-pipeline | ||
root: | ||
dag: | ||
tasks: | ||
echo1-op: | ||
cachingOptions: | ||
enableCache: true | ||
componentRef: | ||
name: comp-echo1-op | ||
inputs: | ||
parameters: | ||
text1: | ||
componentInputParameter: text1 | ||
taskInfo: | ||
name: echo1-op | ||
echo2-op: | ||
cachingOptions: | ||
enableCache: true | ||
componentRef: | ||
name: comp-echo2-op | ||
dependentTasks: | ||
- echo1-op | ||
inputs: | ||
parameters: | ||
text2: | ||
componentInputParameter: text2 | ||
taskInfo: | ||
name: echo2-op | ||
inputDefinitions: | ||
parameters: | ||
text1: | ||
defaultValue: message 1 | ||
isOptional: true | ||
parameterType: STRING | ||
text2: | ||
defaultValue: message 2 | ||
isOptional: true | ||
parameterType: STRING | ||
schemaVersion: 2.1.0 | ||
sdkVersion: kfp-2.6.0 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,180 @@ | ||
# PIPELINE DEFINITION | ||
# Name: pipeline-with-exit-handler | ||
# Inputs: | ||
# message: str [Default: 'Hello World!'] | ||
components: | ||
comp-exit-handler-1: | ||
dag: | ||
tasks: | ||
fail-op: | ||
cachingOptions: | ||
enableCache: true | ||
componentRef: | ||
name: comp-fail-op | ||
inputs: | ||
parameters: | ||
message: | ||
runtimeValue: | ||
constant: Task failed. | ||
taskInfo: | ||
name: fail-op | ||
print-op-2: | ||
cachingOptions: | ||
enableCache: true | ||
componentRef: | ||
name: comp-print-op-2 | ||
inputs: | ||
parameters: | ||
message: | ||
componentInputParameter: pipelinechannel--message | ||
taskInfo: | ||
name: print-op-2 | ||
inputDefinitions: | ||
parameters: | ||
pipelinechannel--message: | ||
parameterType: STRING | ||
comp-fail-op: | ||
executorLabel: exec-fail-op | ||
inputDefinitions: | ||
parameters: | ||
message: | ||
parameterType: STRING | ||
comp-print-op: | ||
executorLabel: exec-print-op | ||
inputDefinitions: | ||
parameters: | ||
message: | ||
parameterType: STRING | ||
comp-print-op-2: | ||
executorLabel: exec-print-op-2 | ||
inputDefinitions: | ||
parameters: | ||
message: | ||
parameterType: STRING | ||
deploymentSpec: | ||
executors: | ||
exec-fail-op: | ||
container: | ||
args: | ||
- --executor_input | ||
- '{{$}}' | ||
- --function_to_execute | ||
- fail_op | ||
command: | ||
- sh | ||
- -c | ||
- "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ | ||
\ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ | ||
\ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.6.0'\ | ||
\ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ | ||
$0\" \"$@\"\n" | ||
- sh | ||
- -ec | ||
- 'program_path=$(mktemp -d) | ||
printf "%s" "$0" > "$program_path/ephemeral_component.py" | ||
_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" | ||
' | ||
- "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ | ||
\ *\n\ndef fail_op(message: str):\n \"\"\"Fails.\"\"\"\n import sys\n\ | ||
\ print(message)\n sys.exit(1)\n\n" | ||
image: quay.io/opendatahub/ds-pipelines-ci-executor-image:v1.0 | ||
exec-print-op: | ||
container: | ||
args: | ||
- --executor_input | ||
- '{{$}}' | ||
- --function_to_execute | ||
- print_op | ||
command: | ||
- sh | ||
- -c | ||
- "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ | ||
\ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ | ||
\ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.6.0'\ | ||
\ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ | ||
$0\" \"$@\"\n" | ||
- sh | ||
- -ec | ||
- 'program_path=$(mktemp -d) | ||
printf "%s" "$0" > "$program_path/ephemeral_component.py" | ||
_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" | ||
' | ||
- "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ | ||
\ *\n\ndef print_op(message: str):\n \"\"\"Prints a message.\"\"\"\n\ | ||
\ print(message)\n\n" | ||
image: quay.io/opendatahub/ds-pipelines-ci-executor-image:v1.0 | ||
exec-print-op-2: | ||
container: | ||
args: | ||
- --executor_input | ||
- '{{$}}' | ||
- --function_to_execute | ||
- print_op | ||
command: | ||
- sh | ||
- -c | ||
- "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ | ||
\ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ | ||
\ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.6.0'\ | ||
\ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ | ||
$0\" \"$@\"\n" | ||
- sh | ||
- -ec | ||
- 'program_path=$(mktemp -d) | ||
printf "%s" "$0" > "$program_path/ephemeral_component.py" | ||
_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" | ||
' | ||
- "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ | ||
\ *\n\ndef print_op(message: str):\n \"\"\"Prints a message.\"\"\"\n\ | ||
\ print(message)\n\n" | ||
image: quay.io/opendatahub/ds-pipelines-ci-executor-image:v1.0 | ||
pipelineInfo: | ||
name: pipeline-with-exit-handler | ||
root: | ||
dag: | ||
tasks: | ||
exit-handler-1: | ||
componentRef: | ||
name: comp-exit-handler-1 | ||
inputs: | ||
parameters: | ||
pipelinechannel--message: | ||
componentInputParameter: message | ||
taskInfo: | ||
name: exit-handler-1 | ||
print-op: | ||
cachingOptions: | ||
enableCache: true | ||
componentRef: | ||
name: comp-print-op | ||
dependentTasks: | ||
- exit-handler-1 | ||
inputs: | ||
parameters: | ||
message: | ||
runtimeValue: | ||
constant: Exit handler has worked! | ||
taskInfo: | ||
name: print-op | ||
triggerPolicy: | ||
strategy: ALL_UPSTREAM_TASKS_COMPLETED | ||
inputDefinitions: | ||
parameters: | ||
message: | ||
defaultValue: Hello World! | ||
isOptional: true | ||
parameterType: STRING | ||
schemaVersion: 2.1.0 | ||
sdkVersion: kfp-2.6.0 |
Oops, something went wrong.