Skip to content

Commit

Permalink
Added new sample IRs to test ExitHandler Non-Parallel Loops
Browse files Browse the repository at this point in the history
Signed-off-by: Achyut Madhusudan <amadhusu@redhat.com>
  • Loading branch information
Achyut Madhusudan committed Feb 19, 2024
1 parent 17c7f1e commit 8dfd0c5
Show file tree
Hide file tree
Showing 6 changed files with 621 additions and 2 deletions.
119 changes: 119 additions & 0 deletions samples/core/execution_order/execution_order.py.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
# PIPELINE DEFINITION
# Name: execution-order-pipeline
# Description: A pipeline to demonstrate execution order management.
# Inputs:
# text1: str [Default: 'message 1']
# text2: str [Default: 'message 2']
components:
comp-echo1-op:
executorLabel: exec-echo1-op
inputDefinitions:
parameters:
text1:
parameterType: STRING
comp-echo2-op:
executorLabel: exec-echo2-op
inputDefinitions:
parameters:
text2:
parameterType: STRING
deploymentSpec:
executors:
exec-echo1-op:
container:
args:
- --executor_input
- '{{$}}'
- --function_to_execute
- echo1_op
command:
- sh
- -c
- "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\
\ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
\ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.6.0'\
\ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\
$0\" \"$@\"\n"
- sh
- -ec
- 'program_path=$(mktemp -d)
printf "%s" "$0" > "$program_path/ephemeral_component.py"
_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@"
'
- "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
\ *\n\ndef echo1_op(text1: str):\n print(text1)\n\n"
image: quay.io/opendatahub/ds-pipelines-ci-executor-image:v1.0
exec-echo2-op:
container:
args:
- --executor_input
- '{{$}}'
- --function_to_execute
- echo2_op
command:
- sh
- -c
- "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\
\ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
\ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.6.0'\
\ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\
$0\" \"$@\"\n"
- sh
- -ec
- 'program_path=$(mktemp -d)
printf "%s" "$0" > "$program_path/ephemeral_component.py"
_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@"
'
- "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
\ *\n\ndef echo2_op(text2: str):\n print(text2)\n\n"
image: quay.io/opendatahub/ds-pipelines-ci-executor-image:v1.0
pipelineInfo:
description: A pipeline to demonstrate execution order management.
name: execution-order-pipeline
root:
dag:
tasks:
echo1-op:
cachingOptions:
enableCache: true
componentRef:
name: comp-echo1-op
inputs:
parameters:
text1:
componentInputParameter: text1
taskInfo:
name: echo1-op
echo2-op:
cachingOptions:
enableCache: true
componentRef:
name: comp-echo2-op
dependentTasks:
- echo1-op
inputs:
parameters:
text2:
componentInputParameter: text2
taskInfo:
name: echo2-op
inputDefinitions:
parameters:
text1:
defaultValue: message 1
isOptional: true
parameterType: STRING
text2:
defaultValue: message 2
isOptional: true
parameterType: STRING
schemaVersion: 2.1.0
sdkVersion: kfp-2.6.0
180 changes: 180 additions & 0 deletions samples/core/exit_handler/exit_handler.py.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,180 @@
# PIPELINE DEFINITION
# Name: pipeline-with-exit-handler
# Inputs:
# message: str [Default: 'Hello World!']
components:
comp-exit-handler-1:
dag:
tasks:
fail-op:
cachingOptions:
enableCache: true
componentRef:
name: comp-fail-op
inputs:
parameters:
message:
runtimeValue:
constant: Task failed.
taskInfo:
name: fail-op
print-op-2:
cachingOptions:
enableCache: true
componentRef:
name: comp-print-op-2
inputs:
parameters:
message:
componentInputParameter: pipelinechannel--message
taskInfo:
name: print-op-2
inputDefinitions:
parameters:
pipelinechannel--message:
parameterType: STRING
comp-fail-op:
executorLabel: exec-fail-op
inputDefinitions:
parameters:
message:
parameterType: STRING
comp-print-op:
executorLabel: exec-print-op
inputDefinitions:
parameters:
message:
parameterType: STRING
comp-print-op-2:
executorLabel: exec-print-op-2
inputDefinitions:
parameters:
message:
parameterType: STRING
deploymentSpec:
executors:
exec-fail-op:
container:
args:
- --executor_input
- '{{$}}'
- --function_to_execute
- fail_op
command:
- sh
- -c
- "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\
\ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
\ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.6.0'\
\ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\
$0\" \"$@\"\n"
- sh
- -ec
- 'program_path=$(mktemp -d)
printf "%s" "$0" > "$program_path/ephemeral_component.py"
_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@"
'
- "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
\ *\n\ndef fail_op(message: str):\n \"\"\"Fails.\"\"\"\n import sys\n\
\ print(message)\n sys.exit(1)\n\n"
image: quay.io/opendatahub/ds-pipelines-ci-executor-image:v1.0
exec-print-op:
container:
args:
- --executor_input
- '{{$}}'
- --function_to_execute
- print_op
command:
- sh
- -c
- "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\
\ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
\ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.6.0'\
\ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\
$0\" \"$@\"\n"
- sh
- -ec
- 'program_path=$(mktemp -d)
printf "%s" "$0" > "$program_path/ephemeral_component.py"
_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@"
'
- "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
\ *\n\ndef print_op(message: str):\n \"\"\"Prints a message.\"\"\"\n\
\ print(message)\n\n"
image: quay.io/opendatahub/ds-pipelines-ci-executor-image:v1.0
exec-print-op-2:
container:
args:
- --executor_input
- '{{$}}'
- --function_to_execute
- print_op
command:
- sh
- -c
- "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\
\ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
\ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.6.0'\
\ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\
$0\" \"$@\"\n"
- sh
- -ec
- 'program_path=$(mktemp -d)
printf "%s" "$0" > "$program_path/ephemeral_component.py"
_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@"
'
- "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
\ *\n\ndef print_op(message: str):\n \"\"\"Prints a message.\"\"\"\n\
\ print(message)\n\n"
image: quay.io/opendatahub/ds-pipelines-ci-executor-image:v1.0
pipelineInfo:
name: pipeline-with-exit-handler
root:
dag:
tasks:
exit-handler-1:
componentRef:
name: comp-exit-handler-1
inputs:
parameters:
pipelinechannel--message:
componentInputParameter: message
taskInfo:
name: exit-handler-1
print-op:
cachingOptions:
enableCache: true
componentRef:
name: comp-print-op
dependentTasks:
- exit-handler-1
inputs:
parameters:
message:
runtimeValue:
constant: Exit handler has worked!
taskInfo:
name: print-op
triggerPolicy:
strategy: ALL_UPSTREAM_TASKS_COMPLETED
inputDefinitions:
parameters:
message:
defaultValue: Hello World!
isOptional: true
parameterType: STRING
schemaVersion: 2.1.0
sdkVersion: kfp-2.6.0
Loading

0 comments on commit 8dfd0c5

Please sign in to comment.