From 17c7f1e13fe9020e596e656c124aa5d30b77b296 Mon Sep 17 00:00:00 2001 From: Achyut Madhusudan Date: Fri, 16 Feb 2024 22:24:38 +0530 Subject: [PATCH 1/2] Sample IR files generated as a result of testing. Signed-off-by: Achyut Madhusudan --- .../v2/component_with_optional_inputs.yaml | 99 ++++++++ samples/v2/pipeline_container_no_input.yaml | 27 ++ samples/v2/pipeline_secret_env.yaml | 64 +++++ samples/v2/pipeline_secret_volume.yaml | 66 +++++ samples/v2/pipeline_with_env.yaml | 75 ++++++ samples/v2/pipeline_with_volume.py | 7 +- samples/v2/pipeline_with_volume.yaml | 234 ++++++++++++++++++ .../v2/two_step_pipeline_containerized.yaml | 76 ++++++ 8 files changed, 644 insertions(+), 4 deletions(-) create mode 100644 samples/v2/component_with_optional_inputs.yaml create mode 100644 samples/v2/pipeline_container_no_input.yaml create mode 100644 samples/v2/pipeline_secret_env.yaml create mode 100644 samples/v2/pipeline_secret_volume.yaml create mode 100644 samples/v2/pipeline_with_env.yaml create mode 100644 samples/v2/pipeline_with_volume.yaml create mode 100644 samples/v2/two_step_pipeline_containerized.yaml diff --git a/samples/v2/component_with_optional_inputs.yaml b/samples/v2/component_with_optional_inputs.yaml new file mode 100644 index 00000000000..b72552fde2f --- /dev/null +++ b/samples/v2/component_with_optional_inputs.yaml @@ -0,0 +1,99 @@ +# PIPELINE DEFINITION +# Name: v2-component-optional-input +components: + comp-component-op: + executorLabel: exec-component-op + inputDefinitions: + parameters: + input_bool1: + defaultValue: true + isOptional: true + parameterType: BOOLEAN + input_bool2: + isOptional: true + parameterType: BOOLEAN + input_dict: + defaultValue: + a: 1.0 + isOptional: true + parameterType: STRUCT + input_int: + defaultValue: 100.0 + isOptional: true + parameterType: NUMBER_INTEGER + input_list: + defaultValue: + - '123' + isOptional: true + parameterType: LIST + input_str1: + defaultValue: string default value + isOptional: true + parameterType: STRING + input_str2: + isOptional: true + parameterType: STRING + input_str3: + isOptional: true + parameterType: STRING +deploymentSpec: + executors: + exec-component-op: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - component_op + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.1'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef component_op(\n input_str1: Optional[str] = 'string default\ + \ value',\n input_str2: Optional[str] = None,\n input_str3: Optional[str]\ + \ = None,\n input_bool1: Optional[bool] = True,\n input_bool2: Optional[bool]\ + \ = None,\n input_dict: Optional[Dict[str, int]] = {\"a\": 1},\n input_list:\ + \ Optional[List[str]] = [\"123\"],\n input_int: Optional[int] = 100,\n\ + ):\n print(f'input_str1: {input_str1}, type: {type(input_str1)}')\n \ + \ print(f'input_str2: {input_str2}, type: {type(input_str2)}')\n print(f'input_str3:\ + \ {input_str3}, type: {type(input_str3)}')\n print(f'input_bool1: {input_bool1},\ + \ type: {type(input_bool1)}')\n print(f'input_bool2: {input_bool2}, type:\ + \ {type(input_bool2)}')\n print(f'input_bool: {input_dict}, type: {type(input_dict)}')\n\ + \ print(f'input_bool: {input_list}, type: {type(input_list)}')\n print(f'input_bool:\ + \ {input_int}, type: {type(input_int)}')\n\n" + image: quay.io/opendatahub/ds-pipelines-ci-executor-image:v1.0 +pipelineInfo: + name: v2-component-optional-input +root: + dag: + tasks: + component-op: + cachingOptions: + enableCache: true + componentRef: + name: comp-component-op + inputs: + parameters: + input_str1: + runtimeValue: + constant: Hello + input_str2: + runtimeValue: + constant: World + taskInfo: + name: component-op +schemaVersion: 2.1.0 +sdkVersion: kfp-2.0.1 diff --git a/samples/v2/pipeline_container_no_input.yaml b/samples/v2/pipeline_container_no_input.yaml new file mode 100644 index 00000000000..84599a29c8a --- /dev/null +++ b/samples/v2/pipeline_container_no_input.yaml @@ -0,0 +1,27 @@ +# PIPELINE DEFINITION +# Name: v2-container-component-no-input +components: + comp-container-no-input: + executorLabel: exec-container-no-input +deploymentSpec: + executors: + exec-container-no-input: + container: + command: + - echo + - hello world + image: quay.io/opendatahub/ds-pipelines-ci-executor-image:v1.0 +pipelineInfo: + name: v2-container-component-no-input +root: + dag: + tasks: + container-no-input: + cachingOptions: + enableCache: true + componentRef: + name: comp-container-no-input + taskInfo: + name: container-no-input +schemaVersion: 2.1.0 +sdkVersion: kfp-2.6.0 diff --git a/samples/v2/pipeline_secret_env.yaml b/samples/v2/pipeline_secret_env.yaml new file mode 100644 index 00000000000..f002c63767a --- /dev/null +++ b/samples/v2/pipeline_secret_env.yaml @@ -0,0 +1,64 @@ +# PIPELINE DEFINITION +# Name: pipeline-secret-env +components: + comp-comp: + executorLabel: exec-comp +deploymentSpec: + executors: + exec-comp: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - comp + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.6.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef comp():\n import os\n import sys\n if os.environ['SECRET_VAR']\ + \ == \"service_account\":\n print(\"Success\")\n return 0\n\ + \ else:\n print(os.environ['SECRET_VAR'] + \" is not service_account\"\ + )\n sys.exit(\"Failure: cannot access secret as env variable\")\n\ + \n" + image: quay.io/opendatahub/ds-pipelines-ci-executor-image:v1.0 +pipelineInfo: + name: pipeline-secret-env +root: + dag: + tasks: + comp: + cachingOptions: + enableCache: true + componentRef: + name: comp-comp + taskInfo: + name: comp +schemaVersion: 2.1.0 +sdkVersion: kfp-2.6.0 +--- +platforms: + kubernetes: + deploymentSpec: + executors: + exec-comp: + secretAsEnv: + - keyToEnv: + - envVar: SECRET_VAR + secretKey: type + secretName: user-gcp-sa diff --git a/samples/v2/pipeline_secret_volume.yaml b/samples/v2/pipeline_secret_volume.yaml new file mode 100644 index 00000000000..043b7c50d90 --- /dev/null +++ b/samples/v2/pipeline_secret_volume.yaml @@ -0,0 +1,66 @@ +# PIPELINE DEFINITION +# Name: pipeline-secret-volume +components: + comp-comp: + executorLabel: exec-comp +deploymentSpec: + executors: + exec-comp: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - comp + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.6.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef comp():\n import os\n\n secret_key = \"type\"\n secret_path\ + \ = os.path.join('/mnt/my_vol', secret_key)\n\n # Check if the secret\ + \ exists\n if not os.path.exists(secret_path):\n raise Exception('Secret\ + \ not found')\n\n # Open the secret\n with open(secret_path, 'rb')\ + \ as secret_file:\n secret_data = secret_file.read()\n\n # Decode\ + \ the secret\n secret_data = secret_data.decode('utf-8')\n\n # Print\ + \ the secret\n print(secret_data)\n\n if secret_data == \"service_account\"\ + :\n print(\"Success\")\n return 0\n else:\n sys.exit(\"\ + Failure: cannot access secret as volume variable\")\n\n" + image: quay.io/opendatahub/ds-pipelines-ci-executor-image:v1.0 +pipelineInfo: + name: pipeline-secret-volume +root: + dag: + tasks: + comp: + cachingOptions: + enableCache: true + componentRef: + name: comp-comp + taskInfo: + name: comp +schemaVersion: 2.1.0 +sdkVersion: kfp-2.6.0 +--- +platforms: + kubernetes: + deploymentSpec: + executors: + exec-comp: + secretAsVolume: + - mountPath: /mnt/my_vol + secretName: user-gcp-sa diff --git a/samples/v2/pipeline_with_env.yaml b/samples/v2/pipeline_with_env.yaml new file mode 100644 index 00000000000..d1521850d9d --- /dev/null +++ b/samples/v2/pipeline_with_env.yaml @@ -0,0 +1,75 @@ +# PIPELINE DEFINITION +# Name: pipeline-with-env +components: + comp-check-env: + executorLabel: exec-check-env + comp-print-env-op: + executorLabel: exec-print-env-op +deploymentSpec: + executors: + exec-check-env: + container: + command: + - sh + - -c + - "set -e -x\nif [ \"$ENV2\" == \"val2\" ]\nthen\n echo \"$ENV2\" \nelse\ + \ \n echo \"ENV2 does not equal val2\"\n exit 1\nfi\necho \"$ENV3\"\n" + env: + - name: ENV2 + value: val2 + - name: ENV3 + value: val3 + image: alpine + exec-print-env-op: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - print_env_op + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.6.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef print_env_op():\n import os\n print('ENV1', os.environ.get('ENV1'))\n\ + \ print('ENV2', os.environ.get('ENV2'))\n\n" + env: + - name: ENV1 + value: val1 + image: quay.io/opendatahub/ds-pipelines-ci-executor-image:v1.0 +pipelineInfo: + name: pipeline-with-env +root: + dag: + tasks: + check-env: + cachingOptions: + enableCache: true + componentRef: + name: comp-check-env + taskInfo: + name: check-env + print-env-op: + cachingOptions: + enableCache: true + componentRef: + name: comp-print-env-op + taskInfo: + name: print-env-op +schemaVersion: 2.1.0 +sdkVersion: kfp-2.6.0 diff --git a/samples/v2/pipeline_with_volume.py b/samples/v2/pipeline_with_volume.py index c1a8b1035e5..29dca680da3 100644 --- a/samples/v2/pipeline_with_volume.py +++ b/samples/v2/pipeline_with_volume.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """Pipeline with volume creation, mount and deletion in v2 engine pipeline.""" -from kfp import dsl +from kfp import compiler, dsl from kfp import kubernetes @@ -40,7 +40,7 @@ def pipeline_with_volume(): pvc_name_suffix='-my-pvc', access_modes=['ReadWriteOnce'], size='5Mi', - storage_class_name='standard', + storage_class_name='standard-csi', ) task1 = producer() @@ -62,7 +62,6 @@ def pipeline_with_volume(): if __name__ == '__main__': # execute only if run as a script - from kfp import compiler compiler.Compiler().compile( pipeline_func=pipeline_with_volume, - package_path='pipeline_with_volume.json') \ No newline at end of file + package_path='pipeline_with_volume.yaml') \ No newline at end of file diff --git a/samples/v2/pipeline_with_volume.yaml b/samples/v2/pipeline_with_volume.yaml new file mode 100644 index 00000000000..7960acc5442 --- /dev/null +++ b/samples/v2/pipeline_with_volume.yaml @@ -0,0 +1,234 @@ +# PIPELINE DEFINITION +# Name: pipeline-with-volume +components: + comp-consumer: + executorLabel: exec-consumer + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-createpvc: + executorLabel: exec-createpvc + inputDefinitions: + parameters: + access_modes: + description: 'AccessModes to request for the provisioned PVC. May + + be one or more of ``''ReadWriteOnce''``, ``''ReadOnlyMany''``, ``''ReadWriteMany''``, + or + + ``''ReadWriteOncePod''``. Corresponds to `PersistentVolumeClaim.spec.accessModes + `_.' + parameterType: LIST + annotations: + description: Annotations for the PVC's metadata. Corresponds to `PersistentVolumeClaim.metadata.annotations + `_. + isOptional: true + parameterType: STRUCT + pvc_name: + description: 'Name of the PVC. Corresponds to `PersistentVolumeClaim.metadata.name + `_. + Only one of ``pvc_name`` and ``pvc_name_suffix`` can + + be provided.' + isOptional: true + parameterType: STRING + pvc_name_suffix: + description: 'Prefix to use for a dynamically generated name, which + + will take the form ``-``. Only one + + of ``pvc_name`` and ``pvc_name_suffix`` can be provided.' + isOptional: true + parameterType: STRING + size: + description: The size of storage requested by the PVC that will be provisioned. + For example, ``'5Gi'``. Corresponds to `PersistentVolumeClaim.spec.resources.requests.storage + `_. + parameterType: STRING + storage_class_name: + defaultValue: '' + description: 'Name of StorageClass from which to provision the PV + + to back the PVC. ``None`` indicates to use the cluster''s default + + storage_class_name. Set to ``''''`` for a statically specified PVC.' + isOptional: true + parameterType: STRING + volume_name: + description: 'Pre-existing PersistentVolume that should back the + + provisioned PersistentVolumeClaim. Used for statically + + specified PV only. Corresponds to `PersistentVolumeClaim.spec.volumeName + `_.' + isOptional: true + parameterType: STRING + outputDefinitions: + parameters: + name: + parameterType: STRING + comp-deletepvc: + executorLabel: exec-deletepvc + inputDefinitions: + parameters: + pvc_name: + description: Name of the PVC to delete. Supports passing a runtime-generated + name, such as a name provided by ``kubernetes.CreatePvcOp().outputs['name']``. + parameterType: STRING + comp-producer: + executorLabel: exec-producer + outputDefinitions: + parameters: + Output: + parameterType: STRING +deploymentSpec: + executors: + exec-consumer: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - consumer + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.6.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef consumer() -> str:\n with open('/data/file.txt', 'r') as file:\n\ + \ content = file.read()\n print(content)\n return content\n\ + \n" + image: quay.io/opendatahub/ds-pipelines-ci-executor-image:v1.0 + exec-createpvc: + container: + image: argostub/createpvc + exec-deletepvc: + container: + image: argostub/deletepvc + exec-producer: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - producer + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.6.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef producer() -> str:\n with open('/data/file.txt', 'w') as file:\n\ + \ file.write('Hello world')\n with open('/data/file.txt', 'r')\ + \ as file:\n content = file.read()\n print(content)\n return\ + \ content\n\n" + image: quay.io/opendatahub/ds-pipelines-ci-executor-image:v1.0 +pipelineInfo: + name: pipeline-with-volume +root: + dag: + tasks: + consumer: + cachingOptions: + enableCache: true + componentRef: + name: comp-consumer + dependentTasks: + - createpvc + - producer + taskInfo: + name: consumer + createpvc: + cachingOptions: + enableCache: true + componentRef: + name: comp-createpvc + inputs: + parameters: + access_modes: + runtimeValue: + constant: + - ReadWriteOnce + pvc_name_suffix: + runtimeValue: + constant: -my-pvc + size: + runtimeValue: + constant: 5Mi + storage_class_name: + runtimeValue: + constant: standard-csi + taskInfo: + name: createpvc + deletepvc: + cachingOptions: + enableCache: true + componentRef: + name: comp-deletepvc + dependentTasks: + - consumer + - createpvc + inputs: + parameters: + pvc_name: + taskOutputParameter: + outputParameterKey: name + producerTask: createpvc + taskInfo: + name: deletepvc + producer: + cachingOptions: + enableCache: true + componentRef: + name: comp-producer + dependentTasks: + - createpvc + taskInfo: + name: producer +schemaVersion: 2.1.0 +sdkVersion: kfp-2.6.0 +--- +platforms: + kubernetes: + deploymentSpec: + executors: + exec-consumer: + pvcMount: + - mountPath: /data + taskOutputParameter: + outputParameterKey: name + producerTask: createpvc + exec-producer: + pvcMount: + - mountPath: /data + taskOutputParameter: + outputParameterKey: name + producerTask: createpvc diff --git a/samples/v2/two_step_pipeline_containerized.yaml b/samples/v2/two_step_pipeline_containerized.yaml new file mode 100644 index 00000000000..91f62ac3c3c --- /dev/null +++ b/samples/v2/two_step_pipeline_containerized.yaml @@ -0,0 +1,76 @@ +# PIPELINE DEFINITION +# Name: two-step-pipeline-containerized +components: + comp-component1: + executorLabel: exec-component1 + inputDefinitions: + parameters: + text: + parameterType: STRING + outputDefinitions: + artifacts: + output_gcs: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + comp-component2: + executorLabel: exec-component2 + inputDefinitions: + artifacts: + input_gcs: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 +deploymentSpec: + executors: + exec-component1: + container: + args: + - '{{$.inputs.parameters[''text'']}}' + - '{{$.outputs.artifacts[''output_gcs''].path}}' + command: + - sh + - -c + - mkdir --parents $(dirname "$1") && echo "$0" > "$1" + image: quay.io/opendatahub/ds-pipelines-ci-executor-image:v1.0 + exec-component2: + container: + args: + - '{{$.inputs.artifacts[''input_gcs''].path}}' + command: + - cat + image: quay.io/opendatahub/ds-pipelines-ci-executor-image:v1.0 +pipelineInfo: + name: two-step-pipeline-containerized +root: + dag: + tasks: + component1: + cachingOptions: + enableCache: true + componentRef: + name: comp-component1 + inputs: + parameters: + text: + runtimeValue: + constant: hi + taskInfo: + name: component1 + component2: + cachingOptions: + enableCache: true + componentRef: + name: comp-component2 + dependentTasks: + - component1 + inputs: + artifacts: + input_gcs: + taskOutputArtifact: + outputArtifactKey: output_gcs + producerTask: component1 + taskInfo: + name: component2 +schemaVersion: 2.1.0 +sdkVersion: kfp-2.6.0 From 8dfd0c5fc8db1f4b18a8e22d13dacb4c02c50dbf Mon Sep 17 00:00:00 2001 From: Achyut Madhusudan Date: Mon, 19 Feb 2024 13:55:51 +0530 Subject: [PATCH 2/2] Added new sample IRs to test ExitHandler Non-Parallel Loops Signed-off-by: Achyut Madhusudan --- .../execution_order/execution_order.py.yaml | 119 +++++++++++ .../core/exit_handler/exit_handler.py.yaml | 180 ++++++++++++++++ samples/core/loop_output/loop_output.py.yaml | 124 +++++++++++ samples/core/loop_static/loop_static.py.yaml | 196 ++++++++++++++++++ samples/v2/pipeline_with_volume.py | 2 +- samples/v2/pipeline_with_volume.yaml | 2 +- 6 files changed, 621 insertions(+), 2 deletions(-) create mode 100644 samples/core/execution_order/execution_order.py.yaml create mode 100644 samples/core/exit_handler/exit_handler.py.yaml create mode 100644 samples/core/loop_output/loop_output.py.yaml create mode 100644 samples/core/loop_static/loop_static.py.yaml diff --git a/samples/core/execution_order/execution_order.py.yaml b/samples/core/execution_order/execution_order.py.yaml new file mode 100644 index 00000000000..872bee5882b --- /dev/null +++ b/samples/core/execution_order/execution_order.py.yaml @@ -0,0 +1,119 @@ +# PIPELINE DEFINITION +# Name: execution-order-pipeline +# Description: A pipeline to demonstrate execution order management. +# Inputs: +# text1: str [Default: 'message 1'] +# text2: str [Default: 'message 2'] +components: + comp-echo1-op: + executorLabel: exec-echo1-op + inputDefinitions: + parameters: + text1: + parameterType: STRING + comp-echo2-op: + executorLabel: exec-echo2-op + inputDefinitions: + parameters: + text2: + parameterType: STRING +deploymentSpec: + executors: + exec-echo1-op: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - echo1_op + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.6.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef echo1_op(text1: str):\n print(text1)\n\n" + image: quay.io/opendatahub/ds-pipelines-ci-executor-image:v1.0 + exec-echo2-op: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - echo2_op + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.6.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef echo2_op(text2: str):\n print(text2)\n\n" + image: quay.io/opendatahub/ds-pipelines-ci-executor-image:v1.0 +pipelineInfo: + description: A pipeline to demonstrate execution order management. + name: execution-order-pipeline +root: + dag: + tasks: + echo1-op: + cachingOptions: + enableCache: true + componentRef: + name: comp-echo1-op + inputs: + parameters: + text1: + componentInputParameter: text1 + taskInfo: + name: echo1-op + echo2-op: + cachingOptions: + enableCache: true + componentRef: + name: comp-echo2-op + dependentTasks: + - echo1-op + inputs: + parameters: + text2: + componentInputParameter: text2 + taskInfo: + name: echo2-op + inputDefinitions: + parameters: + text1: + defaultValue: message 1 + isOptional: true + parameterType: STRING + text2: + defaultValue: message 2 + isOptional: true + parameterType: STRING +schemaVersion: 2.1.0 +sdkVersion: kfp-2.6.0 diff --git a/samples/core/exit_handler/exit_handler.py.yaml b/samples/core/exit_handler/exit_handler.py.yaml new file mode 100644 index 00000000000..e4a491357df --- /dev/null +++ b/samples/core/exit_handler/exit_handler.py.yaml @@ -0,0 +1,180 @@ +# PIPELINE DEFINITION +# Name: pipeline-with-exit-handler +# Inputs: +# message: str [Default: 'Hello World!'] +components: + comp-exit-handler-1: + dag: + tasks: + fail-op: + cachingOptions: + enableCache: true + componentRef: + name: comp-fail-op + inputs: + parameters: + message: + runtimeValue: + constant: Task failed. + taskInfo: + name: fail-op + print-op-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-print-op-2 + inputs: + parameters: + message: + componentInputParameter: pipelinechannel--message + taskInfo: + name: print-op-2 + inputDefinitions: + parameters: + pipelinechannel--message: + parameterType: STRING + comp-fail-op: + executorLabel: exec-fail-op + inputDefinitions: + parameters: + message: + parameterType: STRING + comp-print-op: + executorLabel: exec-print-op + inputDefinitions: + parameters: + message: + parameterType: STRING + comp-print-op-2: + executorLabel: exec-print-op-2 + inputDefinitions: + parameters: + message: + parameterType: STRING +deploymentSpec: + executors: + exec-fail-op: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - fail_op + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.6.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef fail_op(message: str):\n \"\"\"Fails.\"\"\"\n import sys\n\ + \ print(message)\n sys.exit(1)\n\n" + image: quay.io/opendatahub/ds-pipelines-ci-executor-image:v1.0 + exec-print-op: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - print_op + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.6.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef print_op(message: str):\n \"\"\"Prints a message.\"\"\"\n\ + \ print(message)\n\n" + image: quay.io/opendatahub/ds-pipelines-ci-executor-image:v1.0 + exec-print-op-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - print_op + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.6.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef print_op(message: str):\n \"\"\"Prints a message.\"\"\"\n\ + \ print(message)\n\n" + image: quay.io/opendatahub/ds-pipelines-ci-executor-image:v1.0 +pipelineInfo: + name: pipeline-with-exit-handler +root: + dag: + tasks: + exit-handler-1: + componentRef: + name: comp-exit-handler-1 + inputs: + parameters: + pipelinechannel--message: + componentInputParameter: message + taskInfo: + name: exit-handler-1 + print-op: + cachingOptions: + enableCache: true + componentRef: + name: comp-print-op + dependentTasks: + - exit-handler-1 + inputs: + parameters: + message: + runtimeValue: + constant: Exit handler has worked! + taskInfo: + name: print-op + triggerPolicy: + strategy: ALL_UPSTREAM_TASKS_COMPLETED + inputDefinitions: + parameters: + message: + defaultValue: Hello World! + isOptional: true + parameterType: STRING +schemaVersion: 2.1.0 +sdkVersion: kfp-2.6.0 diff --git a/samples/core/loop_output/loop_output.py.yaml b/samples/core/loop_output/loop_output.py.yaml new file mode 100644 index 00000000000..0454edbf75d --- /dev/null +++ b/samples/core/loop_output/loop_output.py.yaml @@ -0,0 +1,124 @@ +# PIPELINE DEFINITION +# Name: pipeline-with-loop-output-v2 +components: + comp-args-generator-op: + executorLabel: exec-args-generator-op + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-for-loop-1: + dag: + tasks: + print-op: + cachingOptions: + enableCache: true + componentRef: + name: comp-print-op + inputs: + parameters: + s: + componentInputParameter: pipelinechannel--args-generator-op-Output-loop-item + taskInfo: + name: print-op + inputDefinitions: + parameters: + pipelinechannel--args-generator-op-Output: + parameterType: STRING + pipelinechannel--args-generator-op-Output-loop-item: + parameterType: STRING + comp-print-op: + executorLabel: exec-print-op + inputDefinitions: + parameters: + s: + parameterType: STRING +deploymentSpec: + executors: + exec-args-generator-op: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - args_generator_op + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.6.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef args_generator_op() -> str:\n return '[1.1, 1.2, 1.3]'\n\n" + image: quay.io/opendatahub/ds-pipelines-ci-executor-image:v1.0 + exec-print-op: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - print_op + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.6.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef print_op(s: str):\n print(s)\n\n" + image: quay.io/opendatahub/ds-pipelines-ci-executor-image:v1.0 +pipelineInfo: + name: pipeline-with-loop-output-v2 +root: + dag: + tasks: + args-generator-op: + cachingOptions: + enableCache: true + componentRef: + name: comp-args-generator-op + taskInfo: + name: args-generator-op + for-loop-1: + componentRef: + name: comp-for-loop-1 + dependentTasks: + - args-generator-op + inputs: + parameters: + pipelinechannel--args-generator-op-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: args-generator-op + parameterIterator: + itemInput: pipelinechannel--args-generator-op-Output-loop-item + items: + inputParameter: pipelinechannel--args-generator-op-Output + taskInfo: + name: for-loop-1 +schemaVersion: 2.1.0 +sdkVersion: kfp-2.6.0 diff --git a/samples/core/loop_static/loop_static.py.yaml b/samples/core/loop_static/loop_static.py.yaml new file mode 100644 index 00000000000..6486513cb0c --- /dev/null +++ b/samples/core/loop_static/loop_static.py.yaml @@ -0,0 +1,196 @@ +# PIPELINE DEFINITION +# Name: pipeline-with-loop-static +# Inputs: +# greeting: str [Default: 'this is a test for looping through parameters'] +components: + comp-concat-op: + executorLabel: exec-concat-op + inputDefinitions: + parameters: + a: + parameterType: STRING + b: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-for-loop-2: + dag: + tasks: + concat-op: + cachingOptions: + enableCache: true + componentRef: + name: comp-concat-op + inputs: + parameters: + a: + componentInputParameter: pipelinechannel--loop-item-param-1 + parameterExpressionSelector: parseJson(string_value)["a"] + b: + componentInputParameter: pipelinechannel--loop-item-param-1 + parameterExpressionSelector: parseJson(string_value)["b"] + taskInfo: + name: concat-op + print-op-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-print-op-2 + dependentTasks: + - concat-op + inputs: + parameters: + text: + taskOutputParameter: + outputParameterKey: Output + producerTask: concat-op + taskInfo: + name: print-op-2 + inputDefinitions: + parameters: + pipelinechannel--loop-item-param-1: + parameterType: STRING + comp-print-op: + executorLabel: exec-print-op + inputDefinitions: + parameters: + text: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-print-op-2: + executorLabel: exec-print-op-2 + inputDefinitions: + parameters: + text: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING +deploymentSpec: + executors: + exec-concat-op: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - concat_op + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.6.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef concat_op(a: str, b: str) -> str:\n print(a + b)\n return\ + \ a + b\n\n" + image: quay.io/opendatahub/ds-pipelines-ci-executor-image:v1.0 + exec-print-op: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - print_op + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.6.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef print_op(text: str) -> str:\n print(text)\n return text\n\ + \n" + image: quay.io/opendatahub/ds-pipelines-ci-executor-image:v1.0 + exec-print-op-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - print_op + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.6.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef print_op(text: str) -> str:\n print(text)\n return text\n\ + \n" + image: quay.io/opendatahub/ds-pipelines-ci-executor-image:v1.0 +pipelineInfo: + name: pipeline-with-loop-static +root: + dag: + tasks: + for-loop-2: + componentRef: + name: comp-for-loop-2 + parameterIterator: + itemInput: pipelinechannel--loop-item-param-1 + items: + raw: '["{\"a\": \"1\", \"b\": \"2\"}", "{\"a\": \"10\", \"b\": \"20\"}"]' + taskInfo: + name: for-loop-2 + print-op: + cachingOptions: + enableCache: true + componentRef: + name: comp-print-op + inputs: + parameters: + text: + componentInputParameter: greeting + taskInfo: + name: print-op + inputDefinitions: + parameters: + greeting: + defaultValue: this is a test for looping through parameters + isOptional: true + parameterType: STRING +schemaVersion: 2.1.0 +sdkVersion: kfp-2.6.0 diff --git a/samples/v2/pipeline_with_volume.py b/samples/v2/pipeline_with_volume.py index 29dca680da3..b77df83b362 100644 --- a/samples/v2/pipeline_with_volume.py +++ b/samples/v2/pipeline_with_volume.py @@ -64,4 +64,4 @@ def pipeline_with_volume(): # execute only if run as a script compiler.Compiler().compile( pipeline_func=pipeline_with_volume, - package_path='pipeline_with_volume.yaml') \ No newline at end of file + package_path='pipeline_with_volume.yaml') diff --git a/samples/v2/pipeline_with_volume.yaml b/samples/v2/pipeline_with_volume.yaml index 7960acc5442..bcb2b90bbda 100644 --- a/samples/v2/pipeline_with_volume.yaml +++ b/samples/v2/pipeline_with_volume.yaml @@ -231,4 +231,4 @@ platforms: - mountPath: /data taskOutputParameter: outputParameterKey: name - producerTask: createpvc + producerTask: createpvc