diff --git a/docs/assets/images/favicon.png b/docs/assets/images/favicon.png new file mode 100644 index 000000000..8a0f42d26 Binary files /dev/null and b/docs/assets/images/favicon.png differ diff --git a/docs/css/custom.css b/docs/css/custom.css new file mode 100644 index 000000000..e9c4fed86 --- /dev/null +++ b/docs/css/custom.css @@ -0,0 +1,205 @@ +:root { + /*--main-text-color: #212121;*/ + --md-primary-fg-color: #1976d2; + --brand-blue: #1976d2; + --brand-dark-blue: #242A36; + --caption-color: #4f4f4f; + --brand-lt-blue: #f0f5fb; + --brand-gray: rgb(118, 118, 118); + --brand-lt-gray: rgb(203,204,207); + --brand-red: #e50914; +} + +/* Grid */ +.row { + display: flex; + flex-direction: row; +} +.col-4 { + flex: 0 0 33.3333333333%; + max-width: 33.3333333333%; +} +.col-6 { + flex: 0 0 50%; + max-width: 50%; +} + + + +/* Navbar */ +.md-header { + background-color: white !important; + color: var(--brand-dark-blue); +} +.md-header__title { + visibility: hidden; +} +.md-logo img{ + height: 38px !important; +} +.home { + margin-bottom: -1.2rem !important; +} +.md-search__form { + transition: none !important; +} +.md-search__input:hover { + background-color: #00000042 !important; +} +.md-search__input.focus-visible:hover { + background-color: #fff !important; +} + +/* Fonts */ +body { + color: var(--brand-dark-blue); + font-family: "Roboto", sans-serif !important; + font-weight: 400 !important; +} + +.md-content h1 { + font-family: "Inter", sans-serif !important; + color: var(--brand-dark-blue) !important; + font-size: 32px !important; + font-weight: 700 !important; +} + +.md-content h2 { + font-family: "Inter", sans-serif !important; + color: var(--brand-dark-blue) !important; + font-size: 24px !important; + font-weight: 700 !important; +} + +.md-content h3 { + font-family: "Roboto", sans-serif !important; + color: var(--brand-dark-blue) !important; + font-size: 20px !important; + font-weight: 500 !important; +} + +.md-content h4 { + font-family: "Roboto", sans-serif !important; + color: var(--brand-dark-blue) !important; + font-size: 18px !important; + font-weight: 400 !important; +} + +.btn { + font-family: "Roboto", sans-serif; + font-size: 14px; + border-radius: 0.25rem; +} +.btn-primary { + background: #1976D2; + border: none; + color: white !important; +} + +.hero { + padding-top: 100px; + padding-bottom: 100px; +} + +.hero .heading { + font-size: 56px; + font-weight: 900; + line-height: 68px; +} + +.hero .btn { + font-size: 16px; + padding: 10px 20px; +} + +.hero .illustration { + margin-left: 35px; +} + + +.bullets .heading, .module .heading { + font-family: "Inter", sans-serif; + font-size: 26px; + font-weight: 700; +} +.bullets .row { + margin-bottom: 60px; +} +.bullets .caption { + padding-top: 10px; + padding-right: 30px; +} +.icon { + height: 25px !important; + margin-right: 5px; + vertical-align: -3px; +} + +.caption { + font-weight: 400; + font-size: 17px; + line-height: 24px; + color: var(--caption-color); +} + +.module { + margin-top: 80px; + margin-bottom: 80px; + padding-top: 50px; + padding-bottom: 50px; +} + +.module .caption { + padding-top: 10px; + padding-right: 80px; +} +.module .screenshot { + width: 600px; + height: 337px; + box-shadow:inset 0 1px 0 rgba(255,255,255,.6), 0 22px 70px 4px rgba(0,0,0,0.56), 0 0 0 1px rgba(0, 0, 0, 0.0); + border-radius: 5px; + background-size: cover; +} + +/* Footer */ +.md-copyright__highlight { + background-image: url('/img/netflix-oss.png'); + background-size: contain; + background-repeat: no-repeat; + color: rgba(0,0,0,0); + height: 60px; +} + +/* Comparison block */ +.compare { + background-color: var(--brand-lt-blue); + padding-top: 80px; + padding-bottom: 80px; + margin: 0px -1000px; + text-align: center; +} +.compare .container { + max-width: 61rem; + margin-left: auto; + margin-right: auto; +} + +.compare .heading { + margin-bottom: 30px; + margin-top: 0px; +} +.compare .bubble { + background: #fff; + border-radius: 10px; + padding: 30px; + height: 100%; +} + +.compare .caption { + font-size: 15px; + line-height: 22px; +} + +.compare .row { + margin: 0 0.8rem; +} \ No newline at end of file diff --git a/docs/devguide/architecture/PollTimeoutSeconds.png b/docs/devguide/architecture/PollTimeoutSeconds.png new file mode 100644 index 000000000..35bd32273 Binary files /dev/null and b/docs/devguide/architecture/PollTimeoutSeconds.png differ diff --git a/docs/docs/img/ResponseTimeoutSeconds.png b/docs/devguide/architecture/ResponseTimeoutSeconds.png similarity index 100% rename from docs/docs/img/ResponseTimeoutSeconds.png rename to docs/devguide/architecture/ResponseTimeoutSeconds.png diff --git a/docs/docs/img/TaskFailure.png b/docs/devguide/architecture/TaskFailure.png similarity index 100% rename from docs/docs/img/TaskFailure.png rename to docs/devguide/architecture/TaskFailure.png diff --git a/docs/docs/img/TimeoutSeconds.png b/docs/devguide/architecture/TimeoutSeconds.png similarity index 100% rename from docs/docs/img/TimeoutSeconds.png rename to docs/devguide/architecture/TimeoutSeconds.png diff --git a/docs/docs/img/conductor-architecture.png b/docs/devguide/architecture/conductor-architecture.png similarity index 100% rename from docs/docs/img/conductor-architecture.png rename to docs/devguide/architecture/conductor-architecture.png diff --git a/docs/docs/img/dag_workflow.png b/docs/devguide/architecture/dag_workflow.png similarity index 100% rename from docs/docs/img/dag_workflow.png rename to docs/devguide/architecture/dag_workflow.png diff --git a/docs/docs/img/dag_workflow2.png b/docs/devguide/architecture/dag_workflow2.png similarity index 100% rename from docs/docs/img/dag_workflow2.png rename to docs/devguide/architecture/dag_workflow2.png diff --git a/docs/docs/reference-docs/directed-acyclic-graph.md b/docs/devguide/architecture/directed-acyclic-graph.md similarity index 73% rename from docs/docs/reference-docs/directed-acyclic-graph.md rename to docs/devguide/architecture/directed-acyclic-graph.md index 5c707eccf..f8d08c0b6 100644 --- a/docs/docs/reference-docs/directed-acyclic-graph.md +++ b/docs/devguide/architecture/directed-acyclic-graph.md @@ -10,11 +10,11 @@ A graph is "a collection of vertices (or point) and edges (or lines) that indica By this definition, this is a graph - just not exactly correct in the context of DAGs: -

pirate vs global warming graph

+![pirate vs global warming graph](pirate_graph.gif) But in the context of workflows, we're thinking of a graph more like this: -

a regular graph (source: wikipedia)

+![a regular graph (source: wikipedia)](regular_graph.png) Imagine each vertex as a microservice, and the lines are how the microservices are connected together. However, this graph is not a directed graph - as there is no direction given to each connection. @@ -22,7 +22,7 @@ Imagine each vertex as a microservice, and the lines are how the microservices a A directed graph means that there is a direction to each connection. For example, this graph is directed: -

directed graph

+![directed graph](directed_graph.png) Each arrow has a direction, Point "N" can proceed directly to "B", but "B" cannot proceed to "N" in the opposite direction. @@ -34,13 +34,13 @@ So a Directed Acyclic Graph is a set of vertices where the connections are direc Since a Conductor workflow is a series of vertices that can connect in only a specific direction and cannot loop, a Conductor workflow is thus a directed acyclic graph: -

Conductor Dag

+![Conductor Dag](dag_workflow.png) ### Can a workflow have loops and still be a DAG? Yes. For example, Conductor workflows have Do-While loops: -

Conductor Dag

+![Conductor Dag](dag_workflow2.png) This is still a DAG, because the loop is just shorthand for running the tasks inside the loop over and over again. For example, if the 2nd loop in the above image is run 3 times, the workflow path will be: diff --git a/docs/docs/img/directed_graph.png b/docs/devguide/architecture/directed_graph.png similarity index 100% rename from docs/docs/img/directed_graph.png rename to docs/devguide/architecture/directed_graph.png diff --git a/docs/docs/architecture/overview.md b/docs/devguide/architecture/index.md similarity index 88% rename from docs/docs/architecture/overview.md rename to docs/devguide/architecture/index.md index 81bd00e38..986003dfb 100644 --- a/docs/docs/architecture/overview.md +++ b/docs/devguide/architecture/index.md @@ -1,15 +1,15 @@ -# Overview +# Architecture Overview -![Architecture diagram](/img/conductor-architecture.png) +![Architecture diagram](conductor-architecture.png) The API and storage layers are pluggable and provide ability to work with different backends and queue service providers. ## Runtime Model Conductor follows RPC based communication model where workers are running on a separate machine from the server. Workers communicate with server over HTTP based endpoints and employs polling model for managing work queues. -![Runtime Model of Conductor](/img/overview.png) +![Runtime Model of Conductor](overview.png) -**Notes** +## Notes * Workers are remote systems that communicate over HTTP with the conductor servers. * Task Queues are used to schedule tasks for workers. We use [dyno-queues][1] internally but it can easily be swapped with SQS or similar pub-sub mechanism. diff --git a/docs/docs/img/overview.png b/docs/devguide/architecture/overview.png similarity index 100% rename from docs/docs/img/overview.png rename to docs/devguide/architecture/overview.png diff --git a/docs/docs/img/pirate_graph.gif b/docs/devguide/architecture/pirate_graph.gif similarity index 100% rename from docs/docs/img/pirate_graph.gif rename to docs/devguide/architecture/pirate_graph.gif diff --git a/docs/docs/img/regular_graph.png b/docs/devguide/architecture/regular_graph.png similarity index 100% rename from docs/docs/img/regular_graph.png rename to docs/devguide/architecture/regular_graph.png diff --git a/docs/docs/img/task_states.png b/docs/devguide/architecture/task_states.png similarity index 100% rename from docs/docs/img/task_states.png rename to docs/devguide/architecture/task_states.png diff --git a/docs/docs/architecture/tasklifecycle.md b/docs/devguide/architecture/tasklifecycle.md similarity index 54% rename from docs/docs/architecture/tasklifecycle.md rename to docs/devguide/architecture/tasklifecycle.md index 036071df8..09d7f5371 100644 --- a/docs/docs/architecture/tasklifecycle.md +++ b/docs/devguide/architecture/tasklifecycle.md @@ -1,47 +1,59 @@ +# Task Lifecycle + ## Task state transitions + The figure below depicts the state transitions that a task can go through within a workflow execution. -![Task_States](/img/task_states.png) +![Task States](task_states.png) ## Retries and Failure Scenarios ### Task failure and retries -Retries for failed task executions of each task can be configured independently. retryCount, retryDelaySeconds and retryLogic can be used to configure the retry mechanism. -![Task Failure](/img/TaskFailure.png) +Retries for failed task executions of each task can be configured independently. `retryCount`, `retryDelaySeconds` and `retryLogic` can be used to configure the retry mechanism. + +![Task Failure](TaskFailure.png) 1. Worker (W1) polls for task T1 from the Conductor server and receives the task. 2. Upon processing this task, the worker determines that the task execution is a failure and reports this to the server with FAILED status after 10 seconds. 3. The server will persist this FAILED execution of T1. A new execution of task T1 will be created and scheduled to be polled. This task will be available to be polled after 5 (retryDelaySeconds) seconds. +### Poll Timeout Seconds + +Poll timeout is the maximum amount of time by which a worker needs to poll a task, else the task will be marked as `TIMED_OUT`. + +![Task Poll Timeout](PollTimeoutSeconds.png) + +In the figure above, task T1 does not get polled by the worker within 60 seconds, so Conductor marks it as `TIMED_OUT`. ### Timeout seconds -Timeout is the maximum amount of time that the task must reach a terminal state in, else the task will be marked as TIMED_OUT. -![Task Timeout](/img/TimeoutSeconds.png) +Timeout is the maximum amount of time that the task must reach a terminal state in, else it will be marked as `TIMED_OUT`. + +![Task Timeout](TimeoutSeconds.png) -**0 seconds** -> Worker polls for task T1 from the Conductor server and receives the task. T1 is put into IN_PROGRESS status by the server. -Worker starts processing the task but is unable to process the task at this time. Worker updates the server with T1 set to IN_PROGRESS status and a callback of 9 seconds. +**0 seconds** -> Worker polls for task T1 from the Conductor server and receives the task. T1 is put into `IN_PROGRESS` status by the server. +Worker starts processing the task but is unable to process the task at this time. Worker updates the server with T1 set to `IN_PROGRESS` status and a callback of 9 seconds. Server puts T1 back in the queue but makes it invisible and the worker continues to poll for the task but does not receive T1 for 9 seconds. **9,18 seconds** -> Worker receives T1 from the server and is still unable to process the task and updates the server with a callback of 9 seconds. **27 seconds** -> Worker polls and receives task T1 from the server and is now able to process this task. -**30 seconds** (T1 timeout) -> Server marks T1 as TIMED_OUT because it is not in a terminal state after first being moved to IN_PROGRESS status. Server schedules a new task based on the retry count. - -**32 seconds** -> Worker completes processing of T1 and updates the server with COMPLETED status. Server will ignore this update since T1 has already been moved to a terminal status (TIMED_OUT). +**30 seconds** (T1 timeout) -> Server marks T1 as `TIMED_OUT` because it is not in a terminal state after first being moved to `IN_PROGRESS` status. Server schedules a new task based on the retry count. +**32 seconds** -> Worker completes processing of T1 and updates the server with `COMPLETED` status. Server will ignore this update since T1 has already been moved to a terminal status (`TIMED_OUT`). ### Response timeout seconds + Response timeout is the time within which the worker must respond to the server with an update for the task, else the task will be marked as TIMED_OUT. -![Response Timeout](/img/ResponseTimeoutSeconds.png) +![Response Timeout](ResponseTimeoutSeconds.png) -**0 seconds** -> Worker polls for the task T1 from the Conductor server and receives the task. T1 is put into IN_PROGRESS status by the server. +**0 seconds** -> Worker polls for the task T1 from the Conductor server and receives the task. T1 is put into `IN_PROGRESS` status by the server. Worker starts processing the task but the worker instance dies during this execution. -**20 seconds** (T1 responseTimeout) -> Server marks T1 as TIMED_OUT since the task has not been updated by the worker within the configured responseTimeoutSeconds (20). A new instance of task T1 is scheduled as per the retry configuration. +**20 seconds** (T1 responseTimeout) -> Server marks T1 as `TIMED_OUT` since the task has not been updated by the worker within the configured responseTimeoutSeconds (20). A new instance of task T1 is scheduled as per the retry configuration. **25 seconds** -> The retried instance of T1 is available to be polled by the worker, after the retryDelaySeconds (5) has elapsed. diff --git a/docs/docs/technicaldetails.md b/docs/devguide/architecture/technicaldetails.md similarity index 94% rename from docs/docs/technicaldetails.md rename to docs/devguide/architecture/technicaldetails.md index afcc8f5a0..934489207 100644 --- a/docs/docs/technicaldetails.md +++ b/docs/devguide/architecture/technicaldetails.md @@ -7,13 +7,13 @@ The proto models are auto-generated at compile time using this ProtoGen library. ### Cassandra Persistence -The Cassandra persistence layer currently provides a partial implementation of the ExecutionDAO that supports all the CRUD operations for tasks and workflow execution. The data modelling is done in a denormalized manner and stored in two tables. The “workflows” table houses all the information for a workflow execution including all its tasks and is the source of truth for all the information regarding a workflow and its tasks. The “task_lookup” table, as the name suggests stores a lookup of taskIds to workflowId. This table facilitates the fast retrieval of task data given a taskId. +The Cassandra persistence layer currently provides a partial implementation of the ExecutionDAO that supports all the CRUD operations for tasks and workflow execution. The data modelling is done in a denormalized manner and stored in two tables. The "workflows" table houses all the information for a workflow execution including all its tasks and is the source of truth for all the information regarding a workflow and its tasks. The "task_lookup" table, as the name suggests stores a lookup of taskIds to workflowId. This table facilitates the fast retrieval of task data given a taskId. All the datastore operations that are used during the critical execution path of a workflow have been implemented currently. Few of the operational abilities of the ExecutionDAO are yet to be implemented. This module also does not provide implementations for QueueDAO, PollDataDAO and RateLimitingDAO. We envision using the Cassandra DAO with an external queue implementation, since implementing a queuing recipe on top of Cassandra is an anti-pattern that we want to stay away from. ### External Payload Storage The implementation of this feature is such that the externalization of payloads is fully transparent and automated to the user. Conductor operators can configure the usage of this feature and is completely abstracted and hidden from the user, thereby allowing the operators full control over the barrier limits. Currently, only AWS S3 is supported as a storage system, however, as with all other Conductor components, this is pluggable and can be extended to enable any other object store to be used as an external payload storage system. -The externalization of payloads is enforced using two kinds of [barriers](/externalpayloadstorage.html). Soft barriers are used when the payload size is warranted enough to be stored as part of workflow execution. These payloads will be stored in external storage and used during execution. Hard barriers are enforced to safeguard against voluminous data, and such payloads are rejected and the workflow execution is failed. +The externalization of payloads is enforced using two kinds of [barriers](../../documentation/advanced/externalpayloadstorage.md). Soft barriers are used when the payload size is warranted enough to be stored as part of workflow execution. These payloads will be stored in external storage and used during execution. Hard barriers are enforced to safeguard against voluminous data, and such payloads are rejected and the workflow execution is failed. The payload size is evaluated in the client before being sent over the wire to the server. If the payload size exceeds the configured soft limit, the client makes a request to the server for the location at which the payload is to be stored. In this case where S3 is being used, the server returns a signed url for the location and the client uploads the payload using this signed url. The relative path to the payload object is then stored in the workflow/task metadata. The server can then download this payload from this path and use as needed during execution. This allows the server to control access to the S3 bucket, thereby making the user applications where the worker processes are run completely agnostic of the permissions needed to access this location. diff --git a/docs/docs/bestpractices.md b/docs/devguide/bestpractices.md similarity index 97% rename from docs/docs/bestpractices.md rename to docs/devguide/bestpractices.md index 3889bddaf..d753cbbc8 100644 --- a/docs/docs/bestpractices.md +++ b/docs/devguide/bestpractices.md @@ -1,3 +1,5 @@ +# Best Practices + ## Response Timeout - Configure the responseTimeoutSeconds of each task to be > 0. - Should be less than or equal to timeoutSeconds. diff --git a/docs/devguide/concepts/index.md b/docs/devguide/concepts/index.md new file mode 100644 index 000000000..6eace3d85 --- /dev/null +++ b/docs/devguide/concepts/index.md @@ -0,0 +1,17 @@ +# Basic Concepts +Conductor allows you to build a complex application using simple and granular tasks that do not +need to be aware of or keep track of the state of your application's execution flow. Conductor keeps track of the state, +calls tasks in the right order (sequentially or in parallel, as defined by you), retry calls if needed, handle failure +scenarios gracefully, and outputs the final result. + + +![Workflow screnshot](../../home/devex.png) + +Leveraging workflows in Conductor enables developers to truly focus on their core mission - building their application +code in the languages of their choice. Conductor does the heavy lifting associated with ensuring high +reliability, transactional consistency, and long durability of their workflows. Simply put, wherever your application's +component lives and whichever languages they were written in, you can build a workflow in Conductor to orchestrate their +execution in a reliable & scalable manner. + +[Workflows](workflows.md) and [Tasks](tasks.md) are the two key concepts that underlie the Conductor system. + diff --git a/docs/devguide/concepts/tasks.md b/docs/devguide/concepts/tasks.md new file mode 100644 index 000000000..062ad20cd --- /dev/null +++ b/docs/devguide/concepts/tasks.md @@ -0,0 +1,32 @@ +# Tasks +Tasks are the building blocks of Conductor Workflows. There must be at least one task configured in each Workflow Definition. A typical Conductor workflow defines a lists of tasks that are executed until the completion or termination of the workflow. + +Tasks can be categorized into three types: + +## Types of Tasks +### System Tasks +[**System Tasks**](../../documentation/configuration/workflowdef/systemtasks/index.md) are built-in tasks that are general purpose and re-usable. They are executed within the JVM of the Conductor server and managed by Conductor for execution and scalability. Such tasks allow you to get started without having to write custom workers. + +### Simple Tasks +[**Simple Tasks**](workers.md) or Worker Tasks are implemented by your application and run in a separate environment from Conductor. These tasks talk to the Conductor server via REST/gRPC to poll for tasks and update its status after execution. + +### Operators +[**Operators**](../../documentation/configuration/workflowdef/operators/index.md) are built-in primitives in Conductor that allow you control the flow of tasks in your workflow. Operators are similar to programming constructs such as `for` loops, `switch` blocks, etc. + +## Task Configuration +Task Configurations appear within the `tasks` array property of the Workflow Definition. This array is the blueprint that describes how a workflow will process an input payload by passing it through successive tasks. + +* For all tasks, the configuration will specifiy what **input parameters** the task takes. +* For SIMPLE (worker based) tasks, the configuration will contain a reference to a registered worker `taskName`. +* For System Tasks and Operators, the task configuration will contain important parameters that control the behavior of the task. For example, the task configuration of an HTTP task will specify an endpoint URL and the templatized payload that it will be called with when the task executes. + +## Task Definition +Not to be confused with Task Configurations, [Task Definitions](../../documentation/configuration/taskdef.md) help define default task level parameters like inputs and outputs, timeouts, retries etc. for SIMPLE (i.e. worker implemented) tasks. + +* All simple tasks need to be registered before they can be used by active workflows. +* Task definitions can be registered via the UI, or through the API. +* A registered task definition can be referenced from within different workflows. + +## Task Execution +Each time a workload is passed into a configured task, a Task Execution object is created. This object has a unique ID and represents the result of the operation. This includes the status (i.e. whether the task was completed successfully), and any input, output and variables associated with the task. + diff --git a/docs/docs/gettingstarted/intro.md b/docs/devguide/concepts/why.md similarity index 86% rename from docs/docs/gettingstarted/intro.md rename to docs/devguide/concepts/why.md index 789a7db4b..8acb0d15b 100644 --- a/docs/docs/gettingstarted/intro.md +++ b/docs/devguide/concepts/why.md @@ -1,5 +1,7 @@ # Why Conductor? -## Conductor was built to help Netflix orchestrate microservices based process flows with the following features: +Conductor was built to help Netflix orchestrate microservices based process flows. + +## Features * A distributed server ecosystem, which stores workflow state information efficiently. * Allow creation of process / business flows in which each individual task can be implemented by the same / different microservices. @@ -23,6 +25,6 @@ With peer to peer task choreography, we found it was harder to scale with growing business needs and complexities. Pub/sub model worked for simplest of the flows, but quickly highlighted some of the issues associated with the approach: -* Process flows are “embedded” within the code of multiple application. +* Process flows are "embedded" within the code of multiple application. * Often, there is tight coupling and assumptions around input/output, SLAs etc, making it harder to adapt to changing needs. -* Almost no way to systematically answer “How much are we done with process X”? +* Almost no way to systematically answer "How much are we done with process X"? diff --git a/docs/devguide/concepts/workers.md b/docs/devguide/concepts/workers.md new file mode 100644 index 000000000..faf6c6c85 --- /dev/null +++ b/docs/devguide/concepts/workers.md @@ -0,0 +1,11 @@ +# Workers +A worker is responsible for executing a task. Workers can be implemented in any language, and Conductor provides a polyglot set of worker frameworks that provide features such as polling threads, metrics and server communication that makes creating workers easy. + +Each worker embodies the Microservice design pattern and follows certain basic principles: + +1. Workers are stateless and do not implement a workflow specific logic. +2. Each worker executes a very specific task and produces well defined output given specific inputs. +3. Workers are meant to be idempotent (or should handle cases where the task that partially executed gets rescheduled due to timeouts etc.) +4. Workers do not implement the logic to handle retries etc, that is taken care by the Conductor server. + +Conductor maintains a registry of worker tasks. A task MUST be registered before being used in a workflow. This can be done by creating and saving a **Task Definition**. \ No newline at end of file diff --git a/docs/devguide/concepts/workflows.md b/docs/devguide/concepts/workflows.md new file mode 100644 index 000000000..d4c17618c --- /dev/null +++ b/docs/devguide/concepts/workflows.md @@ -0,0 +1,13 @@ +# Workflows +We will talk about two distinct topics, *defining* a workflow and *executing* a workflow. + +### Workflow Definition +The Workflow Definition is the Conductor primitive that encompasses the flow of your business logic. It contains all the information necessary to describe the behavior of a workflow. + +A Workflow Definition contains a collection of **Task Configurations**. This is the blueprint which specifies the order of execution of +tasks within a workflow. This blueprint also specifies how data/state is passed from one task to another (using task input/output parameters). + +Additionally, the Workflow Definition contains metadata regulating the runtime behavior workflow, such what input and output parameters are expected for the entire workflow, and the workflow's the timeout and retry settings. + +### Workflow Execution +If Workflow Definitions are like OOP classes, then Workflows Executions are like object instances. Each time a Workflow Definition is invoked with a given input, a new *Workflow Execution* with a unique ID is created. Definitions to Executions have a 1:N relationship. diff --git a/docs/docs/faq.md b/docs/devguide/faq.md similarity index 68% rename from docs/docs/faq.md rename to docs/devguide/faq.md index 858d0ecfe..5d14a1835 100644 --- a/docs/docs/faq.md +++ b/docs/devguide/faq.md @@ -1,41 +1,41 @@ # Frequently asked Questions -### How do you schedule a task to be put in the queue after some time (e.g. 1 hour, 1 day etc.) +## How do you schedule a task to be put in the queue after some time (e.g. 1 hour, 1 day etc.) After polling for the task update the status of the task to `IN_PROGRESS` and set the `callbackAfterSeconds` value to the desired time. The task will remain in the queue until the specified second before worker polling for it will receive it again. If there is a timeout set for the task, and the `callbackAfterSeconds` exceeds the timeout value, it will result in task being TIMED_OUT. -### How long can a workflow be in running state? Can I have a workflow that keeps running for days or months? +## How long can a workflow be in running state? Can I have a workflow that keeps running for days or months? Yes. As long as the timeouts on the tasks are set to handle long running workflows, it will stay in running state. -### My workflow fails to start with missing task error +## My workflow fails to start with missing task error Ensure all the tasks are registered via `/metadata/taskdefs` APIs. Add any missing task definition (as reported in the error) and try again. -### Where does my worker run? How does conductor run my tasks? +## Where does my worker run? How does conductor run my tasks? -Conductor does not run the workers. When a task is scheduled, it is put into the queue maintained by Conductor. Workers are required to poll for tasks using `/tasks/poll` API at periodic interval, execute the business logic for the task and report back the results using `POST /tasks` API call. -Conductor, however will run [system tasks](/configuration/systask.html) on the Conductor server. +Conductor does not run the workers. When a task is scheduled, it is put into the queue maintained by Conductor. Workers are required to poll for tasks using `/tasks/poll` API at periodic interval, execute the business logic for the task and report back the results using `POST {{ api_prefix }}/tasks` API call. +Conductor, however will run [system tasks](../documentation/configuration/workflowdef/systemtasks/index.md) on the Conductor server. -### How can I schedule workflows to run at a specific time? +## How can I schedule workflows to run at a specific time? Netflix Conductor itself does not provide any scheduling mechanism. But there is a community project [_Schedule Conductor Workflows_](https://github.com/jas34/scheduledwf) which provides workflow scheduling capability as a pluggable module as well as workflow server. Other way is you can use any of the available scheduling systems to make REST calls to Conductor to start a workflow. Alternatively, publish a message to a supported eventing system like SQS to trigger a workflow. -More details about [eventing](/configuration/eventhandlers.html). +More details about [eventing](../documentation/configuration/eventhandlers.md). -### How do I setup Dynomite cluster? +## How do I setup Dynomite cluster? Visit Dynomite's [Github page](https://github.com/Netflix/dynomite) to find details on setup and support mechanism. -### Can I use conductor with Ruby / Go / Python? +## Can I use conductor with Ruby / Go / Python? Yes. Workers can be written any language as long as they can poll and update the task results via HTTP endpoints. @@ -44,18 +44,18 @@ Conductor provides frameworks for Java and Python to simplify the task of pollin **Note:** Python and Go clients have been contributed by the community. -### How can I get help with Dynomite? +## How can I get help with Dynomite? Visit Dynomite's [Github page](https://github.com/Netflix/dynomite) to find details on setup and support mechanism. -### My workflow is running and the task is SCHEDULED but it is not being processed. +## My workflow is running and the task is SCHEDULED but it is not being processed. Make sure that the worker is actively polling for this task. Navigate to the `Task Queues` tab on the Conductor UI and select your task name in the search box. Ensure that `Last Poll Time` for this task is current. In Conductor 3.x, ```conductor.redis.availabilityZone``` defaults to ```us-east-1c```. Ensure that this matches where your workers are, and that it also matches```conductor.redis.hosts```. -### How do I configure a notification when my workflow completes or fails? +## How do I configure a notification when my workflow completes or fails? When a workflow fails, you can configure a "failure workflow" to run using the```failureWorkflow``` parameter. By default, three parameters are passed: @@ -65,20 +65,20 @@ When a workflow fails, you can configure a "failure workflow" to run using the`` You can also use the Workflow Status Listener: -* Set the workflowStatusListenerEnabled field in your workflow definition to true which enables [notifications](/configuration/workflowdef.html#workflow-notifications). -* Add a custom implementation of the Workflow Status Listener. Refer [this](/extend.html#workflow-status-listener). -* This notification can be implemented in such a way as to either send a notification to an external system or to send an event on the conductor queue to complete/fail another task in another workflow as described [here](/configuration/eventhandlers.html). +* Set the workflowStatusListenerEnabled field in your workflow definition to true which enables [notifications](../documentation/configuration/workflowdef/index.md#workflow-notifications). +* Add a custom implementation of the Workflow Status Listener. Refer [this](../documentation/advanced/extend.md#workflow-status-listener). +* This notification can be implemented in such a way as to either send a notification to an external system or to send an event on the conductor queue to complete/fail another task in another workflow as described [here](../documentation/configuration/eventhandlers.md). -Refer to this [documentation](/configuration/workflowdef.html#workflow-notifications) to extend conductor to send out events/notifications upon workflow completion/failure. +Refer to this [documentation](../documentation/configuration/workflowdef/index.md#workflow-notifications) to extend conductor to send out events/notifications upon workflow completion/failure. -### I want my worker to stop polling and executing tasks when the process is being terminated. (Java client) +## I want my worker to stop polling and executing tasks when the process is being terminated. (Java client) In a `PreDestroy` block within your application, call the `shutdown()` method on the `TaskRunnerConfigurer` instance that you have created to facilitate a graceful shutdown of your worker in case the process is being terminated. -### Can I exit early from a task without executing the configured automatic retries in the task definition? +## Can I exit early from a task without executing the configured automatic retries in the task definition? Set the status to `FAILED_WITH_TERMINAL_ERROR` in the TaskResult object within your worker. This would mark the task as FAILED and fail the workflow without retrying the task as a fail-fast mechanism. diff --git a/docs/docs/how-tos/Monitoring/Conductor-LogLevel.md b/docs/devguide/how-tos/Monitoring/Conductor-LogLevel.md similarity index 100% rename from docs/docs/how-tos/Monitoring/Conductor-LogLevel.md rename to docs/devguide/how-tos/Monitoring/Conductor-LogLevel.md diff --git a/docs/docs/how-tos/Tasks/creating-tasks.md b/docs/devguide/how-tos/Tasks/creating-tasks.md similarity index 84% rename from docs/docs/how-tos/Tasks/creating-tasks.md rename to docs/devguide/how-tos/Tasks/creating-tasks.md index 54ae74e91..d79da9c96 100644 --- a/docs/docs/how-tos/Tasks/creating-tasks.md +++ b/docs/devguide/how-tos/Tasks/creating-tasks.md @@ -1,24 +1,22 @@ # Creating Task Definitions - Tasks can be created using the tasks metadata API -`POST /api/metadata/taskdefs` +`POST {{ api_prefix }}/metadata/taskdefs` This API takes an array of new task definitions. +## Examples ### Example using curl - ```shell -curl 'http://localhost:8080/api/metadata/taskdefs' \ +curl '{{ server_host }}{{ api_prefix }}/metadata/taskdefs' \ -H 'accept: */*' \ -H 'content-type: application/json' \ --data-raw '[{"createdBy":"user","name":"sample_task_name_1","description":"This is a sample task for demo","responseTimeoutSeconds":10,"timeoutSeconds":30,"inputKeys":[],"outputKeys":[],"timeoutPolicy":"TIME_OUT_WF","retryCount":3,"retryLogic":"FIXED","retryDelaySeconds":5,"inputTemplate":{},"rateLimitPerFrequency":0,"rateLimitFrequencyInSeconds":1}]' ``` ### Example using node fetch - ```javascript -fetch("http://localhost:8080/api/metadata/taskdefs", { +fetch("{{ server_host }}{{ api_prefix }}/metadata/taskdefs", { "headers": { "accept": "*/*", "content-type": "application/json", @@ -28,8 +26,7 @@ fetch("http://localhost:8080/api/metadata/taskdefs", { }); ``` ## Best Practices - 1. You can update a set of tasks together in this API -2. Task configurations are important attributes that controls the behavior of this task in a Workflow. Refer to [Task Configurations](/configuration/taskdef.html) for all the options and details' +2. Task configurations are important attributes that controls the behavior of this task in a Workflow. Refer to [Task Configurations](../../../documentation/configuration/taskdef.md) for all the options and details' 3. You can also use the Conductor Swagger UI to update the tasks diff --git a/docs/devguide/how-tos/Tasks/dynamic-vs-switch-tasks.md b/docs/devguide/how-tos/Tasks/dynamic-vs-switch-tasks.md new file mode 100644 index 000000000..0339f4cc0 --- /dev/null +++ b/docs/devguide/how-tos/Tasks/dynamic-vs-switch-tasks.md @@ -0,0 +1,18 @@ +# Dynamic vs Switch Tasks + +Dynamic Tasks are useful in situations when need to run a task of which the task type is determined at runtime instead +of during the configuration. It is similar to the `SWITCH` use case but with `DYNAMIC` +we won't need to preconfigure all case options in the workflow definition itself. Instead, we can mark the task +as `DYNAMIC` and determine which underlying task does it run during the workflow execution itself. + +* Use DYNAMIC task as a replacement for SWITCH if you have too many case options +* DYNAMIC task is an option when you want to programmatically determine the next task to run instead of using expressions +* DYNAMIC task simplifies the workflow execution UI view which will now only show the selected task +* SWITCH task visualization is helpful as a documentation - showing you all options that the workflow could have + taken +* SWITCH task comes with a default task option which can be useful in some use cases + +Learn more about + +* [Dynamic Tasks](../../../documentation/configuration/workflowdef/operators/dynamic-task.md) +* [Switch Tasks](../../../documentation/configuration/workflowdef/operators/switch-task.md) \ No newline at end of file diff --git a/docs/docs/how-tos/Tasks/extending-system-tasks.md b/docs/devguide/how-tos/Tasks/extending-system-tasks.md similarity index 85% rename from docs/docs/how-tos/Tasks/extending-system-tasks.md rename to docs/devguide/how-tos/Tasks/extending-system-tasks.md index 5661d040c..e219fae61 100644 --- a/docs/docs/how-tos/Tasks/extending-system-tasks.md +++ b/docs/devguide/how-tos/Tasks/extending-system-tasks.md @@ -1,6 +1,6 @@ # Extending System Tasks -[System tasks](/configuration/systask.html) allow Conductor to run simple tasks on the server - removing the need to build (and deploy) workers for basic tasks. This allows for automating more mundane tasks without building specific microservices for them. +[System tasks](../../../documentation/configuration/workflowdef/systemtasks/index.md) allow Conductor to run simple tasks on the server - removing the need to build (and deploy) workers for basic tasks. This allows for automating more mundane tasks without building specific microservices for them. However, sometimes it might be necessary to add additional parameters to a System Task to gain the behavior that is desired. @@ -58,7 +58,7 @@ When this workflow is run - it fails, as expected. Now, sometimes an API call might fail due to an issue on the remote server, and retrying the call will result in a response. With many Conductor tasks, ```retryCount```, ```retryDelaySeconds``` and ```retryLogic``` fields can be applied to retry the worker (with the desired parameters). -By default, the [HTTP Task](/reference-docs/http-task.html) does not have ```retryCount```, ```retryDelaySeconds``` or ```retryLogic``` built in. Attempting to add these parameters to a HTTP Task results in an error. +By default, the [HTTP Task](../../../documentation/configuration/workflowdef/systemtasks/http-task.md) does not have ```retryCount```, ```retryDelaySeconds``` or ```retryLogic``` built in. Attempting to add these parameters to a HTTP Task results in an error. ## The Solution diff --git a/docs/docs/how-tos/Tasks/monitoring-task-queues.md b/docs/devguide/how-tos/Tasks/monitoring-task-queues.md similarity index 72% rename from docs/docs/how-tos/Tasks/monitoring-task-queues.md rename to docs/devguide/how-tos/Tasks/monitoring-task-queues.md index 584b89102..4de10727a 100644 --- a/docs/docs/how-tos/Tasks/monitoring-task-queues.md +++ b/docs/devguide/how-tos/Tasks/monitoring-task-queues.md @@ -1,15 +1,11 @@ ---- -sidebar_position: 1 ---- - # Monitoring Task Queues Conductor offers an API and UI interface to monitor the task queues. This is useful to see details of the number of workers polling and monitoring the queue backlog. -### Using the UI +## Using the UI -```http request +``` /taskQueue ``` @@ -20,20 +16,21 @@ On this screen you can select and view the details of the task queue. The follow 1. Queue Size - The number of tasks waiting to be executed 2. Workers - The count and list of works and their instance reference who are polling for work for this task -### Using APIs +## Using APIs To see the size of the task queue via API: ```shell -curl 'http://localhost:8080/api/tasks/queue/sizes?taskType=' \ +curl '{{ server_host }}{{ api_prefix }}/tasks/queue/sizes?taskType=' \ -H 'accept: */*' ``` To see the worker poll information of the task queue via API: ```shell -curl 'http://localhost:8080/api/tasks/queue/polldata?taskType=' \ +curl '{{ server_host }}{{ api_prefix }}/tasks/queue/polldata?taskType=' \ -H 'accept: */*' ``` -> Replace `` with your task name +!!! note + Replace `` with your task name diff --git a/docs/docs/how-tos/Tasks/reusing-tasks.md b/docs/devguide/how-tos/Tasks/reusing-tasks.md similarity index 96% rename from docs/docs/how-tos/Tasks/reusing-tasks.md rename to docs/devguide/how-tos/Tasks/reusing-tasks.md index 7923a2e43..1b3f2dbd7 100644 --- a/docs/docs/how-tos/Tasks/reusing-tasks.md +++ b/docs/devguide/how-tos/Tasks/reusing-tasks.md @@ -1,7 +1,3 @@ ---- -sidebar_position: 1 ---- - # Reusing Tasks A powerful feature of Conductor is that it supports and enables re-usability out of the box. Task workers typically diff --git a/docs/docs/how-tos/Tasks/task-configurations.md b/docs/devguide/how-tos/Tasks/task-configurations.md similarity index 54% rename from docs/docs/how-tos/Tasks/task-configurations.md rename to docs/devguide/how-tos/Tasks/task-configurations.md index 1fef6c709..7c25282bc 100644 --- a/docs/docs/how-tos/Tasks/task-configurations.md +++ b/docs/devguide/how-tos/Tasks/task-configurations.md @@ -1,12 +1,8 @@ ---- -sidebar_position: 1 ---- - # Task Configurations -Refer to [Task Definitions](/configuration/taskdef.html) for details on how to configure task definitions +Refer to [Task Definitions](../../../documentation/configuration/taskdef.md) for details on how to configure task definitions -### Example +## Example Here is a task template payload with commonly used fields: @@ -29,7 +25,7 @@ Here is a task template payload with commonly used fields: } ``` -### Best Practices +## Best Practices -1. Refer to [Task Timeouts](/how-tos/Tasks/task-timeouts.html) for additional information on how the various timeout settings work -2. Refer to [Monitoring Task Queues](/how-tos/Tasks/monitoring-task-queues.html) on how to monitor task queues +1. Refer to [Task Timeouts](task-timeouts.md) for additional information on how the various timeout settings work +2. Refer to [Monitoring Task Queues](monitoring-task-queues.md) on how to monitor task queues diff --git a/docs/docs/how-tos/Tasks/task-inputs.md b/docs/devguide/how-tos/Tasks/task-inputs.md similarity index 93% rename from docs/docs/how-tos/Tasks/task-inputs.md rename to docs/devguide/how-tos/Tasks/task-inputs.md index 41f6df61a..53cae8d79 100644 --- a/docs/docs/how-tos/Tasks/task-inputs.md +++ b/docs/devguide/how-tos/Tasks/task-inputs.md @@ -1,13 +1,9 @@ ---- -sidebar_position: 1 ---- - # Task Inputs Task inputs can be provided in multiple ways. This is configured in the workflow definition when a task is participating in the workflow. -### Inputs referred from Workflow inputs +## Inputs referred from Workflow inputs When we start a workflow, we can provide inputs to the workflow in a json format. For example: @@ -38,7 +34,7 @@ In this example, the tasks will receive the following inputs after they are eval } ``` -### Inputs referred from other Task outputs +## Inputs referred from other Task outputs Similar to how we can refer to workflow inputs, we can also refer to an output field that was generated by a task that executed before. @@ -66,7 +62,7 @@ We can refer to these as the new task's input by using the following expression: The expression format is based on [Json Path](https://goessner.net/articles/JsonPath/) and you can construct complex input params based on the syntax. -### Hard coded inputs +## Hard coded inputs Task inputs can also be hard coded in the workflow definitions. This is useful when you have a re-usable task which has configurable options that can be applied in different workflow contexts. diff --git a/docs/docs/how-tos/Tasks/task-timeouts.md b/docs/devguide/how-tos/Tasks/task-timeouts.md similarity index 95% rename from docs/docs/how-tos/Tasks/task-timeouts.md rename to docs/devguide/how-tos/Tasks/task-timeouts.md index 74aa51ab6..eda32e377 100644 --- a/docs/docs/how-tos/Tasks/task-timeouts.md +++ b/docs/devguide/how-tos/Tasks/task-timeouts.md @@ -1,7 +1,3 @@ ---- -sidebar_position: 1 ---- - # Task Timeouts Tasks can be configured to handle various scenarios of timeouts. Here are some scenarios and the relevance configuration @@ -17,7 +13,7 @@ fields. > `timeoutSeconds` should always be greater than `responseTimeoutSeconds` -### Timeout Seconds +## Timeout Seconds ```json "timeoutSeconds" : 30 @@ -27,7 +23,7 @@ When configured with a value > `0`, the system will wait for this task to comple seconds from when the task is first polled. We can use this to fail a workflow when a task breaches the overall SLA for completion. -### Response Timeout Seconds +## Response Timeout Seconds ```json "responseTimeoutSeconds" : 10 @@ -37,7 +33,7 @@ When configured with a value > `0`, the system will wait for this number of seco the worker updates back with a status. The worker can keep the task in `IN_PROGRESS` state if it requires more time to complete. -### Poll Timeout Seconds +## Poll Timeout Seconds ```json "pollTimeoutSeconds" : 10 diff --git a/docs/docs/how-tos/Tasks/updating-tasks.md b/docs/devguide/how-tos/Tasks/updating-tasks.md similarity index 85% rename from docs/docs/how-tos/Tasks/updating-tasks.md rename to docs/devguide/how-tos/Tasks/updating-tasks.md index 4978e80b3..f38d113f3 100644 --- a/docs/docs/how-tos/Tasks/updating-tasks.md +++ b/docs/devguide/how-tos/Tasks/updating-tasks.md @@ -1,23 +1,19 @@ ---- -sidebar_position: 1 ---- - # Updating Task Definitions Updates to the task definitions can be made using the following API ```http -PUT /api/metadata/taskdefs +PUT {{ api_prefix }}/metadata/taskdefs ``` This API takes a single task definition and updates itself. - +## Examples ### Example using curl ```shell -curl 'http://localhost:8080/api/metadata/taskdefs' \ +curl '{{ server_host }}{{ api_prefix }}/metadata/taskdefs' \ -X 'PUT' \ -H 'accept: */*' \ -H 'content-type: application/json' \ @@ -27,7 +23,7 @@ curl 'http://localhost:8080/api/metadata/taskdefs' \ ### Example using node fetch ```javascript -fetch("http://localhost:8080/api/metadata/taskdefs", { +fetch("{{ server_host }}{{ api_prefix }}/metadata/taskdefs", { "headers": { "accept": "*/*", "content-type": "application/json", @@ -39,4 +35,4 @@ fetch("http://localhost:8080/api/metadata/taskdefs", { ## Best Practices 1. You can also use the Conductor Swagger UI to update the tasks -2. Task configurations are important attributes that controls the behavior of this task in a Workflow. Refer to [Task Configurations](/how-tos/Tasks/task-configurations.html) for all the options and details' +2. Task configurations are important attributes that controls the behavior of this task in a Workflow. Refer to [Task Configurations](task-configurations.md) for all the options and details' diff --git a/docs/docs/how-tos/Workers/build-a-golang-task-worker.md b/docs/devguide/how-tos/Workers/build-a-golang-task-worker.md similarity index 86% rename from docs/docs/how-tos/Workers/build-a-golang-task-worker.md rename to docs/devguide/how-tos/Workers/build-a-golang-task-worker.md index 5581508ba..20cb668d1 100644 --- a/docs/docs/how-tos/Workers/build-a-golang-task-worker.md +++ b/docs/devguide/how-tos/Workers/build-a-golang-task-worker.md @@ -1,7 +1,3 @@ ---- -sidebar_position: 1 ---- - # Build a Go Task Worker ## Install @@ -11,7 +7,7 @@ go get github.com/netflix/conductor/client/go This will create a Go project under $GOPATH/src and download any dependencies. ## Implementing a Task a Worker -`task`package provides the types used to implement the worker. Here is a reference worker implementation: +`task`package provies the types used to implement the worker. Here is a reference worker implementation: ```go package task @@ -48,7 +44,7 @@ import ( ) func main() { - c := conductor.NewConductorWorker("http://localhost:8080", 1, 10000) + c := conductor.NewConductorWorker("{{ server_host }}", 1, 10000) c.Start("task_1", "", sample.Task_1_Execution_Function, false) c.Start("task_2", "mydomain", sample.Task_2_Execution_Function, true) diff --git a/docs/docs/how-tos/Workers/build-a-java-task-worker.md b/docs/devguide/how-tos/Workers/build-a-java-task-worker.md similarity index 97% rename from docs/docs/how-tos/Workers/build-a-java-task-worker.md rename to docs/devguide/how-tos/Workers/build-a-java-task-worker.md index cf90f656b..322834152 100644 --- a/docs/docs/how-tos/Workers/build-a-java-task-worker.md +++ b/docs/devguide/how-tos/Workers/build-a-java-task-worker.md @@ -1,7 +1,3 @@ ---- -sidebar_position: 1 ---- - # Build a Java Task Worker This guide provides introduction to building Task Workers in Java. @@ -85,7 +81,7 @@ Use the [Builder](https://github.com/Netflix/conductor/blob/main/client/src/main ```java TaskClient taskClient = new TaskClient(); - taskClient.setRootURI("http://localhost:8080/api/"); //Point this to the server API + taskClient.setRootURI("{{ server_host }}{{ api_prefix }}/"); //Point this to the server API int threadCount = 2; //number of threads used to execute workers. To avoid starvation, should be same or more than number of workers diff --git a/docs/docs/how-tos/Workers/build-a-python-task-worker.md b/docs/devguide/how-tos/Workers/build-a-python-task-worker.md similarity index 95% rename from docs/docs/how-tos/Workers/build-a-python-task-worker.md rename to docs/devguide/how-tos/Workers/build-a-python-task-worker.md index d45657fd1..875571df7 100644 --- a/docs/docs/how-tos/Workers/build-a-python-task-worker.md +++ b/docs/devguide/how-tos/Workers/build-a-python-task-worker.md @@ -1,7 +1,3 @@ ---- -sidebar_position: 1 ---- - # Build a Python Task Worker ## Install the python client ```shell @@ -28,7 +24,7 @@ def book_car_task(task): def main(): print('Starting Travel Booking workflows') - cc = ConductorWorker('http://localhost:8080/api', 1, 0.1) + cc = ConductorWorker('{{ server_host }}{{ api_prefix }}', 1, 0.1) cc.start('book_flight', book_flight_task, False) cc.start('book_car', book_car_task, True) @@ -39,7 +35,7 @@ if __name__ == '__main__': ```python server_url: str The url to the server hosting the conductor api. - Ex: 'http://localhost:8080/api' + Ex: '{{ server_host }}{{ api_prefix }}' thread_count: int The number of threads that will be polling for and diff --git a/docs/docs/how-tos/Workflows/debugging-workflows.md b/docs/devguide/how-tos/Workflows/debugging-workflows.md similarity index 92% rename from docs/docs/how-tos/Workflows/debugging-workflows.md rename to docs/devguide/how-tos/Workflows/debugging-workflows.md index d15ec07f1..fbcb6f11c 100644 --- a/docs/docs/how-tos/Workflows/debugging-workflows.md +++ b/docs/devguide/how-tos/Workflows/debugging-workflows.md @@ -1,14 +1,10 @@ ---- -sidebar_position: 1 ---- - # Debugging Workflows Conductor UI is a tool that we can leverage for debugging issues. Refer to the following articles to search and view your workflow execution. -1. [Searching Workflows](/how-tos/Workflows/searching-workflows.html) -2. [View Workflow Executions](/how-tos/Workflows/view-workflow-executions.html) +1. [Searching Workflows](searching-workflows.md) +2. [View Workflow Executions](view-workflow-executions.md) ## Debugging Executions @@ -33,7 +29,7 @@ Note: We can also access the task list from **Tasks > Task List** tab. Here is a screen grab of the fields referred above. -![Debugging Wowkflow Execution](/img/tutorial/workflow_debugging.png) +![Debugging Wowkflow Execution](workflow_debugging.png) ## Recovering From Failures diff --git a/docs/docs/how-tos/Workflows/handling-errors.md b/docs/devguide/how-tos/Workflows/handling-errors.md similarity index 97% rename from docs/docs/how-tos/Workflows/handling-errors.md rename to docs/devguide/how-tos/Workflows/handling-errors.md index 1d07e8cdd..b58fe1e91 100644 --- a/docs/docs/how-tos/Workflows/handling-errors.md +++ b/docs/devguide/how-tos/Workflows/handling-errors.md @@ -54,6 +54,6 @@ Here is a sample failure workflow that sends a message to Slack when the workflo } ``` -## Set ```workflowStatusListenerEnabled``` +## Set ```workfowStatusListenerEnabled``` -When this is enabled, notifications are now possible, and by building a custom implementation of the Workflow Status Listener, a notification can be sent to an external service. [More details.](https://github.com/Netflix/conductor/issues/1017#issuecomment-468869173) +When this is enabled, notifications are now possible, and by building a custom implementation of the Workflow Status Listener, a notification can be sent to an external service. [More details.](https://github.com/Netflix/conductor/issues/1017#issuecomment-468869173) \ No newline at end of file diff --git a/docs/docs/how-tos/Workflows/searching-workflows.md b/docs/devguide/how-tos/Workflows/searching-workflows.md similarity index 88% rename from docs/docs/how-tos/Workflows/searching-workflows.md rename to docs/devguide/how-tos/Workflows/searching-workflows.md index be1f82c56..0e18308bc 100644 --- a/docs/docs/how-tos/Workflows/searching-workflows.md +++ b/docs/devguide/how-tos/Workflows/searching-workflows.md @@ -1,19 +1,7 @@ ---- -sidebar_position: 1 ---- - # Searching Workflows In this article we will learn how to search through workflow executions via the UI. -### Prerequisites - -1. Conductor app and UI installed and running in an environment. If required we can look at the following options to get - an environment up and running. - - 1. [Build and Run Conductor Locally](/gettingstarted/local.html) - 2. [Running via Docker Compose](/gettingstarted/docker.html) - ## UI Workflows View Open the home page of the UI installation. It will take you to the `Workflow Executions` view. This is where we can look diff --git a/docs/docs/how-tos/Workflows/starting-workflows.md b/docs/devguide/how-tos/Workflows/starting-workflows.md similarity index 84% rename from docs/docs/how-tos/Workflows/starting-workflows.md rename to docs/devguide/how-tos/Workflows/starting-workflows.md index e1c6ea411..32f586f57 100644 --- a/docs/docs/how-tos/Workflows/starting-workflows.md +++ b/docs/devguide/how-tos/Workflows/starting-workflows.md @@ -3,7 +3,7 @@ Workflow executions can be started by using the following API: ```http -POST /api/workflow/{name} +POST {{ api_prefix }}/workflow/{name} ``` `{name}` is the placeholder for workflow name. The POST API body is your workflow input parameters which can be empty if @@ -17,7 +17,7 @@ Refer to the SDK documentation to configure a client in your selected language t ### Example using curl ```bash -curl 'https://localhost:8080/api/workflow/sample_workflow' \ +curl '{{ server_host }}{{ api_prefix }}/workflow/sample_workflow' \ -H 'accept: text/plain' \ -H 'content-type: application/json' \ --data-raw '{"service":"fedex"}' @@ -29,7 +29,7 @@ is `sample_workflow` ### Example using node fetch ```javascript -fetch("https://localhost:8080/api/workflow/sample_workflow", { +fetch("{{ server_host }}{{ api_prefix }}/workflow/sample_workflow", { "headers": { "accept": "text/plain", "content-type": "application/json", diff --git a/docs/docs/how-tos/Workflows/updating-workflows.md b/docs/devguide/how-tos/Workflows/updating-workflows.md similarity index 84% rename from docs/docs/how-tos/Workflows/updating-workflows.md rename to docs/devguide/how-tos/Workflows/updating-workflows.md index beb67a404..245feeed2 100644 --- a/docs/docs/how-tos/Workflows/updating-workflows.md +++ b/docs/devguide/how-tos/Workflows/updating-workflows.md @@ -3,13 +3,13 @@ Workflows can be created or updated using the workflow metadata API ```html -PUT /api/metadata/workflow +PUT {{ api_prefix }}/metadata/workflow ``` ### Example using curl ```shell -curl 'http://localhost:8080/api/metadata/workflow' \ +curl '{{ server_host }}{{ api_prefix }}/metadata/workflow' \ -X 'PUT' \ -H 'accept: */*' \ -H 'content-type: application/json' \ @@ -19,7 +19,7 @@ curl 'http://localhost:8080/api/metadata/workflow' \ ### Example using node fetch ```javascript -fetch("http://localhost:8080/api/metadata/workflow", { +fetch("{{ server_host }}{{ api_prefix }}/metadata/workflow", { "headers": { "accept": "*/*", "content-type": "application/json" diff --git a/docs/docs/how-tos/Workflows/versioning-workflows.md b/docs/devguide/how-tos/Workflows/versioning-workflows.md similarity index 97% rename from docs/docs/how-tos/Workflows/versioning-workflows.md rename to docs/devguide/how-tos/Workflows/versioning-workflows.md index e44485176..14c9ea93b 100644 --- a/docs/docs/how-tos/Workflows/versioning-workflows.md +++ b/docs/devguide/how-tos/Workflows/versioning-workflows.md @@ -1,7 +1,3 @@ ---- -sidebar_position: 1 ---- - # Versioning Workflows Every workflow has a version number (this number **must** be an integer.) diff --git a/docs/docs/how-tos/Workflows/view-workflow-executions.md b/docs/devguide/how-tos/Workflows/view-workflow-executions.md similarity index 84% rename from docs/docs/how-tos/Workflows/view-workflow-executions.md rename to docs/devguide/how-tos/Workflows/view-workflow-executions.md index 445fc9b29..09f912978 100644 --- a/docs/docs/how-tos/Workflows/view-workflow-executions.md +++ b/docs/devguide/how-tos/Workflows/view-workflow-executions.md @@ -1,22 +1,10 @@ ---- -sidebar_position: 1 ---- - # View Workflow Executions In this article we will learn how to view workflow executions via the UI. -### Prerequisites - -1. Conductor app and UI installed and running in an environment. If required we can look at the following options to get - an environment up and running. - - 1. [Build and Run Conductor Locally](/gettingstarted/local.html) - 2. [Running via Docker Compose](/gettingstarted/docker.html) - ### Viewing a Workflow Execution -Refer to [Searching Workflows](/how-tos/Workflows/searching-workflows.html) to filter and find an execution you want to +Refer to [Searching Workflows](searching-workflows.md) to filter and find an execution you want to view. Click on the workflow id hyperlink to open the Workflow Execution Details page. The following tabs are available to view the details of the Workflow Execution @@ -50,8 +38,8 @@ opens a flyout panel from the side and contains the following tabs. An exciting feature of conductor is the ability to see the exact execution path of a workflow. The executed paths are shown in green and is easy to follow like the example below. The alternative paths are greyed out for reference -![Conductor UI - Workflow Run](/img/tutorial/workflow_execution_view.png) +![Conductor UI - Workflow Run](workflow_execution_view.png) Errors will be visible on the UI in ref such as the example below -![Conductor UI - Failed Task](/img/tutorial/workflow_task_fail.png) +![Conductor UI - Failed Task](workflow_task_fail.png) diff --git a/docs/docs/img/tutorial/workflow_debugging.png b/docs/devguide/how-tos/Workflows/workflow_debugging.png similarity index 100% rename from docs/docs/img/tutorial/workflow_debugging.png rename to docs/devguide/how-tos/Workflows/workflow_debugging.png diff --git a/docs/docs/img/tutorial/workflow_execution_view.png b/docs/devguide/how-tos/Workflows/workflow_execution_view.png similarity index 100% rename from docs/docs/img/tutorial/workflow_execution_view.png rename to docs/devguide/how-tos/Workflows/workflow_execution_view.png diff --git a/docs/docs/img/tutorial/workflow_task_fail.png b/docs/devguide/how-tos/Workflows/workflow_task_fail.png similarity index 100% rename from docs/docs/img/tutorial/workflow_task_fail.png rename to docs/devguide/how-tos/Workflows/workflow_task_fail.png diff --git a/docs/docs/labs/eventhandlers.md b/docs/devguide/labs/eventhandlers.md similarity index 89% rename from docs/docs/labs/eventhandlers.md rename to docs/devguide/labs/eventhandlers.md index 16130e355..3219f3b5e 100644 --- a/docs/docs/labs/eventhandlers.md +++ b/docs/devguide/labs/eventhandlers.md @@ -1,21 +1,16 @@ # Events and Event Handlers -## About -In this Lab, we shall: +In this exercise, we shall: * Publish an Event to Conductor using `Event` task. * Subscribe to Events, and perform actions: * Start a Workflow * Complete Task -Conductor Supports Eventing with two Interfaces: +Conductor supports eventing with two Interfaces: -* [Event Task](/configuration/systask.html#event) -* [Event Handlers](/configuration/eventhandlers.html#event-handler) - -We shall create a simple cyclic workflow similar to this: - -![img](img/EventHandlerCycle.png) +* [Event Task](../../documentation/configuration/workflowdef/systemtasks/event-task.md) +* [Event Handlers](../../documentation/configuration/eventhandlers.md) ## Create Workflow Definitions @@ -43,8 +38,7 @@ Send `POST` requests to `/metadata/workflow` endpoint with below payloads: "taskReferenceName": "test_task_tobe_completed_by_eventHandler", "type": "WAIT" } - ], - "ownerEmail": "example@email.com" + ] } ``` @@ -63,8 +57,7 @@ Send `POST` requests to `/metadata/workflow` endpoint with below payloads: "type": "EVENT", "sink": "conductor" } - ], - "ownerEmail": "example@email.com" + ] } ``` @@ -73,7 +66,7 @@ Send `POST` requests to `/metadata/workflow` endpoint with below payloads: `EVENT` task is a System task, and we shall define it just like other Tasks in Workflow, with `sink` parameter. Also, `EVENT` task doesn't have to be registered before using in Workflow. This is also true for the `WAIT` task. Hence, we will not be registering any tasks for these workflows. -## Events are sent, but they're not handled (yet) +### Events are sent, but they're not handled (yet) Once you try to start `test_workflow_for_eventHandler` workflow, you would notice that the event is sent successfully, but the second worflow `test_workflow_startedBy_eventHandler` is not started. We have sent the Events, but we also need to define `Event Handlers` for Conductor to take any `actions` based on the Event. Let's create `Event Handlers`. @@ -117,7 +110,7 @@ Event Handler can perform a list of actions defined in `actions` array parameter } ``` -Let's define `start_workflow` action. We shall pass the name of workflow we would like to start. The `start_workflow` parameter can use any of the values from the general [Start Workflow Request](/gettingstarted/startworkflow.html). Here we are passing in the workflowId, so that the Complete Task Event Handler can use it. +Let's define `start_workflow` action. We shall pass the name of workflow we would like to start. The `start_workflow` parameter can use any of the values from the general [Start Workflow Request](../../documentation/api/startworkflow.md). Here we are passing in the workflowId, so that the Complete Task Event Handler can use it. ```json { @@ -171,7 +164,7 @@ Similarly, create another Event Handler to complete task. } ``` -## Final flow of Workflow +## Summary After wiring all of the above, starting the `test_workflow_for_eventHandler` should: diff --git a/docs/docs/labs/running-first-workflow.md b/docs/devguide/labs/first-workflow.md similarity index 75% rename from docs/docs/labs/running-first-workflow.md rename to docs/devguide/labs/first-workflow.md index 1abdc3787..a51aa209d 100644 --- a/docs/docs/labs/running-first-workflow.md +++ b/docs/devguide/labs/first-workflow.md @@ -4,16 +4,9 @@ In this article we will explore how we can run a really simple workflow that run Conductor can orchestrate HTTP services out of the box without implementing any code. We will use that to create and run the first workflow. -See [System Task](/configuration/systask.html) for the list of such built-in tasks. +See [System Task](../../documentation/configuration/workflowdef/systemtasks/index.md) for the list of such built-in tasks. Using system tasks is a great way to run a lot of our code in production. -To bring up a local instance of Conductor follow one of the recommended steps: - -1. [Running Locally - From Code](/gettingstarted/local.html) -2. [Running Locally - Docker Compose](/gettingstarted/docker.html) - ---- - ## Configuring our First Workflow This is a sample workflow that we can leverage for our test. @@ -100,30 +93,30 @@ Ok, now that we have walked through our workflow details, let's run this and see To configure the workflow, head over to the swagger API of conductor server and access the metadata workflow create API: -[http://localhost:8080/swagger-ui/index.html?configUrl=/api-docs/swagger-config#/metadata-resource/create](http://localhost:8080/swagger-ui/index.html?configUrl=/api-docs/swagger-config#/metadata-resource/create) +[http://{{ server_host }}/swagger-ui/index.html?configUrl=/api-docs/swagger-config#/metadata-resource/create](http://{{ server_host }}/swagger-ui/index.html?configUrl=/api-docs/swagger-config#/metadata-resource/create) If the link doesn’t open the right Swagger section, we can navigate to Metadata-Resource -→ `POST /api/metadata/workflow` +→ `POST {{ api_prefix }}/metadata/workflow` -![Swagger UI - Metadata - Workflow](/img/tutorial/metadataWorkflowPost.png) +![Swagger UI - Metadata - Workflow](metadataWorkflowPost.png) Paste the workflow payload into the Swagger API and hit Execute. Now if we head over to the UI, we can see this workflow definition created: -![Conductor UI - Workflow Definition](/img/tutorial/uiWorkflowDefinition.png) +![Conductor UI - Workflow Definition](uiWorkflowDefinition.png) If we click through we can see a visual representation of the workflow: -![Conductor UI - Workflow Definition - Visual Flow](/img/tutorial/uiWorkflowDefinitionVisual.png) +![Conductor UI - Workflow Definition - Visual Flow](uiWorkflowDefinitionVisual.png) -## 2. Running our First Workflow +## Running our First Workflow Let’s run this workflow. To do that we can use the swagger API under the workflow-resources -[http://localhost:8080/swagger-ui/index.html?configUrl=/api-docs/swagger-config#/workflow-resource/startWorkflow_1](http://localhost:8080/swagger-ui/index.html?configUrl=/api-docs/swagger-config#/workflow-resource/startWorkflow_1) +[http://{{ server_host }}/swagger-ui/index.html?configUrl=/api-docs/swagger-config#/workflow-resource/startWorkflow_1](http://{{ server_host }}/swagger-ui/index.html?configUrl=/api-docs/swagger-config#/workflow-resource/startWorkflow_1) -![Swagger UI - Metadata - Workflow - Run](/img/tutorial/metadataWorkflowRun.png) +![Swagger UI - Metadata - Workflow - Run](metadataWorkflowRun.png) Hit **Execute**! @@ -131,7 +124,7 @@ Conductor will return a workflow id. We will need to use this id to load this up search enabled we wouldn't need to copy this. If we don't have search enabled (using Elasticsearch) copy it from the Swagger UI. -![Swagger UI - Metadata - Workflow - Run](/img/tutorial/workflowRunIdCopy.png) +![Swagger UI - Metadata - Workflow - Run](workflowRunIdCopy.png) Ok, we should see this running and get completed soon. Let’s go to the UI to see what happened. @@ -144,16 +137,12 @@ http://localhost:5000/execution/ Replace `` with our workflow id from the previous step. We should see a screen like below. Click on the different tabs to see all inputs and outputs and task list etc. Explore away! -![Conductor UI - Workflow Run](/img/tutorial/workflowLoaded.png) +![Conductor UI - Workflow Run](workflowLoaded.png) ## Summary -In this blog post — we learned how to run a sample workflow in our Conductor installation. Concepts we touched on: +In this article — we learned how to run a sample workflow in our Conductor installation. Concepts we touched on: 1. Workflow creation 2. System tasks such as HTTP 3. Running a workflow via API - -Thank you for reading, and we hope you found this helpful. Please feel free to reach out to us for any questions and we -are happy to help in any way we can. - diff --git a/docs/docs/labs/img/EventHandlerCycle.png b/docs/devguide/labs/img/EventHandlerCycle.png similarity index 100% rename from docs/docs/labs/img/EventHandlerCycle.png rename to docs/devguide/labs/img/EventHandlerCycle.png diff --git a/docs/docs/labs/img/bgnr_complete_workflow.png b/docs/devguide/labs/img/bgnr_complete_workflow.png similarity index 100% rename from docs/docs/labs/img/bgnr_complete_workflow.png rename to docs/devguide/labs/img/bgnr_complete_workflow.png diff --git a/docs/docs/labs/img/bgnr_state_scheduled.png b/docs/devguide/labs/img/bgnr_state_scheduled.png similarity index 100% rename from docs/docs/labs/img/bgnr_state_scheduled.png rename to docs/devguide/labs/img/bgnr_state_scheduled.png diff --git a/docs/docs/labs/img/bgnr_systask_state.png b/docs/devguide/labs/img/bgnr_systask_state.png similarity index 100% rename from docs/docs/labs/img/bgnr_systask_state.png rename to docs/devguide/labs/img/bgnr_systask_state.png diff --git a/docs/devguide/labs/index.md b/docs/devguide/labs/index.md new file mode 100644 index 000000000..99a15fe28 --- /dev/null +++ b/docs/devguide/labs/index.md @@ -0,0 +1,23 @@ +# Guided Tutorial + +## High Level Steps +Generally, these are the steps necessary in order to put Conductor to work for your business workflow: + +1. Create task worker(s) that poll for scheduled tasks at regular interval +2. Create task definitions for these workers and register them. +3. Create the workflow definition + +## Before We Begin +Ensure you have a Conductor instance up and running. This includes both the Server and the UI. We recommend following the [Docker Instructions](../running/docker.md). + +## Tools +For the purpose of testing and issuing API calls, the following tools are useful + +- Linux cURL command +- [Postman](https://www.postman.com) or similar REST client + +## Let's Go +We will begin by defining a simple workflow that utilizes System Tasks. + +[Next](first-workflow.md) + diff --git a/docs/docs/labs/kitchensink.md b/docs/devguide/labs/kitchensink.md similarity index 91% rename from docs/docs/labs/kitchensink.md rename to docs/devguide/labs/kitchensink.md index d3f7f403f..0173851bd 100644 --- a/docs/docs/labs/kitchensink.md +++ b/docs/devguide/labs/kitchensink.md @@ -163,18 +163,19 @@ An example kitchensink workflow that demonstrates the usage of all the schema co } ``` ### Visual Flow -![img](/img/kitchensink.png) +![img](kitchensink.png) ### Running Kitchensink Workflow -1. Start the server as documented [here](/gettingstarted/docker.html). Use ```-DloadSample=true``` java system property when launching the server. This will create a kitchensink workflow, related task definitions and kick off an instance of kitchensink workflow. -2. Once the workflow has started, the first task remains in the ```SCHEDULED``` state. This is because no workers are currently polling for the task. +1. If you are running Conductor locally, use the `-DloadSample=true` Java system property when launching the server. This will create a kitchensink workflow, +related task definitions and kick off an instance of kitchensink workflow. Otherwise, you can create a new Workflow Definition in the UI by copying the sample above. +2. Once the workflow has started, the first task remains in the `SCHEDULED` state. This is because no workers are currently polling for the task. 3. We will use the REST endpoints directly to poll for tasks and updating the status. #### Start workflow execution Start the execution of the kitchensink workflow by posting the following: ```shell -curl -X POST --header 'Content-Type: application/json' --header 'Accept: text/plain' 'http://localhost:8080/api/workflow/kitchensink' -d ' +curl -X POST --header 'Content-Type: application/json' --header 'Accept: text/plain' '{{ server_host }}{{ api_prefix }}/workflow/kitchensink' -d ' { "task2Name": "task_5" } @@ -185,7 +186,7 @@ The response is a text string identifying the workflow instance id. #### Poll for the first task: ```shell -curl http://localhost:8080/api/tasks/poll/task_1 +curl {{ server_host }}{{ api_prefix }}/tasks/poll/task_1 ``` The response should look something like: @@ -223,7 +224,7 @@ The response should look something like: * Update the status of the task as ```COMPLETED``` as below: ```json -curl -H 'Content-Type:application/json' -H 'Accept:application/json' -X POST http://localhost:8080/api/tasks/ -d ' +curl -H 'Content-Type:application/json' -H 'Accept:application/json' -X POST {{ server_host }}{{ api_prefix }}/tasks/ -d ' { "taskId": "b9eea7dd-3fbd-46b9-a9ff-b00279459476", "workflowInstanceId": "b0d1a935-3d74-46fd-92b2-0ca1e388659f", diff --git a/docs/docs/img/kitchensink.png b/docs/devguide/labs/kitchensink.png similarity index 100% rename from docs/docs/img/kitchensink.png rename to docs/devguide/labs/kitchensink.png diff --git a/docs/docs/img/tutorial/metadataWorkflowPost.png b/docs/devguide/labs/metadataWorkflowPost.png similarity index 100% rename from docs/docs/img/tutorial/metadataWorkflowPost.png rename to docs/devguide/labs/metadataWorkflowPost.png diff --git a/docs/docs/img/tutorial/metadataWorkflowRun.png b/docs/devguide/labs/metadataWorkflowRun.png similarity index 100% rename from docs/docs/img/tutorial/metadataWorkflowRun.png rename to docs/devguide/labs/metadataWorkflowRun.png diff --git a/docs/docs/img/tutorial/uiWorkflowDefinition.png b/docs/devguide/labs/uiWorkflowDefinition.png similarity index 100% rename from docs/docs/img/tutorial/uiWorkflowDefinition.png rename to docs/devguide/labs/uiWorkflowDefinition.png diff --git a/docs/docs/img/tutorial/uiWorkflowDefinitionVisual.png b/docs/devguide/labs/uiWorkflowDefinitionVisual.png similarity index 100% rename from docs/docs/img/tutorial/uiWorkflowDefinitionVisual.png rename to docs/devguide/labs/uiWorkflowDefinitionVisual.png diff --git a/docs/docs/img/tutorial/workflowLoaded.png b/docs/devguide/labs/workflowLoaded.png similarity index 100% rename from docs/docs/img/tutorial/workflowLoaded.png rename to docs/devguide/labs/workflowLoaded.png diff --git a/docs/docs/img/tutorial/workflowRunIdCopy.png b/docs/devguide/labs/workflowRunIdCopy.png similarity index 100% rename from docs/docs/img/tutorial/workflowRunIdCopy.png rename to docs/devguide/labs/workflowRunIdCopy.png diff --git a/docs/docs/img/tutorial/conductorUI.png b/docs/devguide/running/conductorUI.png similarity index 100% rename from docs/docs/img/tutorial/conductorUI.png rename to docs/devguide/running/conductorUI.png diff --git a/docs/docs/gettingstarted/docker.md b/docs/devguide/running/docker.md similarity index 69% rename from docs/docs/gettingstarted/docker.md rename to docs/devguide/running/docker.md index 483a57037..5cb2d7c69 100644 --- a/docs/docs/gettingstarted/docker.md +++ b/docs/devguide/running/docker.md @@ -1,5 +1,5 @@ -# Running Conductor using Docker +# Running Conductor Using Docker In this article we will explore how you can set up Netflix Conductor on your local machine using Docker compose. The docker compose will bring up the following: @@ -43,14 +43,14 @@ Once up and running, you will see the following in your Docker dashboard: You can access the UI & Server on your browser to verify that they are running correctly: #### Conductor Server URL -[http://localhost:8080](http://localhost:8080) +[{{ server_host }}]({{ server_host }}) - +![swagger](swagger.png) #### Conductor UI URL [http://localhost:5000/](http://localhost:5000) - +![conductor ui](conductorUI.png) ### 4. Exiting Compose @@ -81,7 +81,7 @@ To build and run the server image, without using `docker-compose`, from the `doc docker build -t conductor:server -f server/Dockerfile ../ docker run -p 8080:8080 -d --name conductor_server conductor:server ``` -This builds the image `conductor:server` and runs it in a container named `conductor_server`. The API should now be accessible at `localhost:8080`. +This builds the image `conductor:server` and runs it in a container named `conductor_server`. The API should now be accessible at `{{ server_host }}`. To 'login' to the running container, use the command: ``` @@ -120,56 +120,55 @@ docker build -t conductor:serverAndUI -f serverAndUI/Dockerfile ../ - With interal DB: `docker run -p 8080:8080 -p 80:5000 -d -t conductor:serverAndUI` - With external DB: `docker run -p 8080:8080 -p 80:5000 -d -t -e "CONFIG_PROP=config.properties" conductor:serverAndUI` +## Elasticsearch +Elasticsearch is optional, please be aware that disable it will make most of the conductor UI not functional. +### How to enable Elasticsearch +* Set `conductor.indexing.enabled=true` in your_config.properties +* Add config related to elasticsearch + E.g.: `conductor.elasticsearch.url=http://es:9200` -## Potential problem when using Docker Images - -#### Not enough memory - - 1. You will need at least 16 GB of memory to run everything. You can modify the docker compose to skip using - Elasticsearch if you have no option to run this with your memory options. - 2. To disable Elasticsearch using Docker Compose - follow the steps listed here: **TODO LINK** +### How to disable Elasticsearch +* Set `conductor.indexing.enabled=false` in your_config.properties +* Comment out all the config related to elasticsearch +E.g.: `conductor.elasticsearch.url=http://es:9200` -#### Elasticsearch fails to come up in arm64 based CPU machines - 1. As of writing this article, Conductor relies on 6.8.x version of Elasticsearch. This version doesn't have an - arm64 based Docker image. You will need to use Elasticsearch 7.x which requires a bit of customization to get up - and running +## Troubleshooting +To troubleshoot a failed startup, check the server logss located at `/app/logs` (default directory in dockerfile) -#### Elasticsearch remains in yellow health state +## Potential problem when using Docker Images -When you run Elasticsearch, sometimes the health remains in the *yellow* state. Conductor server by default requires -*green* state to run when indexing is enabled. To work around this, you can use the following property: `conductor.elasticsearch.clusterHealthColor=yellow`. +### Not enough memory +You will need at least 16 GB of memory to run everything. You can modify the docker compose to skip using +Elasticsearch if you have no option to run this with your memory options. -Reference: [Issue 2262][issue2262] +To disable Elasticsearch using Docker Compose - follow the steps above. -#### Elasticsearch timeout +### Elasticsearch fails to come up in arm64 based CPU machines +As of writing this article, Conductor relies on 6.8.x version of Elasticsearch. This version doesn't have an +arm64 based Docker image. You will need to use Elasticsearch 7.x which requires a bit of customization to get up +and running -By default, a standalone (single node) Elasticsearch has a *yellow* status which will cause timeout (`java.net.SocketTimeoutException`) for Conductor server (required status is *green*). -Spin up a cluster (more than one node) to prevent the timeout or use config option `conductor.elasticsearch.clusterHealthColor=yellow`. +### Elasticsearch remains in Yellow health -Reference: [Issue 2262][issue2262] +When you run Elasticsearch, sometimes the health remains in Yellow state. Conductor server by default requires +Green state to run when indexing is enabled. To work around this, you can use the following property: +`conductor.elasticsearch.clusterHealthColor=yellow` -#### Changes in config-*.properties do not take effect -Config is copy into image during docker build. You have to rebuild the image or better, link a volume to it to reflect new changes. +See Github [issue](https://github.com/Netflix/conductor/issues/2262) -#### To troubleshoot a failed startup -Check the log of the server, which is located at `/app/logs` (default directory in dockerfile) +### Elasticsearch timeout +Symptom: Standalone (single node) Elasticsearch has a yellow status which will cause timeout for the Conductor server at startup (Required: Green). -#### Unable to access to conductor:server API on port 8080 -It may takes some time for conductor server to start. Please check server log for potential error. +Solution: Spin up a cluster (more than one node) to prevent the timeout or use config option `conductor.elasticsearch.clusterHealthColor=yellow`. -#### Elasticsearch -Elasticsearch is optional, please be aware that disable it will make most of the conductor UI not functional. +See Github [issue](https://github.com/Netflix/conductor/issues/2262) -##### How to enable Elasticsearch -* Set `conductor.indexing.enabled=true` in your_config.properties -* Add config related to elasticsearch - E.g.: `conductor.elasticsearch.url=http://es:9200` +### Changes in config-*.properties do not take effect +Config is copied into the image during the docker build. You have to rebuild the image or better, link a volume to it +to reflect new changes automatically. -##### How to disable Elasticsearch -* Set `conductor.indexing.enabled=false` in your_config.properties -* Comment out all the config related to elasticsearch -E.g.: `conductor.elasticsearch.url=http://es:9200` +### Unable to access to conductor:server API on port 8080 +It may takes some time for conductor server to start. Please check server log for errors. -[issue2262]: https://github.com/Netflix/conductor/issues/2262 diff --git a/docs/docs/gettingstarted/hosted.md b/docs/devguide/running/hosted.md similarity index 100% rename from docs/docs/gettingstarted/hosted.md rename to docs/devguide/running/hosted.md diff --git a/docs/docs/gettingstarted/source.md b/docs/devguide/running/source.md similarity index 85% rename from docs/docs/gettingstarted/source.md rename to docs/devguide/running/source.md index f3a478656..97a0f4a98 100644 --- a/docs/docs/gettingstarted/source.md +++ b/docs/devguide/running/source.md @@ -5,7 +5,7 @@ In this article we will explore how you can set up Netflix Conductor on your loc features. ### Prerequisites -1. JDK 17 or greater +1. JDK 11 or greater 2. (Optional) Docker if you want to run tests. You can install docker from [here](https://www.docker.com/get-started/). 3. Node for building and running UI. Instructions at [https://nodejs.org](https://nodejs.org). 4. Yarn for building and running UI. Instructions at [https://classic.yarnpkg.com/en/docs/install](https://classic.yarnpkg.com/en/docs/install). @@ -48,9 +48,9 @@ server $ ../gradlew bootRun ``` Navigate to the swagger API docs: -[http://localhost:8080/swagger-ui/index.html?configUrl=/api-docs/swagger-config](http://localhost:8080/swagger-ui/index.html?configUrl=/api-docs/swagger-config) +[http://{{ server_host }}/swagger-ui/index.html?configUrl=/api-docs/swagger-config](http://{{ server_host }}/swagger-ui/index.html?configUrl=/api-docs/swagger-config) - +![swagger](swagger.png) ## Download and Run As an alternative to building from source, you can download and run the pre-compiled JAR. @@ -61,7 +61,7 @@ export REPO_URL=https://repo1.maven.org/maven2/com/netflix/conductor/conductor-s curl $REPO_URL/$CONDUCTOR_VER/conductor-server-$CONDUCTOR_VER-boot.jar \ --output conductor-server-$CONDUCTOR_VER-boot.jar; java -jar conductor-server-$CONDUCTOR_VER-boot.jar ``` -Navigate to the swagger URL: [http://localhost:8080/swagger-ui/index.html?configUrl=/api-docs/swagger-config](http://localhost:8080/swagger-ui/index.html?configUrl=/api-docs/swagger-config) +Navigate to the swagger URL: [http://{{ server_host }}/swagger-ui/index.html?configUrl=/api-docs/swagger-config](http://{{ server_host }}/swagger-ui/index.html?configUrl=/api-docs/swagger-config) @@ -87,7 +87,7 @@ ui $ yarn run start Launch UI [http://localhost:5000](http://localhost:5000) - +![conductor ui](conductorUI.png) ## Summary 1. By default in-memory persistence is used, so any workflows created or excuted will be wiped out once the server is terminated. diff --git a/docs/docs/img/tutorial/swagger.png b/docs/devguide/running/swagger.png similarity index 100% rename from docs/docs/img/tutorial/swagger.png rename to docs/devguide/running/swagger.png diff --git a/docs/docs/apispec.md b/docs/docs/apispec.md deleted file mode 100644 index 94df57fca..000000000 --- a/docs/docs/apispec.md +++ /dev/null @@ -1,173 +0,0 @@ -# API Specification - -## Task & Workflow Metadata -| Endpoint | Description | Input | -|------------------------------------------|:---------------------------------|-------------------------------------------------------------| -| `GET /metadata/taskdefs` | Get all the task definitions | n/a | -| `GET /metadata/taskdefs/{taskType}` | Retrieve task definition | Task Name | -| `POST /metadata/taskdefs` | Register new task definitions | List of [Task Definitions](/configuration/taskdef.html) | -| `PUT /metadata/taskdefs` | Update a task definition | A [Task Definition](/configuration/taskdef.html) | -| `DELETE /metadata/taskdefs/{taskType}` | Delete a task definition | Task Name | -||| -| `GET /metadata/workflow` | Get all the workflow definitions | n/a | -| `POST /metadata/workflow` | Register new workflow | [Workflow Definition](/configuration/workflowdef.html) | -| `PUT /metadata/workflow` | Register/Update new workflows | List of [Workflow Definition](/configuration/workflowdef.html) | -| `GET /metadata/workflow/{name}?version=` | Get the workflow definitions | workflow name, version (optional) | -||| - -## Start A Workflow -### With Input only -See [Start Workflow Request](/gettingstarted/startworkflow.html). - -#### Output -Id of the workflow (GUID) - -### With Input and Task Domains -``` -POST /workflow -{ - //JSON payload for Start workflow request -} -``` -#### Start workflow request -JSON for start workflow request -``` -{ - "name": "myWorkflow", // Name of the workflow - "version": 1, // Version - “correlationId”: “corr1”, // correlation Id - "priority": 1, // Priority - "input": { - // Input map. - }, - "taskToDomain": { - // Task to domain map - } -} -``` - -#### Output -Id of the workflow (GUID) - - -## Retrieve Workflows -| Endpoint | Description | -|-----------------------------------------------------------------------------|-----------------------------------------------| -| `GET /workflow/{workflowId}?includeTasks=true | false` |Get Workflow State by workflow Id. If includeTasks is set, then also includes all the tasks executed and scheduled.| -| `GET /workflow/running/{name}` | Get all the running workflows of a given type | -| `GET /workflow/running/{name}/correlated/{correlationId}?includeClosed=true | false&includeTasks=true |false`|Get all the running workflows filtered by correlation Id. If includeClosed is set, also includes workflows that have completed running.| -| `GET /workflow/search` | Search for workflows. See Below. | - - -## Search for Workflows -Conductor uses Elasticsearch for indexing workflow execution and is used by search APIs. - -`GET /workflow/search?start=&size=&sort=&freeText=&query=` - -| Parameter | Description | -|-----------|------------------------------------------------------------------------------------------------------------------| -| start | Page number. Defaults to 0 | -| size | Number of results to return | -| sort | Sorting. Format is: `ASC:` or `DESC:` to sort in ascending or descending order by a field | -| freeText | Elasticsearch supported query. e.g. workflowType:"name_of_workflow" | -| query | SQL like where clause. e.g. workflowType = 'name_of_workflow'. Optional if freeText is provided. | - -### Output -Search result as described below: -```json -{ - "totalHits": 0, - "results": [ - { - "workflowType": "string", - "version": 0, - "workflowId": "string", - "correlationId": "string", - "startTime": "string", - "updateTime": "string", - "endTime": "string", - "status": "RUNNING", - "input": "string", - "output": "string", - "reasonForIncompletion": "string", - "executionTime": 0, - "event": "string" - } - ] -} -``` - -## Manage Workflows -| Endpoint | Description | -|-----------------------------------------------------------|----------------------------------------------------------------------------------------------------| -| `PUT /workflow/{workflowId}/pause` | Pause. No further tasks will be scheduled until resumed. Currently running tasks are not paused. | -| `PUT /workflow/{workflowId}/resume` | Resume normal operations after a pause. | -| `POST /workflow/{workflowId}/rerun` | See Below. | -| `POST /workflow/{workflowId}/restart` | Restart workflow execution from the start. Current execution history is wiped out. | -| `POST /workflow/{workflowId}/retry` | Retry the last failed task. | -| `PUT /workflow/{workflowId}/skiptask/{taskReferenceName}` | See below. | -| `DELETE /workflow/{workflowId}` | Terminates the running workflow. | -| `DELETE /workflow/{workflowId}/remove` | Deletes the workflow from system. Use with caution. | - -### Rerun -Re-runs a completed workflow from a specific task. - -`POST /workflow/{workflowId}/rerun` - -```json -{ - "reRunFromWorkflowId": "string", - "workflowInput": {}, - "reRunFromTaskId": "string", - "taskInput": {} -} -``` - -###Skip Task - -Skips a task execution (specified as `taskReferenceName` parameter) in a running workflow and continues forward. -Optionally updating task's input and output as specified in the payload. -`PUT /workflow/{workflowId}/skiptask/{taskReferenceName}?workflowId=&taskReferenceName=` -```json -{ - "taskInput": {}, - "taskOutput": {} -} -``` - -## Manage Tasks -| Endpoint | Description | -|-------------------------------------------------------|-------------------------------------------------------| -| `GET /tasks/{taskId}` | Get task details. | -| `GET /tasks/queue/all` | List the pending task sizes. | -| `GET /tasks/queue/all/verbose` | Same as above, includes the size per shard | -| `GET /tasks/queue/sizes?taskType=&taskType=&taskType` | Return the size of pending tasks for given task types | -||| - -## Polling, Ack and Update Task -These are critical endpoints used to poll for task, send ack (after polling) and finally updating the task result by worker. - - -| Endpoint | Description | -|---------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `GET /tasks/poll/{taskType}?workerid=&domain=` | Poll for a task. `workerid` identifies the worker that polled for the job and `domain` allows the poller to poll for a task in a specific domain | -| `GET /tasks/poll/batch/{taskType}?count=&timeout=&workerid=&domain` | Poll for a task in a batch specified by `count`. This is a long poll and the connection will wait until `timeout` or if there is at-least 1 item available, whichever comes first.`workerid` identifies the worker that polled for the job and `domain` allows the poller to poll for a task in a specific domain | -| `POST /tasks` | Update the result of task execution. See the schema below. | - - -### Schema for updating Task Result -```json -{ - "workflowInstanceId": "Workflow Instance Id", - "taskId": "ID of the task to be updated", - "reasonForIncompletion" : "If failed, reason for failure", - "callbackAfterSeconds": 0, - "status": "IN_PROGRESS|FAILED|COMPLETED", - "outputData": { - //JSON document representing Task execution output - } - -} -``` -!!!Info "Acknowledging tasks after poll" - If the worker fails to ack the task after polling, the task is re-queued and put back in queue and is made available during subsequent poll. diff --git a/docs/docs/configuration/sysoperator.md b/docs/docs/configuration/sysoperator.md deleted file mode 100644 index 181d72fd3..000000000 --- a/docs/docs/configuration/sysoperator.md +++ /dev/null @@ -1,20 +0,0 @@ -# System Operators - -Operators are built-in primitives in Conductor that allow you to define the control flow in the workflow. -Operators are similar to programming constructs such as for loops, decisions, etc. -Conductor has support for most of the programing primitives allowing you to define the most advanced workflows. - -## Supported Operators -Conductor supports the following programming language constructs: - -| Language Construct | Conductor Operator | -|----------------------------------|-------------------------------------------------------------| -| Do/While or Loops | [Do While Task](/reference-docs/do-while-task.html) | -| Dynamic Fork | [Dynamic Fork Task](/reference-docs/dynamic-fork-task.html) | -| Fork / Parallel execution | [Fork Task](/reference-docs/fork-task.html) | -| Join | [Join Task](/reference-docs/join-task.html) | -| Sub Process / Sub-Flow | [Sub Workflow Task](/reference-docs/sub-workflow-task.html) | -| Switch//Decision/if..then...else | [Switch Task](/reference-docs/switch-task.html) | -| Terminate | [Terminate Task](/reference-docs/terminate-task.html) | -| Variables | [Variable Task](/reference-docs/set-variable-task.html) | -| Wait | [Wait Task](/reference-docs/wait-task.html) | diff --git a/docs/docs/configuration/systask.md b/docs/docs/configuration/systask.md deleted file mode 100644 index 4b2f964a6..000000000 --- a/docs/docs/configuration/systask.md +++ /dev/null @@ -1,256 +0,0 @@ -# System Tasks - -System Tasks (Workers) are built-in tasks that are general purpose and re-usable. They run on the Conductor servers. -Such tasks allow you to get started without having to write custom workers. - -## Available System Tasks - -Conductor has the following set of system tasks available. - -| Task | Description | Use Case | -|-----------------------|--------------------------------------------------------|------------------------------------------------------------------------------------| -| Event Publishing | [Event Task](/reference-docs/event-task.html) | External eventing system integration. e.g. amqp, sqs, nats | -| HTTP | [HTTP Task](/reference-docs/http-task.html) | Invoke any HTTP(S) endpoints | -| Inline Code Execution | [Inline Task](/reference-docs/inline-task.html) | Execute arbitrary lightweight javascript code | -| JQ Transform | [JQ Task](/reference-docs/json-jq-transform-task.html) | Use JQ to transform task input/output | -| Kafka Publish | [Kafka Task](/reference-docs/kafka-publish-task.html) | Publish messages to Kafka | - -| Name | Description | -|--------------------------|-------------------------------------------------------------------------------------------------------------------------------------------| -| joinOn | List of task reference names, which the EXCLUSIVE_JOIN will lookout for to capture output. From above example, this could be ["T2", "T3"] | -| defaultExclusiveJoinTask | Task reference name, whose output should be used incase the decision case is undefined. From above example, this could be ["T1"] | - - -**Example** - -``` json -{ - "name": "exclusive_join", - "taskReferenceName": "exclusiveJoin", - "type": "EXCLUSIVE_JOIN", - "joinOn": [ - "task2", - "task3" - ], - "defaultExclusiveJoinTask": [ - "task1" - ] -} -``` - - -## Wait -A wait task is implemented as a gate that remains in ```IN_PROGRESS``` state unless marked as ```COMPLETED``` or ```FAILED``` by an external trigger. -To use a wait task, set the task type as ```WAIT``` - -**Parameters:** -None required. - -**External Triggers for Wait Task** - -Task Resource endpoint can be used to update the status of a task to a terminate state. - -Contrib module provides SQS integration where an external system can place a message in a pre-configured queue that the server listens on. As the messages arrive, they are marked as ```COMPLETED``` or ```FAILED```. - -**SQS Queues** - -* SQS queues used by the server to update the task status can be retrieve using the following API: -``` -GET /queue -``` -* When updating the status of the task, the message needs to conform to the following spec: - * Message has to be a valid JSON string. - * The message JSON should contain a key named ```externalId``` with the value being a JSONified string that contains the following keys: - * ```workflowId```: Id of the workflow - * ```taskRefName```: Task reference name that should be updated. - * Each queue represents a specific task status and tasks are marked accordingly. e.g. message coming to a ```COMPLETED``` queue marks the task status as ```COMPLETED```. - * Tasks' output is updated with the message. - -**Example SQS Payload:** - -```json -{ - "some_key": "valuex", - "externalId": "{\"taskRefName\":\"TASK_REFERENCE_NAME\",\"workflowId\":\"WORKFLOW_ID\"}" -} -``` - - -## Dynamic Task - -Dynamic Task allows to execute one of the registered Tasks dynamically at run-time. It accepts the task name to execute in inputParameters. - -**Parameters:** - -|name|description| -|---|---| -| dynamicTaskNameParam|Name of the parameter from the task input whose value is used to schedule the task. e.g. if the value of the parameter is ABC, the next task scheduled is of type 'ABC'.| - -**Example** -``` json -{ - "name": "user_task", - "taskReferenceName": "t1", - "inputParameters": { - "files": "${workflow.input.files}", - "taskToExecute": "${workflow.input.user_supplied_task}" - }, - "type": "DYNAMIC", - "dynamicTaskNameParam": "taskToExecute" -} -``` -If the workflow is started with input parameter user_supplied_task's value as __user_task_2__, Conductor will schedule __user_task_2__ when scheduling this dynamic task. - -## Inline Task - -Inline Task helps execute ad-hoc logic at Workflow run-time, using any evaluator engine. Supported evaluators -are `value-param` evaluator which simply translates the input parameter to output and `javascript` evaluator that -evaluates Javascript expression. - -This is particularly helpful in running simple evaluations in Conductor server, over creating Workers. - -**Parameters:** - -|name|type|description|notes| -|---|---|---|---| -|evaluatorType|String|Type of the evaluator. Supported evaluators: `value-param`, `javascript` which evaluates javascript expression.| -|expression|String|Expression associated with the type of evaluator. For `javascript` evaluator, Javascript evaluation engine is used to evaluate expression defined as a string. Must return a value.|Must be non-empty.| - -Besides `expression`, any value is accessible as `$.value` for the `expression` to evaluate. - -**Outputs:** - -|name|type|description| -|---|---|---| -|result|Map|Contains the output returned by the evaluator based on the `expression`| - -The task output can then be referenced in downstream tasks like: -```"${inline_test.output.result.testvalue}"``` - -**Example** -``` json -{ - "name": "INLINE_TASK", - "taskReferenceName": "inline_test", - "type": "INLINE", - "inputParameters": { - "inlineValue": "${workflow.input.inlineValue}", - "evaluatorType": "javascript", - "expression": "function scriptFun(){if ($.inlineValue == 1){ return {testvalue: true} } else { return - {testvalue: false} }} scriptFun();" - } -} -``` - -## Terminate Task - -Task that can terminate a workflow with a given status and modify the workflow's output with a given parameter. It can act as a "return" statement for conditions where you simply want to terminate your workflow. - -For example, if you have a decision where the first condition is met, you want to execute some tasks, otherwise you want to finish your workflow. - -**Parameters:** - -|name|type| description | notes | -|---|---|---------------------------------------------------|-------------------------| -|terminationStatus|String| can only accept "COMPLETED" or "FAILED" | task cannot be optional | - |terminationReason|String| reason for incompletion to be set in the workflow | optional | -|workflowOutput|Any| Expected workflow output | optional | - -**Outputs:** - -|name|type|description| -|---|---|---| -|output|Map|The content of `workflowOutput` from the inputParameters. An empty object if `workflowOutput` is not set.| - -```json -{ - "name": "terminate", - "taskReferenceName": "terminate0", - "inputParameters": { - "terminationStatus": "COMPLETED", - "terminationReason": "", - "workflowOutput": "${task0.output}" - }, - "type": "TERMINATE", - "startDelay": 0, - "optional": false -} -``` - - -## Kafka Publish Task - -A kafka Publish task is used to push messages to another microservice via kafka - -**Parameters:** - -The task expects an input parameter named ```kafka_request``` as part of the task's input with the following details: - -|name|description| -|---|---| -| bootStrapServers |bootStrapServers for connecting to given kafka.| -|key|Key to be published| -|keySerializer | Serializer used for serializing the key published to kafka. One of the following can be set :
1. org.apache.kafka.common.serialization.IntegerSerializer
2. org.apache.kafka.common.serialization.LongSerializer
3. org.apache.kafka.common.serialization.StringSerializer.
Default is String serializer | -|value| Value published to kafka| -|requestTimeoutMs| Request timeout while publishing to kafka. If this value is not given the value is read from the property `kafka.publish.request.timeout.ms`. If the property is not set the value defaults to 100 ms | -|maxBlockMs| maxBlockMs while publishing to kafka. If this value is not given the value is read from the property `kafka.publish.max.block.ms`. If the property is not set the value defaults to 500 ms | -|headers|A map of additional kafka headers to be sent along with the request.| -|topic|Topic to publish| - -The producer created in the kafka task is cached. By default the cache size is 10 and expiry time is 120000 ms. To change the defaults following can be modified kafka.publish.producer.cache.size,kafka.publish.producer.cache.time.ms respectively. - -**Kafka Task Output** - -Task status transitions to COMPLETED - -**Example** - -Task sample - -```json -{ - "name": "call_kafka", - "taskReferenceName": "call_kafka", - "inputParameters": { - "kafka_request": { - "topic": "userTopic", - "value": "Message to publish", - "bootStrapServers": "localhost:9092", - "headers": { - "x-Auth":"Auth-key" - }, - "key": "123", - "keySerializer": "org.apache.kafka.common.serialization.IntegerSerializer" - } - }, - "type": "KAFKA_PUBLISH" -} -``` - -The task is marked as ```FAILED``` if the message could not be published to the Kafka queue. - - -## Do While Task - -Sequentially execute a list of task as long as a condition is true. The list of tasks is executed first, before the condition is -checked (even for the first iteration). - -When scheduled, each task of this loop will see its `taskReferenceName` concatenated with `__i`, with `i` being the -iteration number, starting at 1. Warning: `taskReferenceName` containing arithmetic operators must not be used. - -Each task output is stored as part of the `DO_WHILE` task, indexed by the iteration value (see example below), allowing -the condition to reference the output of a task for a specific iteration (eg. ```$.LoopTask['iteration]['first_task']```) - -The `DO_WHILE` task is set to `FAILED` as soon as one of the loopTask fails. In such case retry, iteration starts from 1. - -Limitations: - - Domain or isolation group execution is unsupported; - - Nested `DO_WHILE` is unsupported; - - `SUB_WORKFLOW` is unsupported; - - Since loopover tasks will be executed in loop inside scope of parent do while task, crossing branching outside of DO_WHILE - task is not respected. Branching inside loopover task is supported. - -**Parameters:** - -|name|type|description| - diff --git a/docs/docs/configuration/taskdef.md b/docs/docs/configuration/taskdef.md deleted file mode 100644 index 075912ff4..000000000 --- a/docs/docs/configuration/taskdef.md +++ /dev/null @@ -1,112 +0,0 @@ -# Task Definition - -Tasks are the building blocks of workflow in Conductor. A task can be an operator, system task or custom code written in any programming language. - -A typical Conductor workflow is a list of tasks that are executed until completion or the termination of the workflow. - -Conductor maintains a registry of worker tasks. A task MUST be registered before being used in a workflow. - -**Example** -``` json -{ - "name": "encode_task", - "retryCount": 3, - - "timeoutSeconds": 1200, - "pollTimeoutSeconds": 3600, - "inputKeys": [ - "sourceRequestId", - "qcElementType" - ], - "outputKeys": [ - "state", - "skipped", - "result" - ], - "timeoutPolicy": "TIME_OUT_WF", - "retryLogic": "FIXED", - "retryDelaySeconds": 600, - "responseTimeoutSeconds": 1200, - "concurrentExecLimit": 100, - "rateLimitFrequencyInSeconds": 60, - "rateLimitPerFrequency": 50, - "ownerEmail": "foo@bar.com", - "description": "Sample Encoding task" -} -``` - -| Field | Description | Notes | -|----------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------| -| name | Task Type. Unique name of the Task that resonates with it's function. | Unique | -| description | Description of the task | optional | -| retryCount | No. of retries to attempt when a Task is marked as failure | defaults to 3 | -| retryLogic | Mechanism for the retries | [Retry Logic values](#retry-logic) | -| retryDelaySeconds | Time to wait before retries | defaults to 60 seconds | -| timeoutPolicy | Task's timeout policy | [timeout policy values](#timeout-policy) | -| timeoutSeconds | Time in seconds, after which the task is marked as `TIMED_OUT` if not completed after transitioning to `IN_PROGRESS` status for the first time | No timeouts if set to 0 | -| pollTimeoutSeconds | Time in seconds, after which the task is marked as `TIMED_OUT` if not polled by a worker | No timeouts if set to 0 | -| responseTimeoutSeconds | Must be greater than 0 and less than timeoutSeconds. The task is rescheduled if not updated with a status after this time (heartbeat mechanism). Useful when the worker polls for the task but fails to complete due to errors/network failure. | defaults to 3600 | -| backoffScaleFactor | Must be greater than 0. Scale factor for linearity of the backoff | defaults to 1 | -| inputKeys | Array of keys of task's expected input. Used for documenting task's input. See [Using inputKeys and outputKeys](#using-inputkeys-and-outputkeys). | optional | -| outputKeys | Array of keys of task's expected output. Used for documenting task's output | optional | -| inputTemplate | See [Using inputTemplate](#using-inputtemplate) below. | optional | -| concurrentExecLimit | Number of tasks that can be executed at any given time. | optional | -| rateLimitFrequencyInSeconds, rateLimitPerFrequency | See [Task Rate limits](#task-rate-limits) below. | optional | - - -### Retry Logic - -* FIXED : Reschedule the task after the ```retryDelaySeconds``` -* EXPONENTIAL_BACKOFF : Reschedule after ```retryDelaySeconds 2^(attemptNumber)``` -* LINEAR_BACKOFF : Reschedule after ```retryDelaySeconds * backoffRate * attemptNumber``` - -### Timeout Policy - -* RETRY : Retries the task again -* TIME_OUT_WF : Workflow is marked as TIMED_OUT and terminated -* ALERT_ONLY : Registers a counter (task_timeout) - -### Task Concurrent Execution Limits - -* `concurrentExecLimit` limits the number of simultaneous Task executions at any point. -**Example:** -If you have 1000 task executions waiting in the queue, and 1000 workers polling this queue for tasks, but if you have set `concurrentExecLimit` to 10, only 10 tasks would be given to workers (which would lead to starvation). If any of the workers finishes execution, a new task(s) will be removed from the queue, while still keeping the current execution count to 10. - -### Task Rate limits - -> Note: Rate limiting is only supported for the Redis-persistence module and is not available with other persistence layers. - -* `rateLimitFrequencyInSeconds` and `rateLimitPerFrequency` should be used together. -* `rateLimitFrequencyInSeconds` sets the "frequency window", i.e the `duration` to be used in `events per duration`. Eg: 1s, 5s, 60s, 300s etc. -* `rateLimitPerFrequency`defines the number of Tasks that can be given to Workers per given "frequency window". - -**Example:** -Let's set `rateLimitFrequencyInSeconds = 5`, and `rateLimitPerFrequency = 12`. This means, our frequency window is of 5 seconds duration, and for each frequency window, Conductor would only give 12 tasks to workers. So, in a given minute, Conductor would only give 12*(60/5) = 144 tasks to workers irrespective of the number of workers that are polling for the task. -Note that unlike `concurrentExecLimit`, rate limiting doesn't take into account tasks already in progress/completed. Even if all the previous tasks are executed within 1 sec, or would take a few days, the new tasks are still given to workers at configured frequency, 144 tasks per minute in above example. - - -### Using inputKeys and outputKeys - -* `inputKeys` and `outputKeys` can be considered as parameters and return values for the Task. -* Consider the task Definition as being represented by an interface: ```(value1, value2 .. valueN) someTaskDefinition(key1, key2 .. keyN);``` -* However, these parameters are not strictly enforced at the moment. Both `inputKeys` and `outputKeys` act as a documentation for task re-use. The tasks in workflow need not define all of the keys in the task definition. -* In the future, this can be extended to be a strict template that all task implementations must adhere to, just like interfaces in programming languages. - -### Using inputTemplate - -* `inputTemplate` allows to define default values, which can be overridden by values provided in Workflow. -* Eg: In your Task Definition, you can define your inputTemplate as: - -```json -"inputTemplate": { - "url": "https://some_url:7004" -} -``` - -* Now, in your workflow Definition, when using above task, you can use the default `url` or override with something else in the task's `inputParameters`. - -```json -"inputParameters": { - "url": "${workflow.input.some_new_url}" -} -``` diff --git a/docs/docs/configuration/workerdef.md b/docs/docs/configuration/workerdef.md deleted file mode 100644 index 7081f989f..000000000 --- a/docs/docs/configuration/workerdef.md +++ /dev/null @@ -1,14 +0,0 @@ -# Worker Definition - -A worker is responsible for executing a task. Operator and System tasks are handled by the Conductor server, while user -defined tasks needs to have a worker created that awaits the work to be scheduled by the server for it to be executed. -Workers can be implemented in any language, and Conductor provides support for Java, Golang and Python worker framework that provides features such as -polling threads, metrics and server communication that makes creating workers easy. - -Each worker embodies Microservice design pattern and follows certain basic principles: - -1. Workers are stateless and do not implement a workflow specific logic. -2. Each worker executes a very specific task and produces well defined output given specific inputs. -3. Workers are meant to be idempotent (or should handle cases where the task that partially executed gets rescheduled due to timeouts etc.) -4. Workers do not implement the logic to handle retries etc, that is taken care by the Conductor server. - diff --git a/docs/docs/configuration/workflowdef.md b/docs/docs/configuration/workflowdef.md deleted file mode 100644 index 35e167ac5..000000000 --- a/docs/docs/configuration/workflowdef.md +++ /dev/null @@ -1,229 +0,0 @@ -# Workflow Definition - -## What are Workflows? - -At a high level, a workflow is the Conductor primitive that encompasses the definition and flow of your business logic. -A workflow is a collection (graph) of tasks and sub-workflows. A workflow definition specifies the order of execution of -these [Tasks](taskdef.md). It also specifies how data/state is passed from one task to the other (using the -input/output parameters). These are then combined to give you the final result. This orchestration of Tasks can -happen in a hybrid ecosystem that includes microservices, serverless functions, and monolithic applications. They can -also span across any public cloud and on-premise data center footprints. In addition, the orchestration of tasks can be -across any programming language since Conductor is also language agnostic. - -One key benefit of this approach is that you can build a complex application using simple and granular tasks that do not -need to be aware of or keep track of the state of your application's execution flow. Conductor keeps track of the state, -calls tasks in the right order (sequentially or in parallel, as defined by you), retry calls if needed, handle failure -scenarios gracefully, and outputs the final result. - -Leveraging workflows in Conductor enables developers to truly focus on their core mission - building their application -code in the languages of their choice. Conductor does the heavy lifting associated with ensuring high -reliability, transactional consistency, and long durability of their workflows. Simply put, wherever your application's -component lives and whichever languages they were written in, you can build a workflow in Conductor to orchestrate their -execution in a reliable & scalable manner. - -## What does a Workflow look like? - -Let's start with a basic workflow and understand what are the different aspects of it. In particular, we will talk about -two stages of a workflow, *defining* a workflow and *executing* a workflow - -### Simple Workflow Example - -Assume your business logic is to simply to get some shipping information and then do the shipping. You start by -logically partitioning them into two tasks: - -* **shipping_info** -* **shipping_task** - -First we would build these two task definitions. Let's assume that ```shipping info``` takes an account number, and returns a name and address. - -**Example** -```json -{ - "name": "mail_a_box", - "description": "shipping Workflow", - "version": 1, - "tasks": [ - { - "name": "shipping_info", - "taskReferenceName": "shipping_info_ref", - "inputParameters": { - "account": "${workflow.input.accountNumber}" - }, - "type": "SIMPLE" - }, - { - "name": "shipping_task", - "taskReferenceName": "shipping_task_ref", - "inputParameters": { - "name": "${shipping_info_ref.output.name}", - "streetAddress": "${shipping_info_ref.output.streetAddress}", - "city": "${shipping_info_ref.output.city}", - "state": "${shipping_info_ref.output.state}", - "zipcode": "${shipping_info_ref.output.zipcode}", - }, - "type": "SIMPLE" - } - ], - "outputParameters": { - "trackingNumber": "${shipping_task_ref.output.trackinNumber}" - }, - "failureWorkflow": "shipping_issues", - "restartable": true, - "workflowStatusListenerEnabled": true, - "ownerEmail": "conductor@example.com", - "timeoutPolicy": "ALERT_ONLY", - "timeoutSeconds": 0, - "variables": {}, - "inputTemplate": {} -} -``` - -The mail_a_box workflow has 2 tasks: - 1. The first task takes the provided account number, and outputs an address. - 2. The 2nd task takes the address info and generates a shipping label. - - Upon completion of the 2 tasks, the workflow outputs the tracking number generated in the 2nd task. If the workflow fails, a second workflow named ```shipping_issues``` is run. - -## Fields in a Workflow - -| Field | Description | Notes | -|:------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------|:--------------------------------------------------------------------------------------------------| -| name | Name of the workflow || -| description | Description of the workflow | optional | -| version | Numeric field used to identify the version of the schema. Use incrementing numbers | When starting a workflow execution, if not specified, the definition with highest version is used | -| tasks | An array of task definitions. | [Task properties](#tasks-within-workflow) | -| inputParameters | List of input parameters. Used for documenting the required inputs to workflow | optional | -| inputTemplate | Default input values. See [Using inputTemplate](#using-inputtemplate) | optional | -| outputParameters | JSON template used to generate the output of the workflow | If not specified, the output is defined as the output of the _last_ executed task | -| failureWorkflow | String; Workflow to be run on current Workflow failure. Useful for cleanup or post actions on failure. | optional | -| schemaVersion | Current Conductor Schema version. schemaVersion 1 is discontinued. | Must be 2 | -| restartable | Boolean flag to allow Workflow restarts | defaults to true | -| workflowStatusListenerEnabled | If true, every workflow that gets terminated or completed will send a notification. See [workflow notifictions](#workflow-notifications) | optional (false by default) | - -## Tasks within Workflow -```tasks``` property in a workflow execution defines an array of tasks to be executed in that order. - -| Field | Description | Notes | -|:------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------------------------| -| name | Name of the task. MUST be registered as a task with Conductor before starting the workflow || -| taskReferenceName | Alias used to refer the task within the workflow. MUST be unique within workflow. || -| type | Type of task. SIMPLE for tasks executed by remote workers, or one of the system task types || -| description | Description of the task | optional | -| optional | true or false. When set to true - workflow continues even if the task fails. The status of the task is reflected as `COMPLETED_WITH_ERRORS` | Defaults to `false` | -| inputParameters | JSON template that defines the input given to the task | See [Wiring Inputs and Outputs](#wiring-inputs-and-outputs) for details | -| domain | See [Task Domains](/configuration/taskdomains.html) for more information. | optional | - -In addition to these parameters, System Tasks have their own parameters. Checkout [System Tasks](/configuration/systask.html) for more information. - -## Wiring Inputs and Outputs - -Workflows are supplied inputs by client when a new execution is triggered. -Workflow input is a JSON payload that is available via ```${workflow.input...}``` expressions. - -Each task in the workflow is given input based on the ```inputParameters``` template configured in workflow definition. ```inputParameters``` is a JSON fragment with value containing parameters for mapping values from input or output of a workflow or another task during the execution. - -Syntax for mapping the values follows the pattern as: - -__${SOURCE.input/output.JSONPath}__ - -| field | description | -|--------------|--------------------------------------------------------------------------| -| SOURCE | can be either "workflow" or any of the task reference name | -| input/output | refers to either the input or output of the source | -| JSONPath | JSON path expression to extract JSON fragment from source's input/output | - - -!!! note "JSON Path Support" - Conductor supports [JSONPath](http://goessner.net/articles/JsonPath/) specification and uses Java implementation from [here](https://github.com/jayway/JsonPath). - -!!! note "Escaping expressions" - To escape an expression, prefix it with an extra _$_ character (ex.: ```$${workflow.input...}```). - -**Example** - -Consider a task with input configured to use input/output parameters from workflow and a task named __loc_task__. - -```json -{ - "inputParameters": { - "movieId": "${workflow.input.movieId}", - "url": "${workflow.input.fileLocation}", - "lang": "${loc_task.output.languages[0]}", - "http_request": { - "method": "POST", - "url": "http://example.com/${loc_task.output.fileId}/encode", - "body": { - "recipe": "${workflow.input.recipe}", - "params": { - "width": 100, - "height": 100 - } - }, - "headers": { - "Accept": "application/json", - "Content-Type": "application/json" - } - } - } -} -``` - -Consider the following as the _workflow input_ - -```json -{ - "movieId": "movie_123", - "fileLocation":"s3://moviebucket/file123", - "recipe":"png" -} -``` -And the output of the _loc_task_ as the following; - -```json -{ - "fileId": "file_xxx_yyy_zzz", - "languages": ["en","ja","es"] -} -``` - -When scheduling the task, Conductor will merge the values from workflow input and loc_task's output and create the input to the task as follows: - -```json -{ - "movieId": "movie_123", - "url": "s3://moviebucket/file123", - "lang": "en", - "http_request": { - "method": "POST", - "url": "http://example.com/file_xxx_yyy_zzz/encode", - "body": { - "recipe": "png", - "params": { - "width": 100, - "height": 100 - } - }, - "headers": { - "Accept": "application/json", - "Content-Type": "application/json" - } - } -} -``` - -### Using inputTemplate - -* `inputTemplate` allows to define default values, which can be overridden by values provided in Workflow. -* Eg: In your Workflow Definition, you can define your inputTemplate as: - -```json -"inputTemplate": { - "url": "https://some_url:7004" -} -``` - -And `url` would be `https://some_url:7004` if no `url` was provided as input to your workflow. - -## Workflow notifications - -Conductor can be configured to publish notifications to external systems upon completion/termination of workflows. See [extending conductor](/extend.html) for details. diff --git a/docs/docs/css/custom.css b/docs/docs/css/custom.css deleted file mode 100644 index 3e4728cd5..000000000 --- a/docs/docs/css/custom.css +++ /dev/null @@ -1,313 +0,0 @@ -:root { - /*--main-text-color: #212121;*/ - --brand-blue: #1976d2; - --brand-dark-blue: #242A36; - --caption-color: #4f4f4f; - --brand-lt-blue: #f0f5fb; - --brand-gray: rgb(118, 118, 118); - --brand-lt-gray: rgb(203,204,207); - --brand-red: #e50914; -} -body { - color: var(--brand-dark-blue); - font-family: "Roboto", sans-serif; - font-weight: 400; -} - -body::before { - background: none; - display: none; -} -body > .container { - padding-top: 30px; -} - -.bg-primary { - background: #fff !important; -} - -/* Navbar */ -.navbar { - box-shadow: 0 4px 8px 0 rgb(0 0 0 / 10%), 0 0 2px 0 rgb(0 0 0 / 10%); - padding-left: 30px; - padding-right: 30px; - height: 80px; -} -.navbar-brand { - background-image: url(/img/logo.svg); - background-size: cover; - color: transparent !important; - padding: 0; - text-shadow: none; - margin-top: -6px; - height: 37px; - width: 175px; -} -.navbar-nav { - margin-left: 50px; -} -.navbar-nav > .navitem, .navbar-nav > .dropdown { - margin-left: 30px; -} -.navbar-nav > li .nav-link{ - font-size: 15px; -} - -.navbar-nav .nav-link { - color: #242A36 !important; - font-family: "Inter"; - font-weight: 700; -} - -.navbar-nav.ml-auto > li:first-child { - display: none; -} -.navbar-nav.ml-auto .nav-link{ - font-size: 0px; -} -.navbar-nav.ml-auto .nav-link .fa{ - font-size: 30px; -} -.navbar-nav .dropdown-item { - color: var(--brand-dark-blue); - font-family: "Inter"; - font-weight: 500; - font-size: 14px; - background-color: transparent; -} -.navbar-nav .dropdown-menu > li:hover { - background-color: var(--brand-blue); -} -.navbar-nav .dropdown-menu > li:hover > .dropdown-item { - color: #fff; -} -.navbar-nav .dropdown-submenu:hover > .dropdown-item { - background-color: var(--brand-blue); -} - - -.navbar-nav .dropdown-menu li { - margin: 0px; - padding-top: 5px; - padding-bottom: 5px; -} -.navbar-nav .dropdown-item.active { - background-color: transparent; -} - -.brand-darkblue { - background: #242A36 !important; -} - -.brand-gray { - background: rgb(245,245,245); -} -.brand-blue { - background: #1976D2; -} -.brand-white { - background: #fff; -} -.logo { - height: 444px; -} - -/* Fonts */ -h1, h2, h3, h4, h5, h6 { - color: var(--brand-dark-blue); - margin-bottom: 20px; -} -h1:first-child { - margin-top: 0; -} - -h1 { - font-family: "Inter", sans-serif; - font-size: 32px; - font-weight: 700; - margin-top: 50px; -} - -h2 { - font-family: "Inter", sans-serif; - font-size: 24px; - font-weight: 700; - margin-top: 40px; -} - -h3 { - font-family: "Roboto", sans-serif; - font-size: 20px; - font-weight: 500; - margin-top: 30px; -} - -h4 { - font-family: "Roboto", sans-serif; - font-size: 18px; - font-weight: 400; - margin-top: 20px; -} - -.main li { - margin-bottom: 15px; -} - - -.btn { - font-family: "Roboto", sans-serif; - font-size: 14px; -} -.btn-primary { - background: #1976D2; - border: none; -} - -.hero { - padding-top: 100px; - padding-bottom: 100px; -} - -.hero .heading { - font-size: 56px; - font-weight: 900; - line-height: 68px; -} - -.hero .btn { - font-size: 16px; - padding: 10px 20px; -} - -.hero .illustration { - margin-left: 35px; -} - - -.bullets .heading, .module .heading { - font-family: "Inter", sans-serif; - font-size: 26px; - font-weight: 700; -} -.bullets .row { - margin-bottom: 60px; -} -.bullets .caption { - padding-top: 10px; - padding-right: 30px; -} -.icon { - height: 25px; - margin-right: 5px; - vertical-align: -3px; -} - -.caption { - font-weight: 400; - font-size: 17px; - line-height: 24px; - color: var(--caption-color); -} - -.module { - margin-top: 80px; - margin-bottom: 80px; - padding-top: 50px; - padding-bottom: 50px; -} - -.module .caption { - padding-top: 10px; - padding-right: 80px; -} -.module .screenshot { - width: 600px; - height: 337px; - box-shadow:inset 0 1px 0 rgba(255,255,255,.6), 0 22px 70px 4px rgba(0,0,0,0.56), 0 0 0 1px rgba(0, 0, 0, 0.0); - border-radius: 5px; - background-size: cover; -} - -/* Footer */ -footer { - margin: 0px; - padding: 0px !important; - text-align: left; - font-weight: 400; -} -.footer { - background-color: var(--brand-dark-blue); - padding: 50px 0px; - color: #fff; - font-size: 14px; - margin-top: 50px; -} -.footer a { - color: var(--brand-lt-gray); -} -.footer .subhead { - font-weight: 700; - color: #fff; - font-size: 15px; - margin-bottom: 10px; -} -.footer .red { - color: var(--brand-red); -} -.footer .fr { - text-align: right; -} - -/* TOC menu */ -.toc ul { - list-style: none; - padding: 0px; -} -.toc > ul > li li { - padding-left: 15px; - font-weight: 400; - font-size: 14px; -} -.toc > ul > li { - font-size: 15px; - font-weight: 500; -} -.toc .toc-link { - margin-bottom: 5px; - display: block; - color: var(--brand-dark-blue); -} -.toc .toc-link.active { - font-weight: 700; -} - -/* Homepage Overrides */ -.homepage > .container { - max-width: none; -} -.homepage .toc { - display: none; -} - -/* Comparison block */ -.compare { - background-color: var(--brand-lt-blue); - padding-top: 80px; - padding-bottom: 80px; - margin: 0px -15px; -} -.compare .heading { - margin-bottom: 30px; - margin-top: 0px; -} -.compare .bubble { - background: #fff; - border-radius: 10px; - padding: 30px; - height: 100%; -} - -.compare .caption { - font-size: 15px; - line-height: 22px; -} diff --git a/docs/docs/gettingstarted/basicconcepts.md b/docs/docs/gettingstarted/basicconcepts.md deleted file mode 100644 index 6592c0423..000000000 --- a/docs/docs/gettingstarted/basicconcepts.md +++ /dev/null @@ -1,37 +0,0 @@ -# Basic Concepts - -## Definitions (aka Metadata or Blueprints) -Conductor definitions are like class definitions in OOP paradigm, or templates. You define this once, and use for each workflow execution. Definitions to Executions have 1:N relationship. - -## Tasks -Tasks are the building blocks of Workflow. There must be at least one task in a Workflow. -Tasks can be categorized into two types: - - * [System tasks](/configuration/systask.html) - executed by Conductor server. - * [Worker tasks](/configuration/workerdef.html) - executed by your own workers. - -## Workflow -A Workflow is the container of your process flow. It could include several different types of Tasks, Sub-Workflows, inputs and outputs connected to each other, to effectively achieve the desired result. The tasks are either control tasks (fork, conditional etc) or application tasks (e.g. encode a file) that are executed on a remote machine. - -[Detailed description](/configuration/workflowdef.html) - -## Task Definition -Task definitions help define Task level parameters like inputs and outputs, timeouts, retries etc. - -* All tasks need to be registered before they can be used by active workflows. -* A task can be re-used within multiple workflows. - -[Detailed description](/configuration/taskdef.html) - -## System Tasks -System tasks are executed within the JVM of the Conductor server and managed by Conductor for its execution and scalability. - -See [Systems tasks](/configuration/systask.html) for list of available Task types, and instructions for using them. - -!!! Note - Conductor provides an API to create user defined tasks that are executed in the same JVM as the engine. See [WorkflowSystemTask](https://github.com/Netflix/conductor/blob/main/core/src/main/java/com/netflix/conductor/core/execution/tasks/WorkflowSystemTask.java) interface for details. - -## Worker Tasks -Worker tasks are implemented by your application(s) and run in a separate environment from Conductor. The worker tasks can be implemented in any language. These tasks talk to Conductor server via REST/gRPC to poll for tasks and update its status after execution. - -Worker tasks are identified by task type __SIMPLE__ in the blueprint. diff --git a/docs/docs/gettingstarted/client.md b/docs/docs/gettingstarted/client.md deleted file mode 100644 index 04be2da1b..000000000 --- a/docs/docs/gettingstarted/client.md +++ /dev/null @@ -1,21 +0,0 @@ -# Using the Client -Conductor tasks that are executed by remote workers communicate over HTTP endpoints/gRPC to poll for the task and update the status of the execution. - -## Client APIs -Conductor provides the following java clients to interact with the various APIs - -| Client | Usage | -|-----------------|---------------------------------------------------------------------------| -| Metadata Client | Register / Update workflow and task definitions | -| Workflow Client | Start a new workflow / Get execution status of a workflow | -| Task Client | Poll for task / Update task result after execution / Get status of a task | - -## SDKs - -* [Clojure](/how-tos/clojure-sdk.html) -* [C#](/how-tos/csharp-sdk.html) -* [Go](/how-tos/go-sdk.html) -* [Java](/how-tos/java-sdk.html) -* [Python](/how-tos/python-sdk.html) - -The non-Java Conductor SDKs are hosted on a separate GitHub repository: [github.com/conductor-sdk](https://github.com/conductor-sdk). Contributions from the community are encouraged! diff --git a/docs/docs/gettingstarted/steps.md b/docs/docs/gettingstarted/steps.md deleted file mode 100644 index 695aa6277..000000000 --- a/docs/docs/gettingstarted/steps.md +++ /dev/null @@ -1,36 +0,0 @@ - -# High Level Steps -Steps required for a new workflow to be registered and get executed - -1. Define task definitions used by the workflow. -2. Create the workflow definition -3. Create task worker(s) that polls for scheduled tasks at regular interval - -### Trigger Workflow Execution - -``` -POST /workflow/{name} -{ - ... //json payload as workflow input -} -``` - -### Polling for a task - -``` -GET /tasks/poll/batch/{taskType} -``` - -### Update task status - -``` -POST /tasks -{ - "outputData": { - "encodeResult":"success", - "location": "http://cdn.example.com/file/location.png" - //any task specific output - }, - "status": "COMPLETED" -} -``` diff --git a/docs/docs/googleba55068fa3e0e553.html b/docs/docs/googleba55068fa3e0e553.html deleted file mode 100644 index 0344c087e..000000000 --- a/docs/docs/googleba55068fa3e0e553.html +++ /dev/null @@ -1 +0,0 @@ -google-site-verification: googleba55068fa3e0e553.html \ No newline at end of file diff --git a/docs/docs/how-tos/Tasks/dynamic-vs-switch-tasks.md b/docs/docs/how-tos/Tasks/dynamic-vs-switch-tasks.md deleted file mode 100644 index e81327d49..000000000 --- a/docs/docs/how-tos/Tasks/dynamic-vs-switch-tasks.md +++ /dev/null @@ -1,22 +0,0 @@ ---- -sidebar_position: 1 ---- - -# Dynamic vs Switch Tasks - -Learn more about - -1. [Dynamic Tasks](/reference-docs/dynamic-task.html) -2. [Switch Tasks](/reference-docs/switch-task.html) - -Dynamic Tasks are useful in situations when need to run a task of which the task type is determined at runtime instead -of during the configuration. It is similar to the [SWITCH](/reference-docs/switch-task.html) use case but with `DYNAMIC` -we won't need to preconfigure all case options in the workflow definition itself. Instead, we can mark the task -as `DYNAMIC` and determine which underlying task does it run during the workflow execution itself. - -1. Use DYNAMIC task as a replacement for SWITCH if you have too many case options -2. DYNAMIC task is an option when you want to programmatically determine the next task to run instead of using expressions -3. DYNAMIC task simplifies the workflow execution UI view which will now only show the selected task -4. SWITCH task visualization is helpful as a documentation - showing you all options that the workflow could have - taken -5. SWITCH task comes with a default task option which can be useful in some use cases diff --git a/docs/docs/how-tos/Test/testing-workflows.md b/docs/docs/how-tos/Test/testing-workflows.md deleted file mode 100644 index 1afb835f1..000000000 --- a/docs/docs/how-tos/Test/testing-workflows.md +++ /dev/null @@ -1,34 +0,0 @@ -# Conductor Workflow Testing Guide - -## Unit and Regression testing workflows - -### Unit Tests -Conductor workflows can be unit tested using `POST /workflow/test` endpoint. -The approach is similar to how you unit test using Mock objects in Java or similar languages. - -#### Why Unit Test Workflows? -Unit tests allows you to test for the correctness of the workflow definition ensuring: -1. Given a specific input workflow reaches the terminal state in COMPLETED or FAILED state -2. Given a specific input, the workflow executes specific set of tasks. This is useful for testing branching and dynamic forks -3. Task inputs are wired correctly - e.g. if a task receives its input from the output of another task, this can be verified using the unit test. - -### Unit Testing Workflows -Conductor SDKs provides the following method that allows testing a workflow definition against mock inputs: -```java - public abstract Workflow testWorkflow(WorkflowTestRequest testRequest); -``` -The actual workflow is executed on a real Conductor server ensuring you are testing the behavior that will match the ACTUAL executon of the server. - -### Setting up Conductor server for testing -Tests can be run against a remote server (useful when running integration tests) or local containerized instance. Recommended approach is to use `testcontainers`. - -### Examples - -#### Unit Test -* [LoanWorkflowTest.java](/client/src/test/java/com/netflix/conductor/client/testing/LoanWorkflowTest.java) -* Testing workflows that contain sub-workflows : [SubWorkflowTest.java](/client/src/test/java/com/netflix/conductor/client/testing/SubWorkflowTest.java) - -#### Regression Test -Workflows can be regression tested with golden inputs and outputs. This approach is useful when modifying workflows that are running in production to ensure the behavior remains correct. - -See [RegressionTest.java](/client/src/test/java/com/netflix/conductor/client/testing/RegressionTest.java) for an example, which uses previously captured workflow execution as golden input/output to verify the workflow execution. \ No newline at end of file diff --git a/docs/docs/img/logo.png b/docs/docs/img/logo.png deleted file mode 100644 index 132d52cde..000000000 Binary files a/docs/docs/img/logo.png and /dev/null differ diff --git a/docs/docs/img/logo_dark_background.png b/docs/docs/img/logo_dark_background.png deleted file mode 100644 index 013020f12..000000000 Binary files a/docs/docs/img/logo_dark_background.png and /dev/null differ diff --git a/docs/docs/img/netflix.png b/docs/docs/img/netflix.png deleted file mode 100755 index 151775b3b..000000000 Binary files a/docs/docs/img/netflix.png and /dev/null differ diff --git a/docs/docs/img/task_states.svg b/docs/docs/img/task_states.svg deleted file mode 100644 index 3fd55079b..000000000 --- a/docs/docs/img/task_states.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - - diff --git a/docs/docs/img/tutorial/SubWorkflow.png b/docs/docs/img/tutorial/SubWorkflow.png deleted file mode 100644 index f91865abd..000000000 Binary files a/docs/docs/img/tutorial/SubWorkflow.png and /dev/null differ diff --git a/docs/docs/img/tutorial/Sub_Workflow_Run.png b/docs/docs/img/tutorial/Sub_Workflow_Run.png deleted file mode 100644 index b1635f2b4..000000000 Binary files a/docs/docs/img/tutorial/Sub_Workflow_Run.png and /dev/null differ diff --git a/docs/docs/img/tutorial/Switch_Default.png b/docs/docs/img/tutorial/Switch_Default.png deleted file mode 100644 index b501c469b..000000000 Binary files a/docs/docs/img/tutorial/Switch_Default.png and /dev/null differ diff --git a/docs/docs/img/tutorial/Switch_Fedex.png b/docs/docs/img/tutorial/Switch_Fedex.png deleted file mode 100644 index 63cfc50c3..000000000 Binary files a/docs/docs/img/tutorial/Switch_Fedex.png and /dev/null differ diff --git a/docs/docs/img/tutorial/Switch_Workflow.png b/docs/docs/img/tutorial/Switch_Workflow.png deleted file mode 100644 index dbac1b051..000000000 Binary files a/docs/docs/img/tutorial/Switch_Workflow.png and /dev/null differ diff --git a/docs/docs/img/tutorial/Terminate_Task.png b/docs/docs/img/tutorial/Terminate_Task.png deleted file mode 100644 index b045dd9ab..000000000 Binary files a/docs/docs/img/tutorial/Terminate_Task.png and /dev/null differ diff --git a/docs/docs/img/tutorial/Terminate_Task_Run.png b/docs/docs/img/tutorial/Terminate_Task_Run.png deleted file mode 100644 index 353c0db57..000000000 Binary files a/docs/docs/img/tutorial/Terminate_Task_Run.png and /dev/null differ diff --git a/docs/docs/img/tutorial/Terminate_Task_Successful.png b/docs/docs/img/tutorial/Terminate_Task_Successful.png deleted file mode 100644 index 411f531e3..000000000 Binary files a/docs/docs/img/tutorial/Terminate_Task_Successful.png and /dev/null differ diff --git a/docs/docs/img/tutorial/conductorConsole.png b/docs/docs/img/tutorial/conductorConsole.png deleted file mode 100644 index d5ae6477a..000000000 Binary files a/docs/docs/img/tutorial/conductorConsole.png and /dev/null differ diff --git a/docs/docs/img/tutorial/conductorHome.png b/docs/docs/img/tutorial/conductorHome.png deleted file mode 100644 index 314b216ba..000000000 Binary files a/docs/docs/img/tutorial/conductorHome.png and /dev/null differ diff --git a/docs/docs/img/tutorial/conductorUIHome.png b/docs/docs/img/tutorial/conductorUIHome.png deleted file mode 100644 index c21d9ab8e..000000000 Binary files a/docs/docs/img/tutorial/conductorUIHome.png and /dev/null differ diff --git a/docs/docs/img/tutorial/dockerHome.png b/docs/docs/img/tutorial/dockerHome.png deleted file mode 100644 index 59ac72ec3..000000000 Binary files a/docs/docs/img/tutorial/dockerHome.png and /dev/null differ diff --git a/docs/docs/img/tutorial/docsVersionDropdown.png b/docs/docs/img/tutorial/docsVersionDropdown.png deleted file mode 100644 index ff1cbe688..000000000 Binary files a/docs/docs/img/tutorial/docsVersionDropdown.png and /dev/null differ diff --git a/docs/docs/img/tutorial/elasticSearchHome.png b/docs/docs/img/tutorial/elasticSearchHome.png deleted file mode 100644 index f417b0c1d..000000000 Binary files a/docs/docs/img/tutorial/elasticSearchHome.png and /dev/null differ diff --git a/docs/docs/img/tutorial/firstWorkerWorkflow.png b/docs/docs/img/tutorial/firstWorkerWorkflow.png deleted file mode 100644 index 1f983b96e..000000000 Binary files a/docs/docs/img/tutorial/firstWorkerWorkflow.png and /dev/null differ diff --git a/docs/docs/img/tutorial/localeDropdown.png b/docs/docs/img/tutorial/localeDropdown.png deleted file mode 100644 index d7163f967..000000000 Binary files a/docs/docs/img/tutorial/localeDropdown.png and /dev/null differ diff --git a/docs/docs/img/tutorial/successfulWorkerExecution.png b/docs/docs/img/tutorial/successfulWorkerExecution.png deleted file mode 100644 index 8cd2774e7..000000000 Binary files a/docs/docs/img/tutorial/successfulWorkerExecution.png and /dev/null differ diff --git a/docs/docs/labs/beginner.md b/docs/docs/labs/beginner.md deleted file mode 100644 index 3953cd525..000000000 --- a/docs/docs/labs/beginner.md +++ /dev/null @@ -1,439 +0,0 @@ -# Beginner Lab -## Hands on mode -Please feel free to follow along using any of these resources: - -- Using cURL -- Postman or similar REST client - -## Creating a Workflow - -Let's create a simple workflow that adds Netflix Idents to videos. We'll be mocking the adding Idents part and focusing on actually executing this process flow. - -!!!info "What are Netflix Idents?" - Netflix Idents are those 4 second videos with Netflix logo, which appears at the beginning and end of shows. You might have also noticed they're different for Animation and several other genres. - -!!!warning "Disclaimer" - Obviously, this is not how Netflix adds Idents. Those Workflows are indeed very complex. But, it should give you an idea about how Conductor can be used to implement similar features. - -The workflow in this lab will look like this: - -![img](img/bgnr_complete_workflow.png) - -This workflow contains the following: - -* Worker Task `verify_if_idents_are_added` to verify if Idents are already added. - -* [Switch Task](/reference-docs/switch-task.html) that takes output from the previous task, and decides whether to schedule the `add_idents` task. - -* `add_idents` task which is another worker Task. - -### Creating Task definitions - - -Let's create the [task definition](/configuration/taskdef.html) for `verify_if_idents_are_added` in JSON. This task will be a *SIMPLE* task which is supposed to be executed by an Idents microservice. We'll be mocking the Idents microservice part. - - - -**Note** that at this point, we don't have to specify whether it is a System task or Worker task. We are only specifying the required configurations for the task, like number of times it should be retried, timeouts etc. We shall start by using `name` parameter for task name. -```json -{ - "name": "verify_if_idents_are_added" -} -``` - -We'd like this task to be retried 3 times on failure. - -```json -{ - "name": "verify_if_idents_are_added", - "retryCount": 3, - "retryLogic": "FIXED", - "retryDelaySeconds": 10 -} -``` - -And to timeout after 300 seconds. -i.e. if the task doesn't finish execution within this time limit after transitioning to `IN_PROGRESS` state, the Conductor server cancels this task and schedules a new execution of this task in the queue. - -```json -{ - "name": "verify_if_idents_are_added", - "retryCount": 3, - "retryLogic": "FIXED", - "retryDelaySeconds": 10, - "timeoutSeconds": 300, - "timeoutPolicy": "TIME_OUT_WF" -} -``` - -And a [responseTimeout](/architecture/tasklifecycle.html#response-timeout-seconds) of 180 seconds. - -```json -{ - "name": "verify_if_idents_are_added", - "retryCount": 3, - "retryLogic": "FIXED", - "retryDelaySeconds": 10, - "timeoutSeconds": 300, - "timeoutPolicy": "TIME_OUT_WF", - "responseTimeoutSeconds": 180 -} -``` - - -We can define several other fields defined [here](/configuration/taskdef.html), but this is a good place to start with. - - -Similarly, create another task definition: `add_idents`. - -```json -{ - "name": "add_idents", - "retryCount": 3, - "retryLogic": "FIXED", - "retryDelaySeconds": 10, - "timeoutSeconds": 300, - "timeoutPolicy": "TIME_OUT_WF", - "responseTimeoutSeconds": 180 -} -``` - -Send a `POST` request to `/metadata/taskdefs` endpoint to register these tasks. You can use Swagger, Postman, CURL or similar tools. - -!!!info "Why is the Switch Task not registered?" - System Tasks that are part of control flow do not need to be registered. However, some system tasks where the retries, rate limiting and other mechanisms are required, like `HTTP` Task, are to be registered though. - -!!! Important - Task and Workflow Definition names are unique. The names we use below might have already been registered. For this lab, add a prefix with your username, `{my_username}_verify_if_idents_are_added` for example. This is definitely not recommended for Production usage though. - - -**Example** -``` -curl -X POST \ - http://localhost:8080/api/metadata/taskdefs \ - -H 'Content-Type: application/json' \ - -d '[ - { - "name": "verify_if_idents_are_added", - "retryCount": 3, - "retryLogic": "FIXED", - "retryDelaySeconds": 10, - "timeoutSeconds": 300, - "timeoutPolicy": "TIME_OUT_WF", - "responseTimeoutSeconds": 180, - "ownerEmail": "type your email here" - }, - { - "name": "add_idents", - "retryCount": 3, - "retryLogic": "FIXED", - "retryDelaySeconds": 10, - "timeoutSeconds": 300, - "timeoutPolicy": "TIME_OUT_WF", - "responseTimeoutSeconds": 180, - "ownerEmail": "type your email here" - } -]' -``` - -### Creating Workflow Definition - -Creating Workflow definition is almost similar. We shall use the Task definitions created above. Note that same Task definitions can be used in multiple workflows, or for multiple times in same Workflow (that's where `taskReferenceName` is useful). - -A workflow without any tasks looks like this: -```json -{ - "name": "add_netflix_identation", - "description": "Adds Netflix Identation to video files.", - "version": 1, - "schemaVersion": 2, - "tasks": [] -} -``` - -Add the first task that this workflow has to execute. All the tasks must be added to the `tasks` array. - -```json -{ - "name": "add_netflix_identation", - "description": "Adds Netflix Identation to video files.", - "version": 1, - "schemaVersion": 2, - "tasks": [ - { - "name": "verify_if_idents_are_added", - "taskReferenceName": "ident_verification", - "inputParameters": { - "contentId": "${workflow.input.contentId}" - }, - "type": "SIMPLE" - } - ] -} -``` - -**Wiring Input/Outputs** - -Notice how we were using `${workflow.input.contentId}` to pass inputs to this task. Conductor can wire inputs between workflow and tasks, and between tasks. -i.e The task `verify_if_idents_are_added` is wired to accept inputs from the workflow input using JSONPath expression `${workflow.input.param}`. - - -Learn more about wiring inputs and outputs [here](/configuration/workflowdef.html#wiring-inputs-and-outputs). - -Let's define `decisionCases` now. - - ->Note: in earlier versions of this tutorial, the "decision" task was used. This has been deprecated. - -Checkout the Switch task structure [here](/reference-docs/switch-task.html). - -A Switch task is specified by the `evaulatorType`, `expression` (the expression that defines the Switch) and `decisionCases` which lists all the branches of Switch task. - -In this case, we'll use `"evaluatorType": "value-param"`, meaning that we'll just use the value inputted to make the decision. Alternatively, there is a `"evaluatorType": "JavaScript"` that can be used for more complicated evaluations. - -Adding the switch task (without any decision cases): -```json -{ - "name": "add_netflix_identation", - "description": "Adds Netflix Identation to video files.", - "version": 2, - "schemaVersion": 2, - "tasks": [ - { - "name": "verify_if_idents_are_added", - "taskReferenceName": "ident_verification", - "inputParameters": { - "contentId": "${workflow.input.contentId}" - }, - "type": "SIMPLE" - }, - { - "name": "switch_task", - "taskReferenceName": "is_idents_added", - "inputParameters": { - "case_value_param": "${ident_verification.output.is_idents_added}" - }, - "type": "SWITCH", - "evaluatorType": "value-param", - "expression": "case_value_param", - "decisionCases": { - - } - } - ] -} -``` - -Each switch task can have multiple tasks, so it has to be defined as an array. -```json -{ - "name": "add_netflix_identation", - "description": "Adds Netflix Identation to video files.", - "version": 2, - "schemaVersion": 2, - "tasks": [ - { - "name": "verify_if_idents_are_added", - "taskReferenceName": "ident_verification", - "inputParameters": { - "contentId": "${workflow.input.contentId}" - }, - "type": "SIMPLE" - }, - { - "name": "switch_task", - "taskReferenceName": "is_idents_added", - "inputParameters": { - "case_value_param": "${ident_verification.output.is_idents_added}" - }, - "type": "SWITCH", - "evaluatorType": "value-param", - "expression": "case_value_param", - "decisionCases": { - "false": [ - { - "name": "add_idents", - "taskReferenceName": "add_idents_by_type", - "inputParameters": { - "identType": "${workflow.input.identType}", - "contentId": "${workflow.input.contentId}" - }, - "type": "SIMPLE" - } - ] - } - } - ] -} -``` - -Just like the task definitions, register this workflow definition by sending a POST request to `/workflow` endpoint. - -**Example** -``` -curl -X POST \ - http://localhost:8080/api/metadata/workflow \ - -H 'Content-Type: application/json' \ - -d '{ - "name": "add_netflix_identation", - "description": "Adds Netflix Identation to video files.", - "version": 2, - "schemaVersion": 2, - "tasks": [ - { - "name": "verify_if_idents_are_added", - "taskReferenceName": "ident_verification", - "inputParameters": { - "contentId": "${workflow.input.contentId}" - }, - "type": "SIMPLE" - }, - { - "name": "switch_task", - "taskReferenceName": "is_idents_added", - "inputParameters": { - "case_value_param": "${ident_verification.output.is_idents_added}" - }, - "type": "SWITCH", - "evaluatorType": "value-param", - "expression": "case_value_param", - "decisionCases": { - "false": [ - { - "name": "add_idents", - "taskReferenceName": "add_idents_by_type", - "inputParameters": { - "identType": "${workflow.input.identType}", - "contentId": "${workflow.input.contentId}" - }, - "type": "SIMPLE" - } - ] - } - } - ] -}' -``` - -### Starting the Workflow - -Send a `POST` request to `/workflow` with: -```json -{ - "name": "add_netflix_identation", - "version": 2, - "correlationId": "my_netflix_identation_workflows", - "input": { - "identType": "animation", - "contentId": "my_unique_content_id" - } -} -``` - -Example: -``` -curl -X POST \ - http://localhost:8080/api/workflow/add_netflix_identation \ - -H 'Content-Type: application/json' \ - -d '{ - "identType": "animation", - "contentId": "my_unique_content_id" -}' -``` - -Successful POST request should return a workflow Id, which you can use to find the execution in the UI. - -### Conductor User Interface - -Open the UI and navigate to the RUNNING tab, the Workflow should be in the state as below: - -![img](img/bgnr_state_scheduled.png) - -Feel free to explore the various functionalities that the UI exposes. To elaborate on a few: - -* Workflow Task modals (Opens on clicking any of the tasks in the workflow), which includes task I/O, logs and task JSON. -* Task Details tab, which shows the sequence of task execution, status, start/end time, and link to worker details which executed the task. -* Input/Output tab shows workflow input and output. - - -### Poll for Worker task - -Now that `verify_if_idents_are_added` task is in `SCHEDULED` state, it is the worker's turn to fetch the task, execute it and update Conductor with final status of the task. - -Ideally, the workers implementing the [Client](/gettingstarted/client.html#worker) interface would do this process, executing the tasks on real microservices. But, let's mock this part. - -Send a `GET` request to `/poll` endpoint with your task type. - -For example: - -``` -curl -X GET \ - http://localhost:8080/api/tasks/poll/verify_if_idents_are_added -``` - - -### Return response, add logs - -We can respond to Conductor with any of the following states: - -* Task has COMPLETED. -* Task has FAILED. -* Call back after seconds [Process the task at a later time]. - -Considering our Ident Service has verified that the Ident's are not yet added to given Content Id, let's return the task status by sending the below `POST` request to `/tasks` endpoint, with payload: - -```json -{ - "workflowInstanceId": "{workflowId}", - "taskId": "{taskId}", - "reasonForIncompletion": "", - "callbackAfterSeconds": 0, - "workerId": "localhost", - "status": "COMPLETED", - "outputData": { - "is_idents_added": false - } -} -``` - -Example: - -``` -curl -X POST \ - http://localhost:8080/api/tasks \ - -H 'Content-Type: application/json' \ - -d '{ - "workflowInstanceId": "cb7c5041-aa85-4940-acb4-3bdcfa9f5c5c", - "taskId": "741f362b-ee9a-47b6-81b5-9bbbd5c4c992", - "reasonForIncompletion": "", - "callbackAfterSeconds": 0, - "workerId": "string", - "status": "COMPLETED", - "outputData": { - "is_idents_added": false - }, - "logs": [ - { - "log": "Ident verification successful for title: {some_title_name}, with Id: {some_id}", - "createdTime": 1550178825 - } - ] - }' -``` - -!!! Info "Check logs in UI" - You can find the logs we just sent by clicking the `verify_if_idents_are_added`, upon which a modal should open with `Logs` tab. - -### Why is System task executed, but Worker task is Scheduled. - -You will notice that Workflow is in the state as below after sending the POST request: - -![img](img/bgnr_systask_state.png) - -Conductor has executed `is_idents_added` all through it's lifecycle, without us polling, or returning the status of Task. If it is still unclear, `is_idents_added` is a System task, and System tasks are executed by Conductor Server. - -But, `add_idents` is a SIMPLE task. So, the complete lifecyle of this task (Poll, Update) should be handled by a worker to continue with W\workflow execution. When Conductor has finished executing all the tasks in given flow, the workflow will reach Terminal state (COMPLETED, FAILED, TIMED_OUT etc.) - -## Next steps - -You can play around this workflow by failing one of the Tasks, restarting or retrying the Workflow, or by tuning the number of retries, timeoutSeconds etc. diff --git a/docs/docs/reference-docs/archival-of-workflows.md b/docs/docs/reference-docs/archival-of-workflows.md deleted file mode 100644 index 064d48e81..000000000 --- a/docs/docs/reference-docs/archival-of-workflows.md +++ /dev/null @@ -1,13 +0,0 @@ -# Archival Of Workflows - -Conductor has support for archiving workflow upon termination or completion. Enabling this will delete the workflow from the configured database, but leave the associated data in Elasticsearch so it is still searchable. - -To enable, set the `conductor.workflow-status-listener.type` property to `archive`. - -A number of additional properties are available to control archival. - -| Property | Default Value | Description | -| -- | -- | -- | -| conductor.workflow-status-listener.archival.ttlDuration | 0s | The time to live in seconds for workflow archiving module. Currently, only RedisExecutionDAO supports this | -| conductor.workflow-status-listener.archival.delayQueueWorkerThreadCount | 5 | The number of threads to process the delay queue in workflow archival | -| conductor.workflow-status-listener.archival.delaySeconds | 60 | The time to delay the archival of workflow | diff --git a/docs/docs/reference-docs/event-task.md b/docs/docs/reference-docs/event-task.md deleted file mode 100644 index f0aaba2bd..000000000 --- a/docs/docs/reference-docs/event-task.md +++ /dev/null @@ -1,95 +0,0 @@ ---- -sidebar_position: 4 ---- - -# Event Task - -```json -"type" : "EVENT" -``` - -### Introduction -EVENT is a task used to publish an event into one of the supported eventing systems in Conductor. -Conductor supports the following eventing models: - -1. Conductor internal events (type: conductor) -2. SQS (type: sqs) - -### Use Cases -Consider a use case where at some point in the execution, an event is published to an external eventing system such as SQS. -Event tasks are useful for creating event based dependencies for workflows and tasks. - -Consider an example where we want to publish an event into SQS to notify an external system. - -```json -{ - "type": "EVENT", - "sink": "sqs:sqs_queue_name", - "asyncComplete": false -} -``` - -An example where we want to publish a message to conductor's internal queuing system. -```json -{ - "type": "EVENT", - "sink": "conductor:internal_event_name", - "asyncComplete": false -} -``` - - -### Configuration - -#### Input Configuration - -| Attribute | Description | -|-------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| name | Task Name. A unique name that is descriptive of the task function | -| taskReferenceName | Task Reference Name. A unique reference to this task. There can be multiple references of a task within the same workflow definition | -| type | Task Type. In this case, `EVENT` | -| sink | External event queue in the format of `prefix:location`. Prefix is either `sqs` or `conductor` and `location` specifies the actual queue name. e.g. "sqs:send_email_queue" | -| asyncComplete | Boolean | - -#### asyncComplete -* ```false``` to mark status COMPLETED upon execution -* ```true``` to keep it IN_PROGRESS, wait for an external event (via Conductor or SQS or EventHandler) to complete it. - -#### Output Configuration -Tasks's output are sent as a payload to the external event. In case of SQS the task's output is sent to the SQS message a payload. - - -| name | type | description | -|--------------------|---------|---------------------------------------| -| workflowInstanceId | String | Workflow id | -| workflowType | String | Workflow Name | -| workflowVersion | Integer | Workflow Version | -| correlationId | String | Workflow CorrelationId | -| sink | String | Copy of the input data "sink" | -| asyncComplete | Boolean | Copy of the input data "asyncComplete | -| event_produced | String | Name of the event produced | - -The published event's payload is identical to the output of the task (except "event_produced"). - - -When producing an event with Conductor as sink, the event name follows the structure: -```conductor::``` - -For SQS, use the **name** of the queue and NOT the URI. Conductor looks up the URI based on the name. - -!!!warning - When using SQS add the [ContribsModule](https://github.com/Netflix/conductor/blob/master/contribs/src/main/java/com/netflix/conductor/contribs/ContribsModule.java) to the deployment. The module needs to be configured with AWSCredentialsProvider for Conductor to be able to use AWS APIs. - - -!!!warning - When using Conductor as sink, you have two options: defining the sink as `conductor` in which case the queue name will default to the taskReferenceName of the Event Task, or specifying the queue name in the sink, as `conductor:`. The queue name is in the `event` value of the event Handler, as `conductor::`. - - -### Supported Queuing Systems -Conductor has support for the following external event queueing systems as part of the OSS build - -1. SQS (prefix: sqs) -2. [NATS](https://github.com/Netflix/conductor/tree/main/contribs/src/main/java/com/netflix/conductor/contribs/queue/nats) (prefix: nats) -3. [AMQP](https://github.com/Netflix/conductor/tree/main/contribs/src/main/java/com/netflix/conductor/contribs/queue/amqp) (prefix: amqp_queue or amqp_exchange) -4. Internal Conductor (prefix: conductor) -To add support for other diff --git a/docs/docs/reference-docs/fork-task.md b/docs/docs/reference-docs/fork-task.md deleted file mode 100644 index fac43c243..000000000 --- a/docs/docs/reference-docs/fork-task.md +++ /dev/null @@ -1,140 +0,0 @@ -# Fork -```json -"type" : "FORK_JOIN" -``` - -## Introduction - -A Fork operation lets you run a specified list of tasks or sub workflows in parallel. A fork task is -followed by a join operation that waits on the forked tasks or sub workflows to finish. The `JOIN` -task also collects outputs from each of the forked tasks or sub workflows. - -## Use Cases - -`FORK_JOIN` tasks are typically used when a list of tasks can be run in parallel. E.g In a notification workflow, there -could be multiple ways of sending notifications, i,e e-mail, SMS, HTTP etc.. These notifications are not dependent on -each other, and so they can be run in parallel. In such cases, you can create 3 sub-lists of forked tasks for each of -these operations. - -## Configuration - -A `FORK_JOIN` task has a `forkTasks` attribute that expects an array. Each array is a sub-list of tasks. Each of these -sub-lists are then invoked in parallel. The tasks defined within each sublist can be sequential or any other way as -desired. - -A FORK_JOIN task has to be followed by a JOIN operation. The `JOIN` operator specifies which of the forked tasks -to `joinOn` (wait for completion) -before moving to the next stage in the workflow. - -#### Input Configuration - -| Attribute | Description | -|-------------------|-----------------------------------------------------------------------------------------------------------------------------------------------| -| name | Task Name. A unique name that is descriptive of the task function | -| taskReferenceName | Task Reference Name. A unique reference to this task. There can be multiple references of a task within the same workflow definition | -| type | Task Type. In this case, `FORK_JOIN` | -| inputParameters | The input parameters that will be supplied to this task | -| forkTasks | A list of a list of tasks. Each of the outer list will be invoked in parallel. The inner list can be a graph of other tasks and sub-workflows | - -#### Output Configuration - -This is the output configuration of the `JOIN` task that is used in conjunction with the `FORK_JOIN` task. The output of -the -`JOIN` task is a map, where the keys are the names of the task reference names where were being `joinOn` and the keys -are the corresponding outputs of those tasks. - -| Attribute | Description | -|-----------------|-------------------------------------------------------------------------------------| -| task_ref_name_1 | A task reference name that was being `joinOn`. The value is the output of that task | -| task_ref_name_2 | A task reference name that was being `joinOn`. The value is the output of that task | -| ... | ... | -| task_ref_name_N | A task reference name that was being `joinOn`. The value is the output of that task | - - - -### Example - -Imagine a workflow that sends 3 notifications: email, SMS and HTTP. Since none of these steps are dependent on the others, they can be run in parallel with a fork. - -The diagram will appear as: - -![fork diagram](/img/fork-task-diagram.png) - -Here's the JSON definition for the workflow: - -```json -[ - { - "name": "fork_join", - "taskReferenceName": "my_fork_join_ref", - "type": "FORK_JOIN", - "forkTasks": [ - [ - { - "name": "process_notification_payload", - "taskReferenceName": "process_notification_payload_email", - "type": "SIMPLE" - }, - { - "name": "email_notification", - "taskReferenceName": "email_notification_ref", - "type": "SIMPLE" - } - ], - [ - { - "name": "process_notification_payload", - "taskReferenceName": "process_notification_payload_sms", - "type": "SIMPLE" - }, - { - "name": "sms_notification", - "taskReferenceName": "sms_notification_ref", - "type": "SIMPLE" - } - ], - [ - { - "name": "process_notification_payload", - "taskReferenceName": "process_notification_payload_http", - "type": "SIMPLE" - }, - { - "name": "http_notification", - "taskReferenceName": "http_notification_ref", - "type": "SIMPLE" - } - ] - ] - }, - { - "name": "notification_join", - "taskReferenceName": "notification_join_ref", - "type": "JOIN", - "joinOn": [ - "email_notification_ref", - "sms_notification_ref" - ] - } -] -``` -> Note: There are three parallel 'tines' to this fork, but only two of the outputs are required for the JOIN to continue. The diagram *does* draw an arrow from ```http_notification_ref``` to the ```notification_join```, but it is not required for the workflow to continue. - -Here is how the output of notification_join will look like. The output is a map, where the keys are the names of task -references that were being `joinOn`. The corresponding values are the outputs of those tasks. - -```json - -{ - "email_notification_ref": { - "email_sent_at": "2021-11-06T07:37:17+0000", - "email_sent_to": "test@example.com" - }, - "sms_notification_ref": { - "smm_sent_at": "2021-11-06T07:37:17+0129", - "sms_sen": "+1-425-555-0189" - } -} -``` - -See [JOIN](/reference-docs/join-task.html) for more details on the JOIN aspect of the FORK. diff --git a/docs/docs/reference-docs/http-task.md b/docs/docs/reference-docs/http-task.md deleted file mode 100644 index 35e558a0d..000000000 --- a/docs/docs/reference-docs/http-task.md +++ /dev/null @@ -1,144 +0,0 @@ ---- -sidebar_position: 1 ---- - -# HTTP Task - -```json -"type" : "HTTP" -``` - -### Introduction - -An HTTP task is useful when you have a requirements such as: - -1. Making calls to another service that exposes an API via HTTP -2. Fetch any resource or data present on an endpoint - -### Use Cases - -If we have a scenario where we need to make an HTTP call into another service, we can make use of HTTP tasks. You can -use the data returned from the HTTP call in your subsequent tasks as inputs. Using HTTP tasks you can avoid having to -write the code that talks to these services and instead let Conductor manage it directly. This can reduce the code you -have to maintain and allows for a lot of flexibility. - -### Configuration - -HTTP task is defined directly inside the workflow with the task type `HTTP`. - -| name | type | description | -|--------------|-------------|-------------------------| -| http_request | HttpRequest | JSON object (see below) | - -#### Inputs - -| Name | Type | Description | -|---------------------|------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| uri | String | URI for the service. Can be a partial when using vipAddress or includes the server address. | -| method | String | HTTP method. GET, PUT, POST, DELETE, OPTIONS, HEAD | -| accept | String | Accept header. Default: ```application/json``` | -| contentType | String | Content Type - supported types are ```text/plain```, ```text/html```, and ```application/json``` (Default) | -| headers | Map[String, Any] | A map of additional http headers to be sent along with the request. | -| body | Map[] | Request body | -| vipAddress | String | When using discovery based service URLs. | -| asyncComplete | Boolean | ```false``` to mark status COMPLETED upon execution ; ```true``` to keep it IN_PROGRESS, wait for an external event (via Conductor or SQS or EventHandler) to complete it. | -| oauthConsumerKey | String | [OAuth](https://oauth.net/core/1.0/) client consumer key | -| oauthConsumerSecret | String | [OAuth](https://oauth.net/core/1.0/) client consumer secret | -| connectionTimeOut | Integer | Connection Time Out in milliseconds. If set to 0, equivalent to infinity. Default: 100. | -| readTimeOut | Integer | Read Time Out in milliseconds. If set to 0, equivalent to infinity. Default: 150. | - -#### Output - -| name | type | description | -|--------------|------------------|-----------------------------------------------------------------------------| -| response | Map | JSON body containing the response if one is present | -| headers | Map[String, Any] | Response Headers | -| statusCode | Integer | [Http Status Code](https://en.wikipedia.org/wiki/List_of_HTTP_status_codes) | -| reasonPhrase | String | Http Status Code's reason phrase | - -### Examples - -Following is the example of HTTP task with `GET` method. - -We can use variables in our URI as show in the example below. - -```json -{ - "name": "Get Example", - "taskReferenceName": "get_example", - "inputParameters": { - "http_request": { - "uri": "https://jsonplaceholder.typicode.com/posts/${workflow.input.queryid}", - "method": "GET" - } - }, - "type": "HTTP" -} -``` - -Following is the example of HTTP task with `POST` method. - -> Here we are using variables for our POST body which happens to be data from a previous task. This is an example of how you can **chain** HTTP calls to make complex flows happen without writing any additional code. - -```json -{ - "name": "http_post_example", - "taskReferenceName": "post_example", - "inputParameters": { - "http_request": { - "uri": "https://jsonplaceholder.typicode.com/posts/", - "method": "POST", - "body": { - "title": "${get_example.output.response.body.title}", - "userId": "${get_example.output.response.body.userId}", - "action": "doSomething" - } - } - }, - "type": "HTTP" -} -``` - -Following is the example of HTTP task with `PUT` method. - -```json -{ - "name": "http_put_example", - "taskReferenceName": "put_example", - "inputParameters": { - "http_request": { - "uri": "https://jsonplaceholder.typicode.com/posts/1", - "method": "PUT", - "body": { - "title": "${get_example.output.response.body.title}", - "userId": "${get_example.output.response.body.userId}", - "action": "doSomethingDifferent" - } - } - }, - "type": "HTTP" -} -``` - -Following is the example of HTTP task with `DELETE` method. - -```json -{ - "name": "DELETE Example", - "taskReferenceName": "delete_example", - "inputParameters": { - "http_request": { - "uri": "https://jsonplaceholder.typicode.com/posts/1", - "method": "DELETE" - } - }, - "type": "HTTP" -} -``` - -### Best Practices - -1. Why are my HTTP tasks not getting picked up? - 1. We might have too many HTTP tasks in the queue. There is a concept called Isolation Groups that you can rely on - for prioritizing certain HTTP tasks over others. Read more here: [Isolation Groups](/configuration/isolationgroups.html) - diff --git a/docs/docs/reference-docs/human-task.md b/docs/docs/reference-docs/human-task.md deleted file mode 100644 index 9210de80d..000000000 --- a/docs/docs/reference-docs/human-task.md +++ /dev/null @@ -1,25 +0,0 @@ ---- -sidebar_position: 1 ---- - -# Human -```json -"type" : "HUMAN" -``` -### Introduction - -HUMAN is used when the workflow needs to be paused for an external signal by a human to continue. - -### Use Cases -HUMAN is used when the workflow needs to wait and pause for human intervention -(like manual approval) or an event coming from external source such as Kafka, SQS or Conductor's internal queueing mechanism. - -Some use cases where HUMAN task is used: -1. To add a human approval task. When the task is approved/rejected by HUMAN task is updated using `POST /tasks` API to completion. - - -### Configuration -* taskType: HUMAN -* There are no other configurations required - - diff --git a/docs/docs/reference-docs/inline-task.md b/docs/docs/reference-docs/inline-task.md deleted file mode 100644 index 78f7ce8ae..000000000 --- a/docs/docs/reference-docs/inline-task.md +++ /dev/null @@ -1,76 +0,0 @@ ---- -sidebar_position: 11 ---- - -# Inline Task - -```json -"type": "INLINE" -``` -### Introduction - -Inline Task helps execute necessary logic at Workflow run-time, -using an evaluator. There are two supported evaluators as of now: - -### Configuration -| name | description | -|-------------|---------------------------------------------------| -| value-param | Use a parameter directly as the value | -| javascript | Evaluate Javascript expressions and compute value | - - -### Use Cases - -Consider a scenario, we have to run simple evaluations in -Conductor server while creating Workers. Inline task can be used to run these -evaluations using an evaluator engine. - -### Example 1 - -```json -{ - "name": "inline_task_example", - "taskReferenceName": "inline_task_example", - "type": "INLINE", - "inputParameters": { - "value": "${workflow.input.value}", - "evaluatorType": "javascript", - "expression": "function e() { if ($.value == 1){return {\"result\": true}} else { return {\"result\": false}}} e();" - } -} -``` - -Following are the parameters in the above example : - -1. `"evaluatorType"` - Type of the evaluator. -Supported evaluators: value-param, javascript which evaluates -javascript expression. - -2. `"expression"` - Expression associated with the type of evaluator. -For javascript evaluator, Javascript evaluation engine is used to -evaluate expression defined as a string. Must return a value. - -Besides expression, any of the properties in the input values is accessible as `$.value` for the expression -to evaluate. - -The task output can then be referenced in downstream tasks -like: `"${inline_test.output.result}"` - -### Example 2 - -Perhaps a weather API sometimes returns Celcius, and sometimes returns Fahrenheit temperature values. This task ensures that the downstream tasks ONLY receive Celcius values: - -``` -{ - "name": "INLINE_TASK", - "taskReferenceName": "inline_test", - "type": "INLINE", - "inputParameters": { - "scale": "${workflow.input.tempScale}", - "temperature": "${workflow.input.temperature}", - "evaluatorType": "javascript", - "expression": "function SIvaluesOnly(){if ($.scale === "F"){ centigrade = ($.temperature -32)*5/9; return {temperature: centigrade} } else { return - {temperature: $.temperature} }} SIvaluesOnly();" - } -} -``` diff --git a/docs/docs/reference-docs/wait-task.md b/docs/docs/reference-docs/wait-task.md deleted file mode 100644 index 818a22b61..000000000 --- a/docs/docs/reference-docs/wait-task.md +++ /dev/null @@ -1,74 +0,0 @@ ---- -sidebar_position: 1 ---- - -# Wait -```json -"name": "waiting_task", -"taskReferenceName":"waiting_task_ref", -"type" : "WAIT" -``` -## Introduction - -WAIT is used when the workflow needs to be paused for an external signal to continue. - -## Use Cases -WAIT is used when the workflow needs to wait and pause for an external signal such as a human intervention -(like manual approval) or an event coming from external source such as Kafka, SQS or Conductor's internal queueing mechanism. - -Some use cases where WAIT task is used: - -1. Wait for a certain amount of time (e.g. 2 minutes) or until a certain date time (e.g. 12/25/2022 00:00) -2. To wait for and external signal coming from an event queue mechanism supported by Conductor - -## Configuration -* taskType: WAIT -* Wait for a specific amount of time -format: short: **D**d**H**h**M**m or full: **D**days**H**hours**M**minutes -* The following are the accepted units: *days*, *d*, *hrs*, *hours*, *h*, *minutes*, *mins*, *m*, *seconds*, *secs*, *s* -```json -{ - "taskType": "WAIT", - "inputParameters": { - "duration": "2 days 3 hours" - } -} -``` -* Wait until specific date/time -* e.g. the following Wait task remains blocked until Dec 25, 2022 9am PST -* The date/time can be supplied in one of the following formats: -**yyyy-MM-dd HH:mm**, **yyyy-MM-dd HH:mm**, **yyyy-MM-dd** -```json -{ - "name":"wait_until_date", - "taskReferenceName":"wait_until_date_ref", - "taskType": "WAIT", - "inputParameters": { - "until": "2022-12-25 09:00 PST" - } -} -``` - -## Ending a WAIT when there is no time duration specified - -To conclude a WAIT task, there are three endpoints that can be used. -You'll need the ```workflowId```, ```taskRefName``` or ```taskId``` and the task status (generally ```COMPLETED``` or ```FAILED```). - -1. POST ```/api/tasks``` -2. POST ```api/queue/update/{workflowId}/{taskRefName}/{status}``` -3. POST ```api/queue/update/{workflowId}/task/{taskId}/{status}``` - -Any parameter that is sent in the body of the POST message will be repeated as the output of the task. For example, if we send a COMPLETED message as follows: - -```bash -curl -X "POST" "https://play.orkes.io/api/queue/update/{workflowId}/waiting_around_ref/COMPLETED" -H 'Content-Type: application/json' -d '{"data_key":"somedatatoWait1","data_key2":"somedatatoWAit2"}' -``` - -The output of the task will be: - -```json -{ - "data_key":"somedatatoWait1", - "data_key2":"somedatatoWAit2" -} -``` \ No newline at end of file diff --git a/docs/docs/reference-docs/annotation-processor.md b/docs/documentation/advanced/annotation-processor.md similarity index 85% rename from docs/docs/reference-docs/annotation-processor.md rename to docs/documentation/advanced/annotation-processor.md index 7ebfe84cf..34acb8e31 100644 --- a/docs/docs/reference-docs/annotation-processor.md +++ b/docs/documentation/advanced/annotation-processor.md @@ -1,17 +1,10 @@ # Annotation Processor -- Original Author: Vicent Martí - https://github.com/vmg -- Original Repo: https://github.com/vmg/protogen - This module is strictly for code generation tasks during builds based on annotations. Currently supports `protogen` ### Usage -See example below - -### Example - This is an actual example of this module which is implemented in common/build.gradle ```groovy diff --git a/docs/documentation/advanced/archival-of-workflows.md b/docs/documentation/advanced/archival-of-workflows.md new file mode 100644 index 000000000..89d0e1bf7 --- /dev/null +++ b/docs/documentation/advanced/archival-of-workflows.md @@ -0,0 +1,13 @@ +# Archiving Workflows + +Conductor has support for archiving workflow upon termination or completion. Enabling this will delete the workflow from the configured database, but leave the associated data in Elasticsearch so it is still searchable. + +To enable, set the `conductor.workflow-status-listener.type` property to `archive`. + +A number of additional properties are available to control archival. + +| Property | Default Value | Description | +| ----------------------------------------------------------------------- | ------------- | ---------------------------------------------------------------------------------------------------------- | +| conductor.workflow-status-listener.archival.ttlDuration | 0s | The time to live in seconds for workflow archiving module. Currently, only RedisExecutionDAO supports this | +| conductor.workflow-status-listener.archival.delayQueueWorkerThreadCount | 5 | The number of threads to process the delay queue in workflow archival | +| conductor.workflow-status-listener.archival.delaySeconds | 60 | The time to delay the archival of workflow | diff --git a/docs/docs/reference-docs/azureblob-storage.md b/docs/documentation/advanced/azureblob-storage.md similarity index 100% rename from docs/docs/reference-docs/azureblob-storage.md rename to docs/documentation/advanced/azureblob-storage.md diff --git a/docs/docs/extend.md b/docs/documentation/advanced/extend.md similarity index 87% rename from docs/docs/extend.md rename to docs/documentation/advanced/extend.md index 82895f667..62bffa0b6 100644 --- a/docs/docs/extend.md +++ b/docs/documentation/advanced/extend.md @@ -59,3 +59,9 @@ for eg., with Dynomite and Redlock: * Implement ```Lock``` interface. * Add a binding similar to [this](https://github.com/Netflix/conductor/blob/master/server/src/main/java/com/netflix/conductor/bootstrap/ModulesProvider.java#L115-L129) * Enable locking service: ```conductor.app.workflowExecutionLockEnabled: true``` + +## Event Handling +Provide the implementation of [EventQueueProvider](https://github.com/Netflix/conductor/blob/master/core/src/main/java/com/netflix/conductor/core/events/EventQueueProvider.java). + +E.g. SQS Queue Provider: +[SQSEventQueueProvider.java ](https://github.com/Netflix/conductor/blob/master/contribs/src/main/java/com/netflix/conductor/core/events/sqs/SQSEventQueueProvider.java) diff --git a/docs/docs/externalpayloadstorage.md b/docs/documentation/advanced/externalpayloadstorage.md similarity index 97% rename from docs/docs/externalpayloadstorage.md rename to docs/documentation/advanced/externalpayloadstorage.md index 52775e4ad..29e317258 100644 --- a/docs/docs/externalpayloadstorage.md +++ b/docs/documentation/advanced/externalpayloadstorage.md @@ -50,7 +50,7 @@ Set the following property in the JVM system properties: conductor.external-payload-storage.type=S3 ``` -!!!note +!!! note This [implementation](https://github.com/Netflix/conductor/blob/master/core/src/main/java/com/netflix/conductor/core/utils/S3PayloadStorage.java#L44-L45) assumes that S3 access is configured on the instance. Set the following properties to the desired values in the JVM system properties: @@ -93,13 +93,13 @@ The payloads will be stored as done in [Amazon S3](https://github.com/Netflix/co Frinx provides an implementation of [PostgreSQL Storage](https://www.postgresql.org/) used to externalize large payload storage. !!!note -This implementation assumes that you have an [PostgreSQL database server with all required credentials](https://jdbc.postgresql.org/documentation/94/connect.html). + This implementation assumes that you have an [PostgreSQL database server with all required credentials](https://jdbc.postgresql.org/documentation/94/connect.html). Set the following properties to your application.properties: | Property | Description | default value | |-------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------| -| conductor.external-payload-storage.postgres.conductor-url | URL, that can be used to pull the json configurations, that will be downloaded from PostgreSQL to the conductor server. For example: for local development it is `http://localhost:8080` | `""` | +| conductor.external-payload-storage.postgres.conductor-url | URL, that can be used to pull the json configurations, that will be downloaded from PostgreSQL to the conductor server. For example: for local development it is `{{ server_host }}` | `""` | | conductor.external-payload-storage.postgres.url | PostgreSQL database connection URL. Required to connect to database. | | | conductor.external-payload-storage.postgres.username | Username for connecting to PostgreSQL database. Required to connect to database. | | | conductor.external-payload-storage.postgres.password | Password for connecting to PostgreSQL database. Required to connect to database. | | diff --git a/docs/docs/configuration/isolationgroups.md b/docs/documentation/advanced/isolationgroups.md similarity index 100% rename from docs/docs/configuration/isolationgroups.md rename to docs/documentation/advanced/isolationgroups.md diff --git a/docs/docs/reference-docs/redis.md b/docs/documentation/advanced/redis.md similarity index 100% rename from docs/docs/reference-docs/redis.md rename to docs/documentation/advanced/redis.md diff --git a/docs/documentation/api/index.md b/docs/documentation/api/index.md new file mode 100644 index 000000000..e798090e7 --- /dev/null +++ b/docs/documentation/api/index.md @@ -0,0 +1,13 @@ +# API Specification + +See the following sections for API endpoint documentation. + +- [Metadata API](metadata.md) +- [Start Workflow API](startworkflow.md) +- [Workflow API](workflow.md) +- [Task API](task.md) + +[Task Domains](taskdomains.md) can be used to address tasks to specific pools of workers at runtime. + +## Swagger UI +As an alternative resource, the [Swagger UI]({{ server_host }}/swagger_ui/index.html) will always have the definitive interface description. diff --git a/docs/documentation/api/metadata.md b/docs/documentation/api/metadata.md new file mode 100644 index 000000000..df2e4a4ad --- /dev/null +++ b/docs/documentation/api/metadata.md @@ -0,0 +1,18 @@ +# Metadata API + +## Workflow Metadata +| Endpoint | Description | Input | +|------------------------------------------|----------------------------------|----------------------------------------------------------------| +| `GET {{ api_prefix }}/metadata/workflow` | Get all the workflow definitions | n/a | +| `POST {{ api_prefix }}/metadata/workflow` | Register new workflow | [Workflow Definition](../configuration/workflowdef/index.md) | +| `PUT {{ api_prefix }}/metadata/workflow` | Register/Update new workflows | List of [Workflow Definition](../configuration/workflowdef/index.md) | +| `GET {{ api_prefix }}/metadata/workflow/{name}?version=` | Get the workflow definitions | workflow name, version (optional) | + +## Task Metadata +| Endpoint | Description | Input | +|------------------------------------------|----------------------------------|----------------------------------------------------------------| +| `GET {{ api_prefix }}/metadata/taskdefs` | Get all the task definitions | n/a | +| `GET {{ api_prefix }}/metadata/taskdefs/{taskType}` | Retrieve task definition | Task Name | +| `POST {{ api_prefix }}/metadata/taskdefs` | Register new task definitions | List of [Task Definitions](../configuration/taskdef.md) | +| `PUT {{ api_prefix }}/metadata/taskdefs` | Update a task definition | A [Task Definition](../configuration/taskdef.md) | +| `DELETE {{ api_prefix }}/metadata/taskdefs/{taskType}` | Delete a task definition | Task Name | diff --git a/docs/docs/gettingstarted/startworkflow.md b/docs/documentation/api/startworkflow.md similarity index 75% rename from docs/docs/gettingstarted/startworkflow.md rename to docs/documentation/api/startworkflow.md index 2b285c0a7..eb121e4da 100644 --- a/docs/docs/gettingstarted/startworkflow.md +++ b/docs/documentation/api/startworkflow.md @@ -1,30 +1,40 @@ -# Starting a Workflow -## Start Workflow Endpoint -When starting a Workflow execution with a registered definition, `/workflow` accepts following parameters: +# Start Workflow API + +## API Parameters +When starting a Workflow execution with a registered definition, `{{ api_prefix }}/workflow` accepts following parameters in the `POST` payload: | Field | Description | Notes | |:--------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------|:--------------------------------------------------------------------------------------------------------| | name | Name of the Workflow. MUST be registered with Conductor before starting workflow | | | version | Workflow version | defaults to latest available version | -| input | JSON object with key value params, that can be used by downstream tasks | See [Wiring Inputs and Outputs](/configuration/workflowdef.html#wiring-inputs-and-outputs) for details | +| input | JSON object with key value params, that can be used by downstream tasks | See [Wiring Inputs and Outputs](../configuration/workflowdef/index.md#wiring-inputs-and-outputs) for details | | correlationId | Unique Id that correlates multiple Workflow executions | optional | -| taskToDomain | See [Task Domains](/configuration/taskdomains.html) for more information. | optional | -| workflowDef | An adhoc [Workflow Definition](/configuration/workflowdef.html) to run, without registering. See [Dynamic Workflows](#dynamic-workflows). | optional | -| externalInputPayloadStoragePath | This is taken care of by Java client. See [External Payload Storage](/externalpayloadstorage.html) for more info. | optional | +| taskToDomain | See [Task Domains](taskdomains.md) for more information. | optional | +| workflowDef | An adhoc [Workflow Definition](../configuration/workflowdef/index.md) to run, without registering. See [Dynamic Workflows](#dynamic-workflows). | optional | +| externalInputPayloadStoragePath | This is taken care of by Java client. See [External Payload Storage](../advanced/externalpayloadstorage.md) for more info. | optional | | priority | Priority level for the tasks within this workflow execution. Possible values are between 0 - 99. | optional | -**Example:** +## Output +On success, this API returns the ID of the workflow. -Send a `POST` request to `/workflow` with payload like: -```json + +## Basic Example + +`POST {{ server_host }}{{ api_prefix }}/workflow` with payload body: + +```js { - "name": "encode_and_deploy", - "version": 1, - "correlationId": "my_unique_correlation_id", - "input": { - "param1": "value1", - "param2": "value2" - } + "name": "myWorkflow", // Name of the workflow + "version": 1, // Version + "correlationId": "corr1", // Correlation Id + "priority": 1, // Priority + "input": { // Input Value Map + "param1": "value1", + "param2": "value2" + }, + "taskToDomain": { + // Task to domain map + } } ``` @@ -36,7 +46,7 @@ This enables you to provide a workflow definition embedded with the required tas **Example:** -Send a `POST` request to `/workflow` with payload like: +Send a `POST` request to `{{ api_prefix }}/workflow` with payload like: ```json { "name": "my_adhoc_unregistered_workflow", diff --git a/docs/documentation/api/task.md b/docs/documentation/api/task.md new file mode 100644 index 000000000..f12a92b7a --- /dev/null +++ b/docs/documentation/api/task.md @@ -0,0 +1,37 @@ +# Task API + +## Manage Tasks +| Endpoint | Description | +|-------------------------------------------------------|-------------------------------------------------------| +| `GET {{ api_prefix }}/tasks/{taskId}` | Get task details. | +| `GET {{ api_prefix }}/tasks/queue/all` | List the pending task sizes. | +| `GET {{ api_prefix }}/tasks/queue/all/verbose` | Same as above, includes the size per shard | +| `GET {{ api_prefix }}/tasks/queue/sizes?taskType=&taskType=&taskType` | Return the size of pending tasks for given task types | +||| + +## Polling, Ack and Update Task +These endpoints are used by the worker to poll for task, send ack (after polling) and finally updating the task result. They typically should not be invoked manually. + +| Endpoint | Description | +|---------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `GET {{ api_prefix }}/tasks/poll/{taskType}?workerid=&domain=` | Poll for a task. `workerid` identifies the worker that polled for the job and `domain` allows the poller to poll for a task in a specific domain | +| `GET {{ api_prefix }}/tasks/poll/batch/{taskType}?count=&timeout=&workerid=&domain` | Poll for a task in a batch specified by `count`. This is a long poll and the connection will wait until `timeout` or if there is at-least 1 item available, whichever comes first.`workerid` identifies the worker that polled for the job and `domain` allows the poller to poll for a task in a specific domain | +| `POST {{ api_prefix }}/tasks` | Update the result of task execution. See the schema below. | + + +### Schema for updating Task Result +```json +{ + "workflowInstanceId": "Workflow Instance Id", + "taskId": "ID of the task to be updated", + "reasonForIncompletion" : "If failed, reason for failure", + "callbackAfterSeconds": 0, + "status": "IN_PROGRESS|FAILED|COMPLETED", + "outputData": { + //JSON document representing Task execution output + } + +} +``` +!!!Info "Acknowledging tasks after poll" + If the worker fails to ack the task after polling, the task is re-queued and put back in queue and is made available during subsequent poll. diff --git a/docs/docs/configuration/taskdomains.md b/docs/documentation/api/taskdomains.md similarity index 61% rename from docs/docs/configuration/taskdomains.md rename to docs/documentation/api/taskdomains.md index 2d3f3d7da..28cef9d50 100644 --- a/docs/docs/configuration/taskdomains.md +++ b/docs/documentation/api/taskdomains.md @@ -1,9 +1,9 @@ # Task Domains -Task domains helps support task development. The idea is same “task definition” can be implemented in different “domains”. A domain is some arbitrary name that the developer controls. So when the workflow is started, the caller can specify, out of all the tasks in the workflow, which tasks need to run in a specific domain, this domain is then used to poll for task on the client side to execute it. +Task domains helps support task development. The idea is same "task definition" can be implemented in different "domains". A domain is some arbitrary name that the developer controls. So when the workflow is started, the caller can specify, out of all the tasks in the workflow, which tasks need to run in a specific domain, this domain is then used to poll for task on the client side to execute it. -As an example if a workflow (WF1) has 3 tasks T1, T2, T3. The workflow is deployed and working fine, which means there are T2 workers polling and executing. If you modify T2 and run it locally there is no guarantee that your modified T2 worker will get the task that you are looking for as it coming from the general T2 queue. “Task Domain” feature solves this problem by splitting the T2 queue by domains, so when the app polls for task T2 in a specific domain, it get the correct task. +As an example if a workflow (WF1) has 3 tasks T1, T2, T3. The workflow is deployed and working fine, which means there are T2 workers polling and executing. If you modify T2 and run it locally there is no guarantee that your modified T2 worker will get the task that you are looking for as it coming from the general T2 queue. "Task Domain" feature solves this problem by splitting the T2 queue by domains, so when the app polls for task T2 in a specific domain, it get the correct task. -When starting a workflow multiple domains can be specified as a fall backs, for example "domain1,domain2". Conductor keeps track of last polling time for each task, so in this case it checks if the there are any active workers for "domain1" then the task is put in "domain1", if not then the same check is done for the next domain in sequence "domain2" and so on. +When starting a workflow multiple domains can be specified as a fall backs, for example "domain1,domain2". Conductor keeps track of last polling time for each task, so in this case it checks if the there are any active workers (workers polled at least once in a 10 second window) for "domain1" then the task is put in "domain1", if not then the same check is done for the next domain in sequence "domain2" and so on. If no workers are active for the domains provided: @@ -41,37 +41,37 @@ If you are using the java client then a simple property change will force TaskR conductor.worker.T2.domain=mydomain //Task T2 needs to poll for domain "mydomain" ``` #### REST call -`GET /tasks/poll/batch/T2?workerid=myworker&domain=mydomain` -`GET /tasks/poll/T2?workerid=myworker&domain=mydomain` +`GET {{ api_prefix }}/tasks/poll/batch/T2?workerid=myworker&domain=mydomain` +`GET {{ api_prefix }}/tasks/poll/T2?workerid=myworker&domain=mydomain` ### Change the start workflow call When starting the workflow, make sure the task to domain mapping is passes #### Java Client -``` - Map input = new HashMap<>(); - input.put("wf_input1", "one”); - - Map taskToDomain = new HashMap<>(); - taskToDomain.put("T2", "mydomain"); - - // Other options ... - // taskToDomain.put("*", "mydomain, NO_DOMAIN") - // taskToDomain.put("T2", "mydomain, fallbackDomain1, fallbackDomain2") - - StartWorkflowRequest swr = new StartWorkflowRequest(); - swr.withName(“myWorkflow”) - .withCorrelationId(“corr1”) - .withVersion(1) - .withInput(input) - .withTaskToDomain(taskToDomain); - - wfclient.startWorkflow(swr); +```java +{Map input = new HashMap<>(); +input.put("wf_input1", "one"); + +Map taskToDomain = new HashMap<>(); +taskToDomain.put("T2", "mydomain"); + +// Other options ... +// taskToDomain.put("*", "mydomain, NO_DOMAIN") +// taskToDomain.put("T2", "mydomain, fallbackDomain1, fallbackDomain2") + +StartWorkflowRequest swr = new StartWorkflowRequest(); +swr.withName("myWorkflow") + .withCorrelationId("corr1") + .withVersion(1) + .withInput(input) + .withTaskToDomain(taskToDomain); + +wfclient.startWorkflow(swr); ``` #### REST call -`POST /workflow` +`POST {{ api_prefix }}/workflow` ```json { diff --git a/docs/documentation/api/workflow.md b/docs/documentation/api/workflow.md new file mode 100644 index 000000000..2c2029378 --- /dev/null +++ b/docs/documentation/api/workflow.md @@ -0,0 +1,86 @@ +# Workflow API + +## Retrieve Workflows +| Endpoint | Description | +|-----------------------------------------------------------------------------|-----------------------------------------------| +| `GET {{ api_prefix }}/workflow/{workflowId}?includeTasks=true | false` |Get Workflow State by workflow Id. If includeTasks is set, then also includes all the tasks executed and scheduled.| +| `GET {{ api_prefix }}/workflow/running/{name}` | Get all the running workflows of a given type | +| `GET {{ api_prefix }}/workflow/running/{name}/correlated/{correlationId}?includeClosed=true | false&includeTasks=true |false`|Get all the running workflows filtered by correlation Id. If includeClosed is set, also includes workflows that have completed running.| +| `GET {{ api_prefix }}/workflow/search` | Search for workflows. See Below. | + + +## Workflow Search +Conductor uses Elasticsearch for indexing workflow execution and is used by search APIs. + +`GET {{ api_prefix }}/workflow/search?start=&size=&sort=&freeText=&query=` + +| Parameter | Description | +|-----------|------------------------------------------------------------------------------------------------------------------| +| start | Page number. Defaults to 0 | +| size | Number of results to return | +| sort | Sorting. Format is: `ASC:` or `DESC:` to sort in ascending or descending order by a field | +| freeText | Elasticsearch supported query. e.g. workflowType:"name_of_workflow" | +| query | SQL like where clause. e.g. workflowType = 'name_of_workflow'. Optional if freeText is provided. | + +### Output +Search result as described below: +```json +{ + "totalHits": 0, + "results": [ + { + "workflowType": "string", + "version": 0, + "workflowId": "string", + "correlationId": "string", + "startTime": "string", + "updateTime": "string", + "endTime": "string", + "status": "RUNNING", + "input": "string", + "output": "string", + "reasonForIncompletion": "string", + "executionTime": 0, + "event": "string" + } + ] +} +``` + +## Manage Workflows +| Endpoint | Description | +|-----------------------------------------------------------|----------------------------------------------------------------------------------------------------| +| `PUT {{ api_prefix }}/workflow/{workflowId}/pause` | Pause. No further tasks will be scheduled until resumed. Currently running tasks are not paused. | +| `PUT {{ api_prefix }}/workflow/{workflowId}/resume` | Resume normal operations after a pause. | +| `POST {{ api_prefix }}/workflow/{workflowId}/rerun` | See Below. | +| `POST {{ api_prefix }}/workflow/{workflowId}/restart` | Restart workflow execution from the start. Current execution history is wiped out. | +| `POST {{ api_prefix }}/workflow/{workflowId}/retry` | Retry the last failed task. | +| `PUT {{ api_prefix }}/workflow/{workflowId}/skiptask/{taskReferenceName}` | See below. | +| `DELETE {{ api_prefix }}/workflow/{workflowId}` | Terminates the running workflow. | +| `DELETE {{ api_prefix }}/workflow/{workflowId}/remove` | Deletes the workflow from system. Use with caution. | + +### Rerun +Re-runs a completed workflow from a specific task. + +`POST {{ api_prefix }}/workflow/{workflowId}/rerun` + +```json +{ + "reRunFromWorkflowId": "string", + "workflowInput": {}, + "reRunFromTaskId": "string", + "taskInput": {} +} +``` + +###Skip Task + +Skips a task execution (specified as `taskReferenceName` parameter) in a running workflow and continues forward. +Optionally updating task's input and output as specified in the payload. +`PUT {{ api_prefix }}/workflow/{workflowId}/skiptask/{taskReferenceName}?workflowId=&taskReferenceName=` +```json +{ + "taskInput": {}, + "taskOutput": {} +} +``` diff --git a/docs/docs/how-tos/clojure-sdk.md b/docs/documentation/clientsdks/clojure-sdk.md similarity index 96% rename from docs/docs/how-tos/clojure-sdk.md rename to docs/documentation/clientsdks/clojure-sdk.md index 07875cc0a..9bd1b3940 100644 --- a/docs/docs/how-tos/clojure-sdk.md +++ b/docs/documentation/clientsdks/clojure-sdk.md @@ -1,4 +1,4 @@ -# Conductor Clojure +# Clojure SDK Software Development Kit for Netflix Conductor, written on and providing support for Clojure. @@ -13,7 +13,7 @@ https://clojars.org/io.orkes/conductor-clojure ```clojure (def options { - :url "http://localhost:8080/api/" ;; Conductor Server Path + :url "{{ server_host }}{{ api_prefix }}/" ;; Conductor Server Path :app-key "THIS-IS-SOME-APP-KEY" ;; Optional if using Orkes Conductor :app-secret "THIS-IS-SOME-APP-SECRET" ;; Optional if using Orkes Conductor } ) @@ -44,7 +44,7 @@ https://clojars.org/io.orkes/conductor-clojure ;; Will Register a workflow that uses the above task returns nil (metadata/register-workflow-def options { :name "cool_clj_workflow" - :description "created programmatically from clj" + :description "created programatically from clj" :version 1 :tasks [ { :name "cool_clj_task" @@ -69,7 +69,7 @@ https://clojars.org/io.orkes/conductor-clojure (list { :name "cool_clj_task" :execute (fn [someData] - [:completed {:message "Hi From Clj i was created programmatically"}]) + [:completed {:message "Hi From Clj i was created programatically"}]) }) options )) @@ -81,7 +81,7 @@ https://clojars.org/io.orkes/conductor-clojure Options are a map with optional paremeters ``` (def options { - :url "http://localhost:8080/api/" ;; Api url (Optional will default to "http://localhost:8080") + :url "{{ server_host }}{{ api_prefix }}/" ;; Api url (Optional will default to "{{ server_host }}") :app-key "THIS-IS-SOME-APP-KEY" ;; Application Key (This is only relevant if you are using Orkes Conductor) :app-secret "THIS-IS-SOME-APP-SECRET" ;; Application Secret (This is only relevant if you are using Orkes Conductor) } ) @@ -126,7 +126,7 @@ Takes the option map and a list/vector of tasks to register. on success it will ```clojure (metadata/register-workflow-def options { :name "cool_clj_workflow_2" - :description "created programmatically from clj" + :description "created programatically from clj" :version 1 :tasks [ { :name "cool_clj_task_b" @@ -186,7 +186,7 @@ The client namespace holds the function to start a workflow and running a worker (list { :name "cool_clj_task" :execute (fn [someData] - [:completed {:message "Hi From Clj i was created programmatically"}]) + [:completed {:message "Hi From Clj i was created programatically"}]) }) options )) @@ -205,7 +205,7 @@ Will map a java map to a clojure map which may come in handy for workers impleme ``` clojure (metadata/register-workflow-def options {:name "simple_wf" - :description "created programmatically from clj" + :description "created programatically from clj" :version 1 :tasks [{:name "simplest_task" :taskReferenceName "repl_task_ref" diff --git a/docs/docs/how-tos/csharp-sdk.md b/docs/documentation/clientsdks/csharp-sdk.md similarity index 99% rename from docs/docs/how-tos/csharp-sdk.md rename to docs/documentation/clientsdks/csharp-sdk.md index dc124b267..964c38f70 100644 --- a/docs/docs/how-tos/csharp-sdk.md +++ b/docs/documentation/clientsdks/csharp-sdk.md @@ -1,4 +1,4 @@ -# Netflix Conductor Client C# SDK +# C# SDK `conductor-csharp` repository provides the client SDKs to build Task Workers and Clients in C# diff --git a/docs/docs/how-tos/go-sdk.md b/docs/documentation/clientsdks/go-sdk.md similarity index 98% rename from docs/docs/how-tos/go-sdk.md rename to docs/documentation/clientsdks/go-sdk.md index 9aacea76e..cd2602c75 100644 --- a/docs/docs/how-tos/go-sdk.md +++ b/docs/documentation/clientsdks/go-sdk.md @@ -1,4 +1,4 @@ -# Netflix Conductor Go SDK +# Go SDK The code for the Golang SDk is available on [Github](https://github.com/conductor-sdk/conductor-go). Please feel free to file PRs, issues, etc. there. diff --git a/docs/documentation/clientsdks/index.md b/docs/documentation/clientsdks/index.md new file mode 100644 index 000000000..6cb8f88be --- /dev/null +++ b/docs/documentation/clientsdks/index.md @@ -0,0 +1,10 @@ +# Client SDKs +Conductor tasks that are executed by remote workers communicate over HTTP endpoints/gRPC to poll for the task and update the status of the execution. The follow SDKs are provided for implementing Conductor workers. + +* [Java](java-sdk.md) +* [Clojure](clojure-sdk.md) +* [C#](csharp-sdk.md) +* [Go](go-sdk.md) +* [Python](python-sdk.md) + +The non-Java Conductor SDKs are hosted on a separate GitHub repository: [github.com/conductor-sdk](https://github.com/conductor-sdk). Contributions from the community are encouraged! diff --git a/docs/docs/how-tos/java-sdk.md b/docs/documentation/clientsdks/java-sdk.md similarity index 88% rename from docs/docs/how-tos/java-sdk.md rename to docs/documentation/clientsdks/java-sdk.md index 65443c757..b408438f9 100644 --- a/docs/docs/how-tos/java-sdk.md +++ b/docs/documentation/clientsdks/java-sdk.md @@ -1,4 +1,4 @@ -# Conductor Java SDK +# Java SDK Conductor provides the following java clients to interact with the various APIs @@ -8,24 +8,23 @@ Conductor provides the following java clients to interact with the various APIs | Workflow Client | Start a new workflow / Get execution status of a workflow | | Task Client | Poll for task / Update task result after execution / Get status of a task | -## Java - -#### Worker +## Worker Conductor provides an automated framework to poll for tasks, manage the execution thread and update the status of the execution back to the server. Implement the [Worker](https://github.com/Netflix/conductor/blob/main/client/src/main/java/com/netflix/conductor/client/worker/Worker.java) interface to execute the task. -#### TaskRunnerConfigurer +## TaskRunnerConfigurer The TaskRunnerConfigurer can be used to register the worker(s) and initialize the polling loop. Manages the task workers thread pool and server communication (poll and task update). -Use the [Builder](https://github.com/Netflix/conductor/blob/master/client/src/main/java/com/netflix/conductor/client/automator/TaskRunnerConfigurer.java#L62) to create an instance of the TaskRunnerConfigurer. The builder accepts the following parameters: +Use the [Builder](https://github.com/Netflix/conductor/blob/main/client/src/main/java/com/netflix/conductor/client/automator/TaskRunnerConfigurer.java#L62) to create an instance of the TaskRunnerConfigurer. The Builder constructor takes the following parameters. -Initialize the Builder with the following: +| Parameter | Description | +| ---------- | ------------------------------------------------------------- | +| TaskClient | TaskClient used to communicate to the Conductor server | +| Workers | Workers that will be used for polling work and task execution.| - TaskClient - | TaskClient used to communicate to the Conductor server | -| Workers | Workers that will be used for polling work and task execution. | +The builder accepts the following parameters: | Parameter | Description | Default | |--------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------| @@ -41,7 +40,7 @@ Once an instance is created, call `init()` method to initialize the TaskPollExec !!! tip "Note" To ensure that the TaskRunnerConfigurer stops polling for tasks when the instance becomes unhealthy, call the provided `shutdown()` hook in a `PreDestroy` block. -**Properties** +## Properties The worker behavior can be further controlled by using these properties: | Property | Type | Description | Default | @@ -57,7 +56,7 @@ Further, these properties can be set either by Worker implementation or by setti | `conductor.worker.` | Applies to ALL the workers in the JVM. | | `conductor.worker..` | Applies to the specified worker. Overrides the global property. | -**Examples** +## Examples * [Sample Worker Implementation](https://github.com/Netflix/conductor/blob/main/client/src/test/java/com/netflix/conductor/client/sample/SampleWorker.java) * [Example](https://github.com/Netflix/conductor/blob/main/client/src/test/java/com/netflix/conductor/client/sample/Main.java) diff --git a/docs/docs/how-tos/python-sdk.md b/docs/documentation/clientsdks/python-sdk.md similarity index 95% rename from docs/docs/how-tos/python-sdk.md rename to docs/documentation/clientsdks/python-sdk.md index c245dee3e..45c9097c6 100644 --- a/docs/docs/how-tos/python-sdk.md +++ b/docs/documentation/clientsdks/python-sdk.md @@ -1,7 +1,3 @@ ---- -sidebar_position: 1 ---- - # Python SDK Software Development Kit for Netflix Conductor, written on and providing support for Python. @@ -88,14 +84,14 @@ The code for the Python SDk is available on [Github](https://github.com/conducto You should be able to access: * Conductor API: - * http://localhost:8080/swagger-ui/index.html + * {{ server_host }}/swagger-ui/index.html * Conductor UI: * http://localhost:5000 6. Create a `Task` within `Conductor`. Example: $ curl -X 'POST' \ - 'http://localhost:8080/api/metadata/taskdefs' \ + '{{ server_host }}{{ api_prefix }}/metadata/taskdefs' \ -H 'accept: */*' \ -H 'Content-Type: application/json' \ -d '[ @@ -115,7 +111,7 @@ The code for the Python SDk is available on [Github](https://github.com/conducto 7. Create a `Workflow` within `Conductor`. Example: $ curl -X 'POST' \ - 'http://localhost:8080/api/metadata/workflow' \ + '{{ server_host }}{{ api_prefix }}/metadata/workflow' \ -H 'accept: */*' \ -H 'Content-Type: application/json' \ -d '{ @@ -146,7 +142,7 @@ The code for the Python SDk is available on [Github](https://github.com/conducto 8. Start a new workflow: $ curl -X 'POST' \ - 'http://localhost:8080/api/workflow/workflow_with_python_task_example' \ + '{{ server_host }}{{ api_prefix }}/workflow/workflow_with_python_task_example' \ -H 'accept: text/plain' \ -H 'Content-Type: application/json' \ -d '{}' diff --git a/docs/docs/configuration/eventhandlers.md b/docs/documentation/configuration/eventhandlers.md similarity index 70% rename from docs/docs/configuration/eventhandlers.md rename to docs/documentation/configuration/eventhandlers.md index 8417d67c0..4a0c6edc8 100644 --- a/docs/docs/configuration/eventhandlers.md +++ b/docs/documentation/configuration/eventhandlers.md @@ -3,7 +3,7 @@ Eventing in Conductor provides for loose coupling between workflows and support This includes: -1. Being able to produce an event (message) in an external system like SQS or internal to Conductor. +1. Being able to produce an event (message) in an external system like SQS or internal to Conductor. 2. Start a workflow when a specific event occurs that matches the provided criteria. Conductor provides SUB_WORKFLOW task that can be used to embed a workflow inside parent workflow. Eventing supports provides similar capability without explicitly adding dependencies and provides **fire-and-forget** style integrations. @@ -11,7 +11,7 @@ Conductor provides SUB_WORKFLOW task that can be used to embed a workflow inside ## Event Task Event task provides ability to publish an event (message) to either Conductor or an external eventing system like SQS. Event tasks are useful for creating event based dependencies for workflows and tasks. -See [Event Task](/reference-docs/event-task.html) for documentation. +See [Event Task](workflowdef/systemtasks/event-task.md) for documentation. ## Event Handler Event handlers are listeners registered that executes an action when a matching event occurs. The supported actions are: @@ -22,10 +22,10 @@ Event handlers are listeners registered that executes an action when a matching Event Handlers can be configured to listen to Conductor Events or an external event like SQS. -### Configuration +## Configuration Event Handlers are configured via ```/event/``` APIs. -#### Structure: +### Structure ```json { "name" : "descriptive unique name", @@ -34,12 +34,11 @@ Event Handlers are configured via ```/event/``` APIs. "actions": ["see examples below"] } ``` -#### Condition -Condition is an expression that MUST evaluate to a boolean value. A Javascript like syntax is supported that can be used to evaluate condition based on the payload. +`condition` is an expression that MUST evaluate to a boolean value. A Javascript like syntax is supported that can be used to evaluate condition based on the payload. Actions are executed only when the condition evaluates to `true`. -**Examples** - +## Examples +### Condition Given the following payload in the message: ```json @@ -53,16 +52,19 @@ Given the following payload in the message: } ``` -|Expression|Result| -|---|---| -|`$.version > 1`|true| -|`$.version > 10`|false| -|`$.metadata.length == 300`|true| +The following expressions can be used in `condition` with the indicated results: + +| Expression | Result | +| -------------------------- | ------ | +| `$.version > 1` | true | +| `$.version > 10` | false | +| `$.metadata.length == 300` | true | ### Actions +Examples of actions that can be configured in the `actions` array: -**Start A Workflow** +**To start a workflow** ```json { @@ -77,7 +79,7 @@ Given the following payload in the message: } ``` -**Complete Task*** +**To complete a task** ```json { @@ -93,7 +95,7 @@ Given the following payload in the message: } ``` -**Fail Task*** +**To fail a task*** ```json { @@ -108,15 +110,8 @@ Given the following payload in the message: "expandInlineJSON": true } ``` -Input for starting a workflow and output when completing / failing task follows the same [expressions](/configuration/workflowdef.html#wiring-inputs-and-outputs) used for wiring workflow inputs. +Input for starting a workflow and output when completing / failing task follows the same [expressions](workflowdef/index.md#using-expressions) used for wiring task inputs. !!!info "Expanding stringified JSON elements in payload" `expandInlineJSON` property, when set to true will expand the inlined stringified JSON elements in the payload to JSON documents and replace the string value with JSON document. This feature allows such elements to be used with JSON path expressions. - -## Extending - -Provide the implementation of [EventQueueProvider](https://github.com/Netflix/conductor/blob/master/core/src/main/java/com/netflix/conductor/core/events/EventQueueProvider.java). - -SQS Queue Provider: -[SQSEventQueueProvider.java ](https://github.com/Netflix/conductor/blob/master/contribs/src/main/java/com/netflix/conductor/core/events/sqs/SQSEventQueueProvider.java) diff --git a/docs/documentation/configuration/taskdef.md b/docs/documentation/configuration/taskdef.md new file mode 100644 index 000000000..499c49b58 --- /dev/null +++ b/docs/documentation/configuration/taskdef.md @@ -0,0 +1,117 @@ +# Task Definition + +Task Definitions are used to register SIMPLE tasks (workers). Conductor maintains a registry of user task types. A task type MUST be registered before being used in a workflow. + +This should not be confused with [*Task Configurations*](./workflowdef/index.md#task-configurations) which are part of the Workflow Definition, and are iterated in the `tasks` property in the definition. + + +## Schema + +| Field | Type | Description | Notes | +| :-------------------------- | :----------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | :------------------------------------------------------------------------------- | +| name | string | Task Name. Unique name of the Task that resonates with its function. | Must be unique | +| description | string | Description of the task | Optional | +| retryCount | number | Number of retries to attempt when a Task is marked as failure | Defaults to 3 with maximum allowed capped at 10 | +| retryLogic | string (enum) | Mechanism for the retries | See [Retry Logic](#retry-logic) | +| retryDelaySeconds | number | Time to wait before retries | Defaults to 60 seconds | +| timeoutPolicy | string (enum) | Task's timeout policy | Defaults to `TIME_OUT_WF`; See [Timeout Policy](#timeout-policy) | +| timeoutSeconds | number | Time in seconds, after which the task is marked as `TIMED_OUT` if not completed after transitioning to `IN_PROGRESS` status for the first time | No timeouts if set to 0 | +| responseTimeoutSeconds | number | If greater than 0, the task is rescheduled if not updated with a status after this time (heartbeat mechanism). Useful when the worker polls for the task but fails to complete due to errors/network failure. | Defaults to 3600 | +| pollTimeoutSeconds | number | Time in seconds, after which the task is marked as `TIMED_OUT` if not polled by a worker | No timeouts if set to 0 | +| inputKeys | array of string(s) | Array of keys of task's expected input. Used for documenting task's input. | Optional. See [Using inputKeys and outputKeys](#using-inputkeys-and-outputkeys). | +| outputKeys | array of string(s) | Array of keys of task's expected output. Used for documenting task's output. | Optional. See [Using inputKeys and outputKeys](#using-inputkeys-and-outputkeys). | +| inputTemplate | object | Define default input values. | Optional. See [Using inputTemplate](#using-inputtemplate) | +| concurrentExecLimit | number | Number of tasks that can be executed at any given time | Optional | +| rateLimitFrequencyInSeconds | number | Sets the rate limit frequency window. | Optional. See [Task Rate limits](#task-rate-limits) | +| rateLimitPerFrequency | number | Sets the max number of tasks that can be given to workers within window. | Optional. See [Task Rate limits](#task-rate-limits) below | +| ownerEmail | string | Email address of the team that owns the task | Required | + +### Retry Logic + +* FIXED: Reschedule the task after `retryDelaySeconds` +* EXPONENTIAL_BACKOFF: Reschedule the task after `retryDelaySeconds * (2 ^ attemptNumber)` +* LINEAR_BACKOFF: Reschedule after `retryDelaySeconds * backoffRate * attemptNumber` + +### Timeout Policy + +* RETRY: Retries the task again +* TIME_OUT_WF: Workflow is marked as TIMED_OUT and terminated. This is the default value. +* ALERT_ONLY: Registers a counter (task_timeout) + +### Task Concurrent Execution Limits + +`concurrentExecLimit` limits the number of simultaneous Task executions at any point. + +**Example** +You have 1000 task executions waiting in the queue, and 1000 workers polling this queue for tasks, but if you have set `concurrentExecLimit` to 10, only 10 tasks would be given to workers (which would lead to starvation). If any of the workers finishes execution, a new task(s) will be removed from the queue, while still keeping the current execution count to 10. + +### Task Rate Limits + +!!! note "Rate Limiting" + Rate limiting is only supported for the Redis-persistence module and is not available with other persistence layers. + +* `rateLimitFrequencyInSeconds` and `rateLimitPerFrequency` should be used together. +* `rateLimitFrequencyInSeconds` sets the "frequency window", i.e the `duration` to be used in `events per duration`. Eg: 1s, 5s, 60s, 300s etc. +* `rateLimitPerFrequency`defines the number of Tasks that can be given to Workers per given "frequency window". No rate limit if set to 0. + +**Example** +Let's set `rateLimitFrequencyInSeconds = 5`, and `rateLimitPerFrequency = 12`. This means our frequency window is of 5 seconds duration, and for each frequency window, Conductor would only give 12 tasks to workers. So, in a given minute, Conductor would only give 12*(60/5) = 144 tasks to workers irrespective of the number of workers that are polling for the task. + +Note that unlike `concurrentExecLimit`, rate limiting doesn't take into account tasks already in progress/completed. Even if all the previous tasks are executed within 1 sec, or would take a few days, the new tasks are still given to workers at configured frequency, 144 tasks per minute in above example. + + +### Using `inputKeys` and `outputKeys` + +* `inputKeys` and `outputKeys` can be considered as parameters and return values for the Task. +* Consider the task Definition as being represented by an interface: ```(value1, value2 .. valueN) someTaskDefinition(key1, key2 .. keyN);``` +* However, these parameters are not strictly enforced at the moment. Both `inputKeys` and `outputKeys` act as a documentation for task re-use. The tasks in workflow need not define all of the keys in the task definition. +* In the future, this can be extended to be a strict template that all task implementations must adhere to, just like interfaces in programming languages. + +### Using `inputTemplate` + +* `inputTemplate` allows to define default values, which can be overridden by values provided in Workflow. +* Eg: In your Task Definition, you can define your inputTemplate as: + +```json +"inputTemplate": { + "url": "https://some_url:7004" +} +``` + +* Now, in your workflow Definition, when using above task, you can use the default `url` or override with something else in the task's `inputParameters`. + +```json +"inputParameters": { + "url": "${workflow.input.some_new_url}" +} +``` + +## Complete Example +This is an example of a Task Definition for a worker implementation named `encode_task`. + +``` json +{ + "name": "encode_task", + "retryCount": 3, + "timeoutSeconds": 1200, + "inputKeys": [ + "sourceRequestId", + "qcElementType" + ], + "outputKeys": [ + "state", + "skipped", + "result" + ], + "timeoutPolicy": "TIME_OUT_WF", + "retryLogic": "FIXED", + "retryDelaySeconds": 600, + "responseTimeoutSeconds": 3600, + "pollTimeoutSeconds": 3600, + "concurrentExecLimit": 100, + "rateLimitFrequencyInSeconds": 60, + "rateLimitPerFrequency": 50, + "ownerEmail": "foo@bar.com", + "description": "Sample Encoding task" +} +``` diff --git a/docs/documentation/configuration/workflowdef/index.md b/docs/documentation/configuration/workflowdef/index.md new file mode 100644 index 000000000..d657d7be0 --- /dev/null +++ b/docs/documentation/configuration/workflowdef/index.md @@ -0,0 +1,302 @@ +# Workflow Definition + +The Workflow Definition contains all the information necessary to define the behavior of a workflow. The most important part of this definition is the `tasks` property, which is an array of [**Task Configurations**](#task-configurations). + +## Workflow Properties +| Field | Type | Description | Notes | +| :---------------------------- | :------------------------------- | :------------------------------------------------------------------------------------------------------------------------------ | :------------------------------------------------------------------------------------------------ | +| name | string | Name of the workflow | | +| description | string | Description of the workflow | Optional | +| version | number | Numeric field used to identify the version of the schema. Use incrementing numbers. | When starting a workflow execution, if not specified, the definition with highest version is used | +| tasks | array of object(s) | An array of task configurations. [Details](#task-configurations) | | +| inputParameters | array of string(s) | List of input parameters. Used for documenting the required inputs to workflow | Optional. | +| outputParameters | object | JSON template used to generate the output of the workflow | If not specified, the output is defined as the output of the _last_ executed task | +| inputTemplate | object | Default input values. See [Using inputTemplate](#default-input-with-inputtemplate) | Optional. | +| failureWorkflow | string | Workflow to be run on current Workflow failure. Useful for cleanup or post actions on failure. [Explanation](#failure-workflow) | Optional. | +| schemaVersion | number | Current Conductor Schema version. schemaVersion 1 is discontinued. | Must be 2 | +| restartable | boolean | Flag to allow Workflow restarts | Defaults to true | +| workflowStatusListenerEnabled | boolean | Enable status callback. [Explanation](#workflow-status-listener) | Defaults to false | +| ownerEmail | string | Email address of the team that owns the workflow | Required | +| timeoutSeconds | number | The timeout in seconds after which the workflow will be marked as `TIMED_OUT` if it hasn't been moved to a terminal state | No timeouts if set to 0 | +| timeoutPolicy | string ([enum](#timeout-policy)) | Workflow's timeout policy | Defaults to `TIME_OUT_WF` | + +### Failure Workflow + +The failure workflow gets the _original failed workflow’s input_ along with 3 additional items, + +* `workflowId` - The id of the failed workflow which triggered the failure workflow. +* `reason` - A string containing the reason for workflow failure. +* `failureStatus` - A string status representation of the failed workflow. +* `failureTaskId` - The id of the failed task of the workflow that triggered the failure workflow. + +### Timeout Policy + +* TIME_OUT_WF: Workflow is marked as TIMED_OUT and terminated +* ALERT_ONLY: Registers a counter (workflow_failure with status tag set to `TIMED_OUT`) + +### Workflow Status Listener +Setting the `workflowStatusListenerEnabled` field in your Workflow Definition to `true` enables notifications. + +To add a custom implementation of the Workflow Status Listener. Refer to [this](../../advanced/extend.md#workflow-status-listener) . + +The listener can be implemented in such a way as to either send a notification to an external system or to send an event on the conductor queue to complete/fail another task in another workflow as described [here](../../configuration/eventhandlers.md). + +### Default Input with `inputTemplate` + +* `inputTemplate` allows you to define default input values, which can optionally be overridden at runtime (when the workflow is invoked). +* Eg: In your Workflow Definition, you can define your inputTemplate as: + +```json +"inputTemplate": { + "url": "https://some_url:7004" +} +``` + +And `url` would be `https://some_url:7004` if no `url` was provided as input to your workflow. + + + + +## Task Configurations + +The `tasks` property in a Workflow Definition defines an array of *Task Configurations*. This is the blueprint for the workflow. Task Configurations can reference different types of Tasks. + +* Simple Tasks +* System Tasks +* Operators + +Note: Task Configuration should not be confused with **Task Definitions**, which are used to register SIMPLE (worker based) tasks. + +| Field | Type | Description | Notes | +| :---------------- | :------ | :--------------------------------------------------------------------------------------------------------------------------------------------- | :-------------------------------------------------------------------- | +| name | string | Name of the task. MUST be registered as a Task Type with Conductor before starting workflow | | +| taskReferenceName | string | Alias used to refer the task within the workflow. MUST be unique within workflow. | | +| type | string | Type of task. SIMPLE for tasks executed by remote workers, or one of the system task types | | +| description | string | Description of the task | optional | +| optional | boolean | true or false. When set to true - workflow continues even if the task fails. The status of the task is reflected as `COMPLETED_WITH_ERRORS` | Defaults to `false` | +| inputParameters | object | JSON template that defines the input given to the task. Only one of `inputParameters` or `inputExpression` can be used in a task. | See [Using Expressions](#using-expressions) for details | +| inputExpression | object | JSONPath expression that defines the input given to the task. Only one of `inputParameters` or `inputExpression` can be used in a task. | See [Using Expressions](#using-expressions) for details | +| asyncComplete | boolean | `false` to mark status COMPLETED upon execution; `true` to keep the task IN_PROGRESS and wait for an external event to complete it. | Defaults to `false` | +| startDelay | number | Time in seconds to wait before making the task available to be polled by a worker. | Defaults to 0. | + + +In addition to these parameters, System Tasks have their own parameters. Check out [System Tasks](systemtasks/index.md) for more information. + +### Using Expressions +Each executed task is given an input based on the `inputParameters` template or the `inputExpression` configured in the task configuration. Only one of `inputParameters` or `inputExpression` can be used in a task. + +#### inputParameters +`inputParameters` can use JSONPath **expressions** to extract values out of the workflow input and other tasks in the workflow. + +For example, workflows are supplied an `input` by the client/caller when a new execution is triggered. The workflow `input` is available via an *expression* of the form `${workflow.input...}`. Likewise, the `input` and `output` data of a previously executed task can also be extracted using an *expression* for use in the `inputParameters` of a subsequent task. + +Generally, `inputParameters` can use *expressions* of the following syntax: + +> `${SOURCE.input/output.JSONPath}` + +| Field | Description | +| ------------ | ------------------------------------------------------------------------ | +| SOURCE | Can be either `"workflow"` or the reference name of any task | +| input/output | Refers to either the input or output of the source | +| JSONPath | JSON path expression to extract JSON fragment from source's input/output | + + +!!! note "JSON Path Support" + Conductor supports [JSONPath](http://goessner.net/articles/JsonPath/) specification and uses Java implementation from [here](https://github.com/jayway/JsonPath). + +!!! note "Escaping expressions" + To escape an expression, prefix it with an extra _$_ character (ex.: ```$${workflow.input...}```). + +#### inputExpression + +`inputExpression` can be used to select an entire object from the workflow input, or the output of another task. The field supports all [definite](https://github.com/json-path/JsonPath#what-is-returned-when) JSONPath expressions. + +The syntax for mapping values in `inputExpression` follows the pattern, + +> `SOURCE.input/output.JSONPath` + +**NOTE:** The ```inputExpression``` field does not require the expression to be wrapped in `${}`. + +See [example](#example-3-inputexpression) below. + +## Examples + +### Example 1 - A Basic Workflow Definition + +Assume your business logic is to simply to get some shipping information and then do the shipping. You start by +logically partitioning them into two tasks: + + 1. *shipping_info* - The first task takes the provided account number, and outputs an address. + 2. *shipping_task* - The 2nd task takes the address info and generates a shipping label. + +We can configure these two tasks in the `tasks` array of our Workflow Definition. Let's assume that ```shipping info``` takes an account number, and returns a name and address. + +```json +{ + "name": "mail_a_box", + "description": "shipping Workflow", + "version": 1, + "tasks": [ + { + "name": "shipping_info", + "taskReferenceName": "shipping_info_ref", + "inputParameters": { + "account": "${workflow.input.accountNumber}" + }, + "type": "SIMPLE" + }, + { + "name": "shipping_task", + "taskReferenceName": "shipping_task_ref", + "inputParameters": { + "name": "${shipping_info_ref.output.name}", + "streetAddress": "${shipping_info_ref.output.streetAddress}", + "city": "${shipping_info_ref.output.city}", + "state": "${shipping_info_ref.output.state}", + "zipcode": "${shipping_info_ref.output.zipcode}", + }, + "type": "SIMPLE" + } + ], + "outputParameters": { + "trackingNumber": "${shipping_task_ref.output.trackinNumber}" + }, + "failureWorkflow": "shipping_issues", + "restartable": true, + "workflowStatusListenerEnabled": true, + "ownerEmail": "conductor@example.com", + "timeoutPolicy": "ALERT_ONLY", + "timeoutSeconds": 0, + "variables": {}, + "inputTemplate": {} +} +``` + +Upon completion of the 2 tasks, the workflow outputs the tracking number generated in the 2nd task. If the workflow fails, a second workflow named ```shipping_issues``` is run. + + +### Example 2 - Task Configuration +Consider a task `http_task` with input configured to use input/output parameters from workflow and a task named `loc_task`. + +```json +{ + "name": "encode_workflow", + "description": "Encode movie.", + "version": 1, + "inputParameters": [ + "movieId", "fileLocation", "recipe" + ], + "tasks": [ + { + "name": "loc_task", + "taskReferenceName": "loc_task_ref", + "taskType": "SIMPLE", + ... + }, + { + "name": "http_task", + "taskReferenceName": "http_task_ref", + "taskType": "HTTP", + "inputParameters": { + "movieId": "${workflow.input.movieId}", + "url": "${workflow.input.fileLocation}", + "lang": "${loc_task.output.languages[0]}", + "http_request": { + "method": "POST", + "url": "http://example.com/${loc_task.output.fileId}/encode", + "body": { + "recipe": "${workflow.input.recipe}", + "params": { + "width": 100, + "height": 100 + } + }, + "headers": { + "Accept": "application/json", + "Content-Type": "application/json" + } + } + } + } + ], + "ownerEmail": "conductor@example.com", + "variables": {}, + "inputTemplate": {} +} + +``` + +Consider the following as the _workflow input_ + +```json +{ + "movieId": "movie_123", + "fileLocation":"s3://moviebucket/file123", + "recipe":"png" +} +``` +And the output of the _loc_task_ as the following; + +```json +{ + "fileId": "file_xxx_yyy_zzz", + "languages": ["en","ja","es"] +} +``` + +When scheduling the task, Conductor will merge the values from workflow input and `loc_task`'s output and create the input to the `http_task` as follows: + +```json +{ + "movieId": "movie_123", + "url": "s3://moviebucket/file123", + "lang": "en", + "http_request": { + "method": "POST", + "url": "http://example.com/file_xxx_yyy_zzz/encode", + "body": { + "recipe": "png", + "params": { + "width": 100, + "height": 100 + } + }, + "headers": { + "Accept": "application/json", + "Content-Type": "application/json" + } + } +} +``` + +### Example 3 - inputExpression +Given the following task configuration: +```json +{ + "name": "loc_task", + "taskReferenceName": "loc_task_ref", + "taskType": "SIMPLE", + "inputExpression": { + "expression": "workflow.input", + "type": "JSON_PATH" + } +} +``` + +When the workflow is invoked with the following _workflow input_ +```json +{ + "movieId": "movie_123", + "fileLocation":"s3://moviebucket/file123", + "recipe":"png" +} +``` + +When the task `loc_task` is scheduled, the entire workflow input object will be passed in as the task input: +```json +{ + "movieId": "movie_123", + "fileLocation":"s3://moviebucket/file123", + "recipe":"png" +} +``` diff --git a/docs/docs/img/tutorial/ShippingWorkflow.png b/docs/documentation/configuration/workflowdef/operators/ShippingWorkflow.png similarity index 100% rename from docs/docs/img/tutorial/ShippingWorkflow.png rename to docs/documentation/configuration/workflowdef/operators/ShippingWorkflow.png diff --git a/docs/docs/img/tutorial/ShippingWorkflowRunning.png b/docs/documentation/configuration/workflowdef/operators/ShippingWorkflowRunning.png similarity index 100% rename from docs/docs/img/tutorial/ShippingWorkflowRunning.png rename to docs/documentation/configuration/workflowdef/operators/ShippingWorkflowRunning.png diff --git a/docs/docs/img/tutorial/ShippingWorkflowUPS.png b/docs/documentation/configuration/workflowdef/operators/ShippingWorkflowUPS.png similarity index 100% rename from docs/docs/img/tutorial/ShippingWorkflowUPS.png rename to docs/documentation/configuration/workflowdef/operators/ShippingWorkflowUPS.png diff --git a/docs/docs/img/Switch_Fedex.png b/docs/documentation/configuration/workflowdef/operators/Switch_Fedex.png similarity index 100% rename from docs/docs/img/Switch_Fedex.png rename to docs/documentation/configuration/workflowdef/operators/Switch_Fedex.png diff --git a/docs/docs/img/Terminate_Task.png b/docs/documentation/configuration/workflowdef/operators/Terminate_Task.png similarity index 100% rename from docs/docs/img/Terminate_Task.png rename to docs/documentation/configuration/workflowdef/operators/Terminate_Task.png diff --git a/docs/docs/reference-docs/do-while-task.md b/docs/documentation/configuration/workflowdef/operators/do-while-task.md similarity index 86% rename from docs/docs/reference-docs/do-while-task.md rename to docs/documentation/configuration/workflowdef/operators/do-while-task.md index 80e249c6a..48398bf25 100644 --- a/docs/docs/reference-docs/do-while-task.md +++ b/docs/documentation/configuration/workflowdef/operators/do-while-task.md @@ -1,14 +1,10 @@ ---- -sidebar_position: 1 ---- - # Do-While ```json "type" : "DO_WHILE" ``` -## Introduction -Sequentially execute a list of task as long as a condition is true. -The list of tasks is executed first, before the condition is checked (even for the first iteration). + +The `DO_WHILE` task sequentially executes a list of tasks as long as a given condition is true. +The list of tasks is executed first, before the condition is checked (the first iteration will always execute). When scheduled, each task of this loop will see its `taskReferenceName` concatenated with __i, with i being the iteration number, starting at 1. Warning: taskReferenceName containing arithmetic operators must not be used. @@ -16,7 +12,7 @@ Each task output is stored as part of the DO_WHILE task, indexed by the iteratio The DO_WHILE task is set to `FAILED` as soon as one of the loopOver fails. In such case retry, iteration starts from 1. -### Limitations +## Limitations - Domain or isolation group execution is unsupported; - Nested DO_WHILE is unsupported, however, DO_WHILE task supports SUB_WORKFLOW as loopOver task, so we can achieve similar functionality. - Since loopover tasks will be executed in loop inside scope of parent do while task, crossing branching outside of DO_WHILE task is not respected. @@ -24,26 +20,24 @@ The DO_WHILE task is set to `FAILED` as soon as one of the loopOver fails. In su Branching inside loopOver task is supported. - - ## Configuration +The following fields must be specified at the top level of the task configuration. -### Input Parameters: - -| name | type | description | -|---------------|------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| loopCondition | String | Condition to be evaluated after every iteration. This is a Javascript expression, evaluated using the Nashorn engine. If an exception occurs during evaluation, the DO_WHILE task is set to FAILED_WITH_TERMINAL_ERROR. | -| loopOver | List[Task] | List of tasks that needs to be executed as long as the condition is true. | +| name | type | description | +| ------------- | ----------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| loopCondition | String | Condition to be evaluated after every iteration. This is a Javascript expression, evaluated using the Nashorn engine. If an exception occurs during evaluation, the DO_WHILE task is set to FAILED_WITH_TERMINAL_ERROR. | +| loopOver | List\[Task] | List of tasks that needs to be executed as long as the condition is true. | -### Output Parameters +## Output | name | type | description | -|-----------|------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| --------- | ---------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | iteration | Integer | Iteration number: the current one while executing; the final one once the loop is finished | | `i` | Map[String, Any] | Iteration number as a string, mapped to the task references names and their output. | | * | Any | Any state can be stored here if the `loopCondition` does so. For example `storage` will exist if `loopCondition` is `if ($.LoopTask['iteration'] <= 10) {$.LoopTask.storage = 3; true } else {false}` | ## Examples +### Basic Example The following definition: ```json @@ -137,7 +131,7 @@ will produce the following execution, assuming 3 executions occurred (alongside } ``` -## Example using iteration key +### Example using iteration key Sometimes, you may want to use the iteration value/counter in the tasks used in the loop. In this example, an API call is made to GitHub (to the Netflix Conductor repository), but each loop increases the pagination. diff --git a/docs/docs/reference-docs/dynamic-fork-task.md b/docs/documentation/configuration/workflowdef/operators/dynamic-fork-task.md similarity index 67% rename from docs/docs/reference-docs/dynamic-fork-task.md rename to docs/documentation/configuration/workflowdef/operators/dynamic-fork-task.md index e8f8721f6..c95d9ebf9 100644 --- a/docs/docs/reference-docs/dynamic-fork-task.md +++ b/docs/documentation/configuration/workflowdef/operators/dynamic-fork-task.md @@ -3,15 +3,10 @@ "type" : "FORK_JOIN_DYNAMIC" ``` -## Introduction +The `DYNAMIC_FORK` operation in Conductor lets you execute a list of tasks or sub-workflows in parallel. This list will +be determined at run-time and be of variable length. -A Fork operation in conductor, lets you run a specified list of other tasks or sub workflows in parallel after the fork -task. A fork task is followed by a join operation that waits on the forked tasks or sub workflows to finish. The `JOIN` -task also collects outputs from each of the forked tasks or sub workflows. - -In a regular fork operation (`FORK_JOIN` task), the list of tasks or sub workflows that need to be forked and run in -parallel are already known at the time of workflow definition creation time. However, there are cases when that list can -only be determined at run-time and that is when the dynamic fork operation (FORK_JOIN_DYNAMIC task) is needed. +A `DYNAMIC_FORK` is typically followed by `JOIN` task that collects outputs from each of the forked tasks or sub workflows. There are three things that are needed to configure a `FORK_JOIN_DYNAMIC` task. @@ -21,7 +16,6 @@ There are three things that are needed to configure a `FORK_JOIN_DYNAMIC` task. the `FORK_JOIN_DYNAMIC` tasks ## Use Cases - A `FORK_JOIN_DYNAMIC` is useful, when a set of tasks or sub-workflows needs to be executed and the number of tasks or sub-workflows are determined at run time. E.g. Let's say we have a task that resizes an image, and we need to create a workflow that will resize an image into multiple sizes. In this case, a task can be created prior to @@ -31,24 +25,43 @@ invokes of the single image resize task. Here, the responsibilities are clearly task does the core job and `FORK_JOIN_DYNAMIC` manages the orchestration and fault tolerance aspects. ## Configuration +To use the `DYNAMIC_FORK` task, you need to provide the following attributes at the top level of the task configuration, **as well as** corresponding values inside `inputParameters`. + +| Attribute | Description | +| ------------------------------ | ------------------------------------------------------------------------------------------------------- | +| dynamicForkTasksParam | Name of attribute in `inputParameters` to read array of task or subworkflow names from. | +| dynamicForkTasksInputParamName | Name of attribute in `inputParameters` to read map of inputs to use for spawned tasks or subworkflows. | + +### inputParameters +| Attribute | Description | +| ------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| *dynamicForkTasksParam | This is a JSON array of tasks or sub-workflow objects that needs to be forked and run in parallel (Note: This has a different format for ```SUB_WORKFLOW``` compared to ```SIMPLE``` tasks.) | +| *dynamicForkTasksInputParamName | A JSON map, where the keys are task or sub-workflow names, and the values are the `inputParameters` to be passed into the corresponding spawned tasks or sub-workflows. | + +Note: * means the de-referenced name. + +## Examples +### Example 1 Here is an example of a `FORK_JOIN_DYNAMIC` task followed by a `JOIN` task ```json -{ - "inputParameters": { - "dynamicTasks": "${fooBarTask.output.dynamicTasksJSON}", - "dynamicTasksInput": "${fooBarTask.output.dynamicTasksInputJSON}" +[ + { + "inputParameters": { + "dynamicTasks": "${fooBarTask.output.dynamicTasksJSON}", + "dynamicTasksInput": "${fooBarTask.output.dynamicTasksInputJSON}" + }, + "type": "FORK_JOIN_DYNAMIC", + "dynamicForkTasksParam": "dynamicTasks", + "dynamicForkTasksInputParamName": "dynamicTasksInput" }, - "type": "FORK_JOIN_DYNAMIC", - "dynamicForkTasksParam": "dynamicTasks", - "dynamicForkTasksInputParamName": "dynamicTasksInput" -}, -{ -"name": "image_multiple_convert_resize_join", -"taskReferenceName": "image_multiple_convert_resize_join_ref", -"type": "JOIN" -} + { + "name": "image_multiple_convert_resize_join", + "taskReferenceName": "image_multiple_convert_resize_join_ref", + "type": "JOIN" + } +] ``` Dissecting into this example above, let's look at the three things that are needed to configured for @@ -61,21 +74,7 @@ fooBarTask This is a task that is defined prior to the FORK_JOIN_DYNAMIC in the to output (outputParameters) 1 and 2 above so that it can be wired into inputParameters of the FORK_JOIN_DYNAMIC tasks. (dynamicTasks and dynamicTasksInput) -## Input Configuration - - -| Attribute | Description | -| ----------- | ----------- | -| name | Task Name. A unique name that is descriptive of the task function | -| taskReferenceName | Task Reference Name. A unique reference to this task. There can be multiple references of a task within the same workflow definition | -| type | Task Type. In this case, `FORK_JOIN_DYNAMIC` | -| inputParameters | The input parameters that will be supplied to this task. | -| dynamicForkTasksParam | This is a JSON array of tasks or sub-workflow objects that needs to be forked and run in parallel (Note: This has a different format for ```SUB_WORKFLOW``` compared to ```SIMPLE``` tasks.) | -| dynamicForkTasksInputParamName | A JSON map, where the keys are task or sub-workflow names, and the values are its corresponding inputParameters | - - -## Example - +### Example 2 Let's say we have a task that resizes an image, and we need to create a workflow that will resize an image into multiple sizes. In this case, a task can be created prior to the `FORK_JOIN_DYNAMIC` task that will prepare the input that needs to be passed into the `FORK_JOIN_DYNAMIC` task. These will be: @@ -88,32 +87,34 @@ the `FORK_JOIN_DYNAMIC` task that will prepare the input that needs to be passed The ```image_resize``` task works to resize just one image. The `FORK_JOIN_DYNAMIC` and the following `JOIN` will manage the multiple invocations of the single ```image_resize``` task. The responsibilities are clearly broken out, where the individual ```image_resize``` tasks do the core job and `FORK_JOIN_DYNAMIC` manages the orchestration and fault tolerance aspects of handling multiple invocations of the task. -## The workflow +#### Workflow Definition - Task Configuration Here is an example of a `FORK_JOIN_DYNAMIC` task followed by a `JOIN` task. The fork is named and given a taskReferenceName, but all of the input parameters are JSON variables that we will discuss next: ```json -{ - "name": "image_multiple_convert_resize_fork", - "taskReferenceName": "image_multiple_convert_resize_fork_ref", - "inputParameters": { - "dynamicTasks": "${fooBarTask.output.dynamicTasksJSON}", - "dynamicTasksInput": "${fooBarTask.output.dynamicTasksInputJSON}" +[ + { + "name": "image_multiple_convert_resize_fork", + "taskReferenceName": "image_multiple_convert_resize_fork_ref", + "inputParameters": { + "dynamicTasks": "${fooBarTask.output.dynamicTasksJSON}", + "dynamicTasksInput": "${fooBarTask.output.dynamicTasksInputJSON}" + }, + "type": "FORK_JOIN_DYNAMIC", + "dynamicForkTasksParam": "dynamicTasks", + "dynamicForkTasksInputParamName": "dynamicTasksInput" }, - "type": "FORK_JOIN_DYNAMIC", - "dynamicForkTasksParam": "dynamicTasks", - "dynamicForkTasksInputParamName": "dynamicTasksInput" -}, -{ -"name": "image_multiple_convert_resize_join", -"taskReferenceName": "image_multiple_convert_resize_join_ref", -"type": "JOIN" -} + { + "name": "image_multiple_convert_resize_join", + "taskReferenceName": "image_multiple_convert_resize_join_ref", + "type": "JOIN" + } +] ``` This appears in the UI as follows: -![diagram of dynamic fork](/img/dynamic-task-diagram.png) +![diagram of dynamic fork](dynamic-task-diagram.png) Let's assume this data is sent to the workflow: @@ -137,7 +138,7 @@ With 2 file formats and 2 sizes in the input, we'll be creating 4 images total. * `dynamicForkTasksParam` This is a JSON array of task or sub-workflow objects that specifies the list of tasks or sub-workflows that needs to be forked and run in parallel. This JSON varies depeding oon the type of task. -### ```dynamicForkTasksParam``` Simple task +#### ```dynamicForkTasksParam``` Simple task In this case, our fork is running a SIMPLE task: ```image_convert_resize```: ``` @@ -164,7 +165,7 @@ In this case, our fork is running a SIMPLE task: ```image_convert_resize```: } ]} ``` -### ```dynamicForkTasksParam``` SubWorkflow task +#### ```dynamicForkTasksParam``` SubWorkflow task In this case, our Dynamic fork is running a SUB_WORKFLOW task: ```image_convert_resize_subworkflow``` ``` @@ -245,6 +246,6 @@ sub-workflow objects and all the input parameters that these tasks will need to } ``` -### The Join +#### Join Task -The [JOIN](/reference-docs/join-task.html) task will run after all of the dynamic tasks, collecting the output for all of the tasks. \ No newline at end of file +The [JOIN](join-task.md) task will run after all of the dynamic tasks, collecting the output for all of the tasks. \ No newline at end of file diff --git a/docs/docs/img/dynamic-task-diagram.png b/docs/documentation/configuration/workflowdef/operators/dynamic-task-diagram.png similarity index 100% rename from docs/docs/img/dynamic-task-diagram.png rename to docs/documentation/configuration/workflowdef/operators/dynamic-task-diagram.png diff --git a/docs/docs/reference-docs/dynamic-task.md b/docs/documentation/configuration/workflowdef/operators/dynamic-task.md similarity index 71% rename from docs/docs/reference-docs/dynamic-task.md rename to docs/documentation/configuration/workflowdef/operators/dynamic-task.md index ffe26b817..bc2280274 100644 --- a/docs/docs/reference-docs/dynamic-task.md +++ b/docs/documentation/configuration/workflowdef/operators/dynamic-task.md @@ -2,39 +2,35 @@ ```json "type" : "DYNAMIC" ``` +The `DYNAMIC` task allows one to execute a task whose name is resolved dynamically at run-time. +The task name to execute is specified as `taskToExecute` in `inputParameters`. -### Introduction -Dynamic Task allows to execute one of the registered Tasks dynamically at run-time. -It accepts the task name to execute as `taskToExecute` in `inputParameters`. - -### Use Cases +## Use Cases Consider a scenario, when we have to make decision of executing a task dynamically i.e. while the workflow is still running. In such cases, Dynamic Task would be useful. -### Configuration - -Dynamic task is defined directly inside the workflow with type `DYNAMIC`. - -#### Inputs - -Following are the input parameters : +## Configuration +To use the `DYNAMIC` task, you need to provide `dynamicTaskNameParam` at the top level of the task configuration, **as well as** an attribute in `inputParameters` matching the value you selected for `dynamicTaskNameParam`. -| name | description | -|----------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| dynamicTaskNameParam | Name of the parameter from the task input whose value is used to schedule the task. e.g. if the value of the parameter is ABC, the next task scheduled is of type 'ABC'. | +| name | description | +| -------------------- | ------------------------------------------------------------------------------------------------------------- | +| dynamicTaskNameParam | Name of the parameter from `inputParameters` whose value is used to schedule the task. e.g. `"taskToExecute"` | -#### Output +### inputParameters +| name | description | +| ----------------------------------------- | ------------------------ | +| *dynamicTaskNameParam e.g. `taskToExecute` | Name of task to execute. | -TODO: Talk about output of the task, what to expect - -### Examples +## Example Suppose in a workflow, we have to take decision to ship the courier with the shipping service providers on the basis of Post Code. -Following task `shipping_info` generates an output on the basis of which decision would be +Consider the following 3 task definitions. + +The following task `shipping_info` generates an output on the basis of which decision would be taken to run the next task. ```json @@ -54,7 +50,7 @@ taken to run the next task. } ``` -Following are the two worker tasks, one among them would execute on the basis of output generated +The following are the two worker tasks, one among them would execute on the basis of output generated by the `shipping_info` task : ```json @@ -88,8 +84,7 @@ by the `shipping_info` task : } ``` - -We will create the Workflow with the following definition : +We will create a workflow with the following definition : ```json { @@ -122,10 +117,10 @@ We will create the Workflow with the following definition : } ``` -Workflow is the created as shown in the below diagram. +The workflow created is shown in the below diagram. -![Conductor UI - Workflow Diagram](/img/tutorial/ShippingWorkflow.png) +![Conductor UI - Workflow Diagram](ShippingWorkflow.png) Note : `shipping_task` is a `DYNAMIC` task and the `taskToExecute` parameter can be set @@ -147,11 +142,11 @@ decided. Based on given set of inputs i.e. Post Code starts with '9' hence, `ship_via_fedex` is executed - -![Conductor UI - Workflow Run](/img/tutorial/ShippingWorkflowRunning.png) +![Conductor UI - Workflow Run](ShippingWorkflowRunning.png) If the Post Code started with anything other than 9 `ship_via_ups` is executed - -![Conductor UI - Workflow Run](/img/tutorial/ShippingWorkflowUPS.png) +![Conductor UI - Workflow Run](ShippingWorkflowUPS.png) If the incorrect task name or the task that doesn't exist is provided then the workflow fails and we get the error `"Invalid task specified. Cannot find task by name in the task definitions."` diff --git a/docs/docs/img/fork-task-diagram.png b/docs/documentation/configuration/workflowdef/operators/fork-task-diagram.png similarity index 100% rename from docs/docs/img/fork-task-diagram.png rename to docs/documentation/configuration/workflowdef/operators/fork-task-diagram.png diff --git a/docs/documentation/configuration/workflowdef/operators/fork-task.md b/docs/documentation/configuration/workflowdef/operators/fork-task.md new file mode 100644 index 000000000..cfa6c9d4d --- /dev/null +++ b/docs/documentation/configuration/workflowdef/operators/fork-task.md @@ -0,0 +1,123 @@ +# Fork +```json +"type" : "FORK_JOIN" +``` + +A `FORK_JOIN` operation lets you run a specified list of tasks or sub workflows in parallel. A `FORK_JOIN` task is +followed by a `JOIN` operation that waits on the forked tasks or sub workflows to finish and collects their outputs. + +This is also known as a **Static Fork** to distinguish it from the [DYNAMIC_FORK](dynamic-fork-task.md). + +## Use Cases + +`FORK_JOIN` tasks are typically used when a list of tasks can be run in parallel. E.g In a notification workflow, there +could be multiple ways of sending notifications, i,e e-mail, SMS, HTTP etc. These notifications are not dependent on +each other, and so they can be run in parallel. In such cases, you can create 3 sub-lists of forked tasks for each of +these operations. + +## Configuration + +A `FORK_JOIN` task has a `forkTasks` attribute at the top level of the task configuration that is an array of arrays (`[[...], [...]]`); + +| Attribute | Description | +|-------------------|-----------------------------------------------------------------------------------------------------------------------------------------------| +| forkTasks | A list of lists of tasks. Each of the outer list will be invoked in parallel. The inner list can be a graph of other tasks and sub-workflows | + +Each element of `forkTasks` is itself a list of tasks. These sub-lists are invoked in parallel. The tasks defined within each sublist +can be sequential or even more nested forks. + +A FORK_JOIN is typically followed by a [JOIN](join-task.md) operation. + +The behavior of a `FORK_JOIN` task is not affected by `inputParameters`. + +## Output +`FORK_JOIN` has no output. + +The `FORK_JOIN` task is used in conjunction with the [JOIN](join-task.md) task, which aggregates the output from the parallelized workflows. + +## Example + +Imagine a workflow that sends 3 notifications: email, SMS and HTTP. Since none of these steps are dependant on the others, they can be run in parallel with a fork. + +The diagram will appear as: + +![fork diagram](fork-task-diagram.png) + +Here's the JSON definition for the workflow: + +```json +[ + { + "name": "fork_join", + "taskReferenceName": "my_fork_join_ref", + "type": "FORK_JOIN", + "forkTasks": [ + [ + { + "name": "process_notification_payload", + "taskReferenceName": "process_notification_payload_email", + "type": "SIMPLE" + }, + { + "name": "email_notification", + "taskReferenceName": "email_notification_ref", + "type": "SIMPLE" + } + ], + [ + { + "name": "process_notification_payload", + "taskReferenceName": "process_notification_payload_sms", + "type": "SIMPLE" + }, + { + "name": "sms_notification", + "taskReferenceName": "sms_notification_ref", + "type": "SIMPLE" + } + ], + [ + { + "name": "process_notification_payload", + "taskReferenceName": "process_notification_payload_http", + "type": "SIMPLE" + }, + { + "name": "http_notification", + "taskReferenceName": "http_notification_ref", + "type": "SIMPLE" + } + ] + ] + }, + { + "name": "notification_join", + "taskReferenceName": "notification_join_ref", + "type": "JOIN", + "joinOn": [ + "email_notification_ref", + "sms_notification_ref" + ] + } +] +``` +!!! note Note + There are three parallel 'tines' to this fork, but only two of the outputs are required for the JOIN to continue, owing to the definition of `joinOn`. The diagram *does* draw an arrow from ```http_notification_ref``` to the ```notification_join```, but it is not required for the workflow to continue. + +Here is how the output of notification_join will look like. The output is a map, where the keys are the names of task +references that were being `joinOn`. The corresponding values are the outputs of those tasks. + +``` +{ + "email_notification_ref": { + "email_sent_at": "2021-11-06T07:37:17+0000", + "email_sent_to": "test@example.com" + }, + "sms_notification_ref": { + "smm_sent_at": "2021-11-06T07:37:17+0129", + "sms_sen": "+1-425-555-0189" + } +} +``` + +See [JOIN](join-task.md) for more details on the JOIN aspect of the FORK. diff --git a/docs/documentation/configuration/workflowdef/operators/index.md b/docs/documentation/configuration/workflowdef/operators/index.md new file mode 100644 index 000000000..a325a3531 --- /dev/null +++ b/docs/documentation/configuration/workflowdef/operators/index.md @@ -0,0 +1,19 @@ +# Operators + +Operators are built-in primitives in Conductor that allow you to define the control flow in the workflow. +Operators are similar to programming constructs such as for loops, decisions, etc. +Conductor has support for most of the programing primitives allowing you to define the most advanced workflows. + +Conductor supports the following programming language constructs: + +| Language Construct | Conductor Operator | +| -------------------------- | ----------------------------------------- | +| Do-While or For Loops | [Do While Task](do-while-task.md) | +| Function Pointer | [Dynamic Task](dynamic-task.md) | +| Dynamic Parallel execution | [Dynamic Fork Task](dynamic-fork-task.md) | +| Static Parallel execution | [Fork Task](fork-task.md) | +| Map | [Join Task](join-task.md) | +| Subroutine / Fork Process | [Sub Workflow Task](sub-workflow-task.md) | +| Switch/if..then...else | [Switch Task](switch-task.md) | +| Exit | [Terminate Task](terminate-task.md) | +| Global Variables | [Variable Task](set-variable-task.md) | diff --git a/docs/docs/reference-docs/join-task.md b/docs/documentation/configuration/workflowdef/operators/join-task.md similarity index 57% rename from docs/docs/reference-docs/join-task.md rename to docs/documentation/configuration/workflowdef/operators/join-task.md index fe1c66c37..8b07536fc 100644 --- a/docs/docs/reference-docs/join-task.md +++ b/docs/documentation/configuration/workflowdef/operators/join-task.md @@ -1,38 +1,35 @@ ---- -sidebar_position: 1 ---- # Join ```json "type" : "JOIN" ``` -### Introduction +A `JOIN` task is used in conjunction with a `FORK_JOIN` or `FORK_JOIN_DYNAMIC` task. Each of the aggregated task outputs is given a corresponding key in the `JOIN` task output. -A `JOIN` task is used in conjunction with a `FORK_JOIN` or `FORK_JOIN_DYNAMIC` task. When `JOIN` is used along with -a `FORK_JOIN` task, tt waits for a list of zero or more of the forked tasks to be completed. However, when used with -a `FORK_JOIN_DYNAMIC` task, it implicitly waits for all of the dynamically forked tasks to complete. +* When used with a `FORK_JOIN` task, it waits for a **user provided** list of zero or more of the forked tasks to be completed. +* When used with a `FORK_JOIN_DYNAMIC` task, it implicitly waits for all of the dynamically forked tasks to complete. -### Use Cases +`JOIN` used in this context is loosely analogous to the *Map* phase of the *Map-Reduce* programming pattern. In Conductor, the *reduce* step could +be implemented as a subsequent task that references the output of the `JOIN` task. -[FORK_JOIN](/reference-docs/fork-task.html) and [FORK_JOIN_DYNAMIC](/reference-docs/dynamic-fork-task.html) task are used to execute a collection of other tasks or sub workflows in parallel. In -such cases, there is a need for these forked tasks to complete before moving to the next stage in the workflow. +## Use Cases -### Configuration +[FORK_JOIN](fork-task.md) and [FORK_JOIN_DYNAMIC](dynamic-fork-task.md) task are used to execute a collection of other tasks or sub workflows in parallel. In +such cases, there is a need to collect the output from the forked tasks before moving to the next stage in the workflow. -#### Input Configuration +## Configuration +When used with `FORK_JOIN` (Static Fork), The `joinOn` attribute is provided at the top level of the task configuration. -| Attribute | Description | -|-------------------|--------------------------------------------------------------------------------------------------------------------------------------| -| name | Task Name. A unique name that is descriptive of the task function | -| taskReferenceName | Task Reference Name. A unique reference to this task. There can be multiple references of a task within the same workflow definition | -| type | Task Type. In this case, `JOIN` | -| joinOn | A list of task reference names, that this `JOIN` task will wait for completion | +| Attribute | Description | +| --------- | --------------------------------------------------------------------------------------------------------- | +| joinOn | A list of task reference names that this `JOIN` task will wait for completion. Omitted for `DYNAMIC_FORK` | -#### Output Configuration +The `JOIN` task does not utilize `inputParameters`. + +## Output | Attribute | Description | -|-----------------|-------------------------------------------------------------------------------------| +| --------------- | ----------------------------------------------------------------------------------- | | task_ref_name_1 | A task reference name that was being `joinOn`. The value is the output of that task | | task_ref_name_2 | A task reference name that was being `joinOn`. The value is the output of that task | | ... | ... | @@ -40,9 +37,9 @@ such cases, there is a need for these forked tasks to complete before moving to -### Examples +## Examples -#### Simple Example +### Simple Example Here is an example of a _`JOIN`_ task. This task will wait for the completion of tasks `my_task_ref_1` and `my_task_ref_2` as specified by the `joinOn` attribute. @@ -59,12 +56,12 @@ and `my_task_ref_2` as specified by the `joinOn` attribute. ``` -#### Example - ignoring one fork +### Example - Ignoring one fork Here is an example of a `JOIN` task used in conjunction with a `FORK_JOIN` task. The 'FORK_JOIN' spawns 3 tasks. -An `email_notification` task, a `sms_notification` task, and a `http_notification` task. Email and SMS are usually best +An `email_notification` task, a `sms_notification` task and a `http_notification` task. Email and SMS are usually best effort delivery systems. However, in case of a http based notification you get a return code and you can retry until it -succeeds or eventually give up. When you setup a notification workflow, you may decide to continue, if you kicked off an -email and sms notification. In that case, you can decide to `joinOn` those specific tasks. However, +succeeds or eventually give up. When you setup a notification workflow, you may decide to continue ,if you kicked off an +email and sms notification. Im that case, you can decide to `joinOn` those specific tasks. However, the `http_notification` task will still continue to execute, but it will not block the rest of the workflow from proceeding. diff --git a/docs/docs/reference-docs/set-variable-task.md b/docs/documentation/configuration/workflowdef/operators/set-variable-task.md similarity index 56% rename from docs/docs/reference-docs/set-variable-task.md rename to docs/documentation/configuration/workflowdef/operators/set-variable-task.md index c9c8b55e6..d29a562ca 100644 --- a/docs/docs/reference-docs/set-variable-task.md +++ b/docs/documentation/configuration/workflowdef/operators/set-variable-task.md @@ -1,29 +1,26 @@ ---- -sidebar_position: 1 ---- - # Set Variable ```json "type" : "SET_VARIABLE" ``` -### Introduction -Set Variable allows us to set workflow variables by creating or updating them -with new values. -### Use Cases +The `SET_VARIABLE` task allows users to create global workflow variables, and update them with new values. + +Variables can be initialized in the workflow definition as well as during the workflow run. Once a variable is initialized +it can be read using the *expression* `${workflow.variables.NAME}` by any other task. -Variables can be initialized in the workflow definition as well as during -the workflow run. Once a variable was initialized it can be read or -overwritten with a new value by any other task. +It can be overwritten by a subsequent `SET_VARIABLE` task. -### Configuration +!!!warning + There is a hard barrier for variables payload size in KB defined in the JVM system properties (`conductor.max.workflow.variables.payload.threshold.kb`) the default value is `256`. Passing this barrier will fail the task and the workflow. -Set Variable task is defined directly inside the workflow with type -`SET_VARIABLE`. +## Use Cases +For example, you might want to track shared state at the workflow level, and have the state be accessible by any task executed as part of the workflow. -## Examples +## Configuration +Global variables can be set in `inputParameters` using the desired variable names and their respective values. +## Example Suppose in a workflow, we have to store a value in a variable and then later in workflow reuse the value stored in the variable just as we do in programming, in such scenarios `Set Variable` task can be used. @@ -48,7 +45,7 @@ Following is the workflow definition with `SET_VARIABLE` task. "name": "Read_Name", "taskReferenceName": "Read_Name", "inputParameters": { - "var_name" : "${workflow.variables.name}" + "saved_name" : "${workflow.variables.name}" }, "type": "SIMPLE" } diff --git a/docs/docs/reference-docs/start-workflow-task.md b/docs/documentation/configuration/workflowdef/operators/start-workflow-task.md similarity index 52% rename from docs/docs/reference-docs/start-workflow-task.md rename to docs/documentation/configuration/workflowdef/operators/start-workflow-task.md index 53db54886..bdf515ee8 100644 --- a/docs/docs/reference-docs/start-workflow-task.md +++ b/docs/documentation/configuration/workflowdef/operators/start-workflow-task.md @@ -1,35 +1,29 @@ ---- -sidebar_position: 1 ---- # Start Workflow ```json "type" : "START_WORKFLOW" ``` -### Introduction -Start Workflow starts another workflow. Unlike `SUB_WORKFLOW`, `START_WORKFLOW` does +The `START_WORKFLOW` task starts another workflow. Unlike `SUB_WORKFLOW`, `START_WORKFLOW` does not create a relationship between starter and the started workflow. It also does not wait for the started workflow to complete. A `START_WORKFLOW` is -considered successful once the requested workflow is started successfully. In other words, `START_WORKFLOW` is marked as COMPLETED, once the started -workflow is in RUNNING state. +considered successful once the requested workflow is started successfully. In other words, `START_WORKFLOW` is marked as `COMPLETED` once the started +workflow is in `RUNNING` state. -### Use Cases +There is no ability to access the `output` of the started workflow. -When another workflow needs to be started from a workflow, `START_WORKFLOW` can be used. +## Use Cases +When another workflow needs to be started from the current workflow, `START_WORKFLOW` can be used. -### Configuration - -Start Workflow task is defined directly inside the workflow with type `START_WORKFLOW`. - -#### Input - -**Parameters:** +## Configuration +The workflow invocation payload is passed into `startWorkflow` under `inputParameters`. +### inputParameters | name | type | description | |---------------|------------------|---------------------------------------------------------------------------------------------------------------------| -| startWorkflow | Map[String, Any] | The value of this parameter is [Start Workflow Request](/gettingstarted/startworkflow.html#start-workflow-request). | - -#### Output +| startWorkflow | Map[String, Any] | The value of this parameter is [Start Workflow Request](../../../api/startworkflow.md#start-workflow-request). | +## Output | name | type | description | |------------|--------|--------------------------------| | workflowId | String | The id of the started workflow | + +Note: `START_WORKFLOW` will neither wait for the completion of, nor pass back the `output` of the spawned workflow. diff --git a/docs/docs/reference-docs/sub-workflow-task.md b/docs/documentation/configuration/workflowdef/operators/sub-workflow-task.md similarity index 63% rename from docs/docs/reference-docs/sub-workflow-task.md rename to docs/documentation/configuration/workflowdef/operators/sub-workflow-task.md index 0c5686ed6..8c206f5fa 100644 --- a/docs/docs/reference-docs/sub-workflow-task.md +++ b/docs/documentation/configuration/workflowdef/operators/sub-workflow-task.md @@ -1,60 +1,50 @@ ---- -sidebar_position: 1 ---- # Sub Workflow ```json "type" : "SUB_WORKFLOW" ``` -### Introduction Sub Workflow task allows for nesting a workflow within another workflow. Nested workflows contain a reference to their parent. -### Use Cases +## Use Cases Suppose we want to include another workflow inside our current workflow. In that case, Sub Workflow Task would be used. -### Configuration - -Sub Workflow task is defined directly inside the workflow with type `SUB_WORKFLOW`. - -#### Input - -**Parameters:** +## Configuration +`subWorkflowParam` is provided at the top level of the task configuration. | name | type | description | -|------------------|------------------|-------------| +| ---------------- | ---------------- | ----------- | | subWorkflowParam | Map[String, Any] | See below | -**subWorkflowParam** +`inputParameters` will be passed down to the invoked sub-workflow. -| name | type | description | -|--------------------|-------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| name | String | Name of the workflow to execute | -| version | Integer | Version of the workflow to execute | -| taskToDomain | Map[String, String] | Allows scheduling the sub workflow's tasks per given mappings.
See [Task Domains](/configuration/taskdomains.html) for instructions to configure taskDomains. | -| workflowDefinition | [WorkflowDefinition](/configuration/workflowdef.html) | Allows starting a subworkflow with a dynamic workflow definition. | +### subWorkflowParam -#### Output +| name | type | description | +| ------------------ | --------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| name | String | Name of the workflow to execute | +| version | Integer | Version of the workflow to execute | +| taskToDomain | Map[String, String] | Allows scheduling the sub workflow's tasks per given mappings.
See [Task Domains](../../../api/taskdomains.md) for instructions to configure taskDomains. | +| workflowDefinition | [WorkflowDefinition](../index.md) | Allows starting a subworkflow with a dynamic workflow definition. | + +## Output | name | type | description | -|---------------|--------|-------------------------------------------------------------------| +| ------------- | ------ | ----------------------------------------------------------------- | | subWorkflowId | String | Sub-workflow execution Id generated when running the sub-workflow | -### Examples - - +## Examples Imagine we have a workflow that has a fork in it. In the example below, we input one image, but using a fork to create 2 images simultaneously: - -![workflow with fork](/img/workflow_fork.png) +![workflow with fork](workflow_fork.png) The left fork will create a JPG, and the right fork a WEBP image. Maintaining this workflow might be difficult, as changes made to one side of the fork do not automatically propagate the other. Rather than using 2 tasks, we can define a ```image_convert_resize``` workflow that we can call for both forks as a sub-workflow: ```json -{{ +{ "name": "image_convert_resize_subworkflow1", "description": "Image Processing Workflow", "version": 1, @@ -144,7 +134,7 @@ The left fork will create a JPG, and the right fork a WEBP image. Maintaining th ``` Now our diagram will appear as: -![workflow with 2 subworkflows](/img/subworkflow_diagram.png) +![workflow with 2 subworkflows](subworkflow_diagram.png) @@ -155,24 +145,24 @@ Looking at the subworkflow (the WEBP version): ``` { - "name": "image_convert_resize_sub", - "taskReferenceName": "subworkflow_webp_ref", - "inputParameters": { - "fileLocation": "${workflow.input.fileLocation}", - "recipeParameters": { - "outputSize": { - "width": "${workflow.input.recipeParameters.outputSize.width}", - "height": "${workflow.input.recipeParameters.outputSize.height}" - }, - "outputFormat": "webp" - } - }, - "type": "SUB_WORKFLOW", - "subWorkflowParam": { - "name": "image_convert_resize", - "version": 1 - } - } + "name": "image_convert_resize_sub", + "taskReferenceName": "subworkflow_webp_ref", + "inputParameters": { + "fileLocation": "${workflow.input.fileLocation}", + "recipeParameters": { + "outputSize": { + "width": "${workflow.input.recipeParameters.outputSize.width}", + "height": "${workflow.input.recipeParameters.outputSize.height}" + }, + "outputFormat": "webp" + } + }, + "type": "SUB_WORKFLOW", + "subWorkflowParam": { + "name": "image_convert_resize", + "version": 1 + } +} ``` The ```subWorkflowParam``` tells conductor which workflow to call. The task is marked as completed upon the completion of the spawned workflow. diff --git a/docs/docs/img/subworkflow_diagram.png b/docs/documentation/configuration/workflowdef/operators/subworkflow_diagram.png similarity index 100% rename from docs/docs/img/subworkflow_diagram.png rename to docs/documentation/configuration/workflowdef/operators/subworkflow_diagram.png diff --git a/docs/docs/reference-docs/switch-task.md b/docs/documentation/configuration/workflowdef/operators/switch-task.md similarity index 52% rename from docs/docs/reference-docs/switch-task.md rename to docs/documentation/configuration/workflowdef/operators/switch-task.md index f43d8f401..5b5b5704a 100644 --- a/docs/docs/reference-docs/switch-task.md +++ b/docs/documentation/configuration/workflowdef/operators/switch-task.md @@ -1,49 +1,41 @@ ---- -sidebar_position: 1 ---- - # Switch - ```json "type" : "SWITCH" ``` -### Introduction -A switch task is similar to `case...switch` statement in a programming language. The `switch` expression, is -a configuration on the `SWITCH` task type. Currently, two evaluators are supported: - -### Configuration -Following are the task configuration parameters : +A `SWITCH` task is similar to the `switch...case` statement in a programming language. Two [evaluators](#evaluator-types-and-expression) are supported. In either case, the `expression` is evaluated (this could be a simply a referefence to `inputParameters`, or a more complex Javascript expression), and the appropriate task is executed based on the output of the `expression`, and the `decisionCases` defined in the task configuration. -| name | type | description | -|---------------|-------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------| -| evaluatorType | String | [evaluatortType values](#evaluator-types) | -| expression | String | Depends on the [evaluatortType value](#evaluator-types) | -| decisionCases | Map[String, List[task]] | Map where key is possible values that can result from `expression` being evaluated by `evaluatorType` with value being list of tasks to be executed. | -| defaultCase | List[task] | List of tasks to be executed when no matching value is found in decision case (default condition) | +## Use Cases +Useful in any situation where we have to execute one of many task options. +## Configuration -#### Evaluator Types -| name | description | expression | -|-------------|---------------------------------------------------|-----------------------| -| value-param | Use a parameter directly as the value | input parameter | -| javascript | Evaluate JavaScript expressions and compute value | JavaScript expression | +The following parameters are specified at the top level of the task configuration. -### Use Cases +| name | type | description | +| ------------- | ------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| evaluatorType | String (enum) | Type of the evaluator used. Supported types: `value-param`, `javascript`. | +| expression | String | Depends on the [evaluatorType](#evaluator-types-and-expression) | +| decisionCases | Map[String, List\[task]] | Map where the keys are the possible values that can result from `expression` being evaluated by `evaluatorType` with values being lists of tasks to be executed. | +| defaultCase | List\[task] | List of tasks to be executed when no matching value if found in `decisionCases` | -Useful in any situation where we have to execute one of many task options. +Note: If the evaluation type is `value-param`, `inputParameters` must be populated with the key specified in `expression`. +### Evaluator Types and Expression +| evaluatorType | expression | description | +| ------------- | ------------------------------ | ------------------------------------------------- | +| value-param | String (name of key) | Reference to provided key in `inputParameters` | +| javascript | String (JavaScript expression) | Evaluate JavaScript expressions and compute value | -### Output -Following is/are output generated by the `Switch` Task. +## Output | name | type | description | -|------------------|--------------|---------------------------------------------------------------| +| ---------------- | ------------ | ------------------------------------------------------------- | | evaluationResult | List[String] | A List of string representing the list of cases that matched. | -### Examples +## Examples In this example workflow, we have to ship a package with the shipping service providers on the basis of input provided while running the workflow. @@ -85,7 +77,7 @@ is used to determine the switch-case. The evaluator type is `value-param` and th the name of an input parameter. If the value of `switch_case_value` is `fedex` then the decision case `ship_via_fedex`is executed as shown below. -![Conductor UI - Workflow Run](/img/Switch_Fedex.png) +![Conductor UI - Workflow Run](Switch_Fedex.png) In a similar way - if the input was `ups`, then `ship_via_ups` will be executed. If none of the cases match then the default option is executed. diff --git a/docs/docs/reference-docs/terminate-task.md b/docs/documentation/configuration/workflowdef/operators/terminate-task.md similarity index 62% rename from docs/docs/reference-docs/terminate-task.md rename to docs/documentation/configuration/workflowdef/operators/terminate-task.md index b7229db92..9dba25b01 100644 --- a/docs/docs/reference-docs/terminate-task.md +++ b/docs/documentation/configuration/workflowdef/operators/terminate-task.md @@ -1,21 +1,33 @@ ---- -sidebar_position: 1 ---- # Terminate - ```json "type" : "TERMINATE" ``` -### Introduction -Task that can terminate a workflow with a given status and modify the workflow's output with a given parameter, -it can act as a `return` statement for conditions where you simply want to terminate your workflow. -### Use Cases +The `TERMINATE` task can terminate the workflow with a given status and set the workflow's output with the provided value. +It can act as a `return` statement for conditions where you simply want to terminate your workflow. + +## Use Cases Use it when you want to terminate the workflow without continuing the execution. For example, if you have a decision where the first condition is met, you want to execute some tasks, otherwise you want to finish your workflow. -### Configuration +## Configuration +The `TERMINATE` task is configured with the following attributes inside `inputParameters`. + +### inputParameters +| name | type | description | notes | +| ----------------- | ------ | ---------------------------------------------------------------------------------------- | -------- | +| terminationStatus | String | Either `COMPLETED` or `FAILED` | required | +| workflowOutput | Any | Workflow output to be set | | +| terminationReason | String | For failed tasks, this reason is recorded and passed to any configured `failureWorkflow` | | + +## Output +| name | type | description | +| ------ | ---- | --------------------------------------------------------------------------------------------------------- | +| output | Map | The content of `workflowOutput` from the inputParameters. An empty object if `workflowOutput` is not set. | + +## Examples +### Basic Example Terminate task is defined directly inside the workflow with type `TERMINATE`. @@ -34,27 +46,8 @@ Terminate task is defined directly inside the workflow with type } ``` -#### Inputs - -**Parameters:** - -| name | type | description | notes | -|-------------------|--------|-----------------------------------------|-------------------------| -| terminationStatus | String | can only accept "COMPLETED" or "FAILED" | task cannot be optional | -| workflowOutput | Any | Expected workflow output || -|terminationReason|String| For failed tasks, this reason is passed to a failureWorkflow| - -### Output - -**Outputs:** - -| name | type | description | -|--------|------|-----------------------------------------------------------------------------------------------------------| -| output | Map | The content of `workflowOutput` from the inputParameters. An empty object if `workflowOutput` is not set. | - -### Examples - -Let's consider the same example we had in [Switch Task](/reference-docs/switch-task.html). +### Use with SWITCH +Let's consider the same example we had in [Switch Task](switch-task.md). Suppose in a workflow, we have to take decision to ship the courier with the shipping service providers on the basis of input provided while running the workflow. @@ -85,9 +78,9 @@ Here is a snippet that shows the default switch case terminating the workflow: Workflow gets created as shown in the diagram. -![Conductor UI - Workflow Diagram](/img/Terminate_Task.png) +![Conductor UI - Workflow Diagram](Terminate_Task.png) -### Best Practices +## Best Practices 1. Include termination reason when terminating the workflow with failure status to make it easy to understand the cause. 2. Include any additional details (e.g. output of the tasks, switch case etc) that helps understand the path taken to termination. diff --git a/docs/docs/img/workflow_fork.png b/docs/documentation/configuration/workflowdef/operators/workflow_fork.png similarity index 100% rename from docs/docs/img/workflow_fork.png rename to docs/documentation/configuration/workflowdef/operators/workflow_fork.png diff --git a/docs/documentation/configuration/workflowdef/systemtasks/event-task.md b/docs/documentation/configuration/workflowdef/systemtasks/event-task.md new file mode 100644 index 000000000..8153e7ce1 --- /dev/null +++ b/docs/documentation/configuration/workflowdef/systemtasks/event-task.md @@ -0,0 +1,83 @@ +# Event Task + +```json +"type" : "EVENT" +``` + +The `EVENT` task is a task used to publish an event into one of the supported eventing systems in Conductor. + +## Use Cases +Consider a use case where at some point in the execution, an event is published to an external eventing system such as SQS. +Event tasks are useful for creating event based dependencies for workflows and tasks. + +## Supported Queuing Systems +Conductor supports the the following eventing models: + +1. Conductor internal events (prefix: `conductor`) +2. SQS (prefix: `sqs`) +3. [NATS](https://github.com/Netflix/conductor/tree/main/contribs/src/main/java/com/netflix/conductor/contribs/queue/nats) (prefix: `nats`) +4. [AMQP](https://github.com/Netflix/conductor/tree/main/contribs/src/main/java/com/netflix/conductor/contribs/queue/amqp) (prefix: `amqp_queue or amqp_exchange`) + + +## Configuration +The following parameters are specified at the top level of the task configuration. + +| Attribute | Description | +| ------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| sink | External event queue in the format of `prefix:location`. Prefix is either `sqs` or `conductor` and `location` specifies the actual queue name. e.g. `sqs:send_email_queue` | +| asyncComplete | Boolean. [See below](#asynccomplete) | + +### asyncComplete +* ```false``` to mark status COMPLETED upon execution +* ```true``` to keep it IN_PROGRESS, wait for an external event (via Conductor or SQS or EventHandler) to complete it. + +### Conductor sink +When producing an event with Conductor as sink, the event name follows the structure: +```conductor::``` + +When using Conductor as sink, you have two options: defining the sink as `conductor` in which case the queue name will default to the taskReferenceName of the Event Task, or specifying the queue name in the sink, as `conductor:`. The queue name is in the `event` value of the event Handler, as `conductor::`. + +### SQS sink +For SQS, use the **name** of the queue and NOT the URI. Conductor looks up the URI based on the name. + +!!!warning + When using SQS add the [ContribsModule](https://github.com/Netflix/conductor/blob/master/contribs/src/main/java/com/netflix/conductor/contribs/ContribsModule.java) to the deployment. The module needs to be configured with AWSCredentialsProvider for Conductor to be able to use AWS APIs. + + +## Output +Tasks's output are sent as a payload to the external event. In case of SQS the task's output is sent to the SQS message a a payload. + + +| name | type | description | +| ------------------ | ------- | ------------------------------------- | +| workflowInstanceId | String | Workflow id | +| workflowType | String | Workflow Name | +| workflowVersion | Integer | Workflow Version | +| correlationId | String | Workflow CorrelationId | +| sink | String | Copy of the input data "sink" | +| asyncComplete | Boolean | Copy of the input data "asyncComplete | +| event_produced | String | Name of the event produced | + +The published event's payload is identical to the output of the task (except "event_produced"). + + +## Examples + +Consider an example where we want to publish an event into SQS to notify an external system. + +```json +{ + "type": "EVENT", + "sink": "sqs:sqs_queue_name", + "asyncComplete": false +} +``` + +An example where we want to publish a messase to conductor's internal queuing system. +```json +{ + "type": "EVENT", + "sink": "conductor:internal_event_name", + "asyncComplete": false +} +``` \ No newline at end of file diff --git a/docs/documentation/configuration/workflowdef/systemtasks/http-task.md b/docs/documentation/configuration/workflowdef/systemtasks/http-task.md new file mode 100644 index 000000000..a3fa715c3 --- /dev/null +++ b/docs/documentation/configuration/workflowdef/systemtasks/http-task.md @@ -0,0 +1,141 @@ +# HTTP Task + +```json +"type" : "HTTP" +``` + +The `HTTP` task is useful when you have a requirements such as: + +1. Making calls to another service that exposes an API via HTTP +2. Fetch any resource or data present on an endpoint + +The `HTTP` task is moved to `COMPLETED` status once the remote service responds successfully. + + +## Use Cases + +If we have a scenario where we need to make an HTTP call into another service, we can make use of HTTP tasks. You can +use the data returned from the HTTP call in your subsequent tasks as inputs. Using HTTP tasks you can avoid having to +write the code that talks to these services and instead let Conductor manage it directly. This can reduce the code you +have to maintain and allows for a lot of flexibility. + +## Configuration + +HTTP task is configured using the following key inside the `inputParameters` of a task with type `HTTP`. + +### inputParameters +| name | type | description | +| ------------ | ----------- | ---------------------------------------- | +| http_request | HttpRequest | JSON object (see [below](#http_request)) | + +### http_request + +| Name | Type | Description | +| ----------------- | ---------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| uri | String | URI for the service. Can be a partial when using vipAddress or includes the server address. | +| method | String | HTTP method. GET, PUT, POST, DELETE, OPTIONS, HEAD | +| accept | String | Accept header. Default: ```application/json``` | +| contentType | String | Content Type - supported types are ```text/plain```, ```text/html```, and ```application/json``` (Default) | +| headers | Map[String, Any] | A map of additional http headers to be sent along with the request. | +| body | Map[] | Request body | +| asyncComplete | Boolean | ```false``` to mark status COMPLETED upon execution ; ```true``` to keep it IN_PROGRESS, wait for an external event (via Conductor or SQS or EventHandler) to complete it. | +| connectionTimeOut | Integer | Connection Time Out in milliseconds. If set to 0, equivalent to infinity. Default: 100. | +| readTimeOut | Integer | Read Time Out in milliseconds. If set to 0, equivalent to infinity. Default: 150. | + +!!!tip Asynchronous Requests + In the case that remote service sends an asynchronous event to signal the completion of the request, consider setting the `asyncComplete` flag on the HTTP task to `true`. In this case, you will need + to transition the HTTP task to COMPLETED manually. + +!!!tip Authorization Header + If the remote address that you are connecting to is a secure location, add the Authorization header with `Bearer ` to headers. + +## Output + +| name | type | description | +| ------------ | ---------------- | --------------------------------------------------------------------------- | +| response | Map | JSON body containing the response if one is present | +| headers | Map[String, Any] | Response Headers | +| statusCode | Integer | [Http Status Code](https://en.wikipedia.org/wiki/List_of_HTTP_status_codes) | +| reasonPhrase | String | Http Status Code's reason phrase | + +## Examples +### GET Method +We can use variables in our URI as show in the example below. + +```json +{ + "name": "Get Example", + "taskReferenceName": "get_example", + "inputParameters": { + "http_request": { + "uri": "https://jsonplaceholder.typicode.com/posts/${workflow.input.queryid}", + "method": "GET" + } + }, + "type": "HTTP" +} +``` + +### POST Method +Here we are using variables for our POST body which happens to be data from a previous task. This is an example of how you can **chain** HTTP calls to make complex flows happen without writing any additional code. + +```json +{ + "name": "http_post_example", + "taskReferenceName": "post_example", + "inputParameters": { + "http_request": { + "uri": "https://jsonplaceholder.typicode.com/posts/", + "method": "POST", + "body": { + "title": "${get_example.output.response.body.title}", + "userId": "${get_example.output.response.body.userId}", + "action": "doSomething" + } + } + }, + "type": "HTTP" +} +``` + +### PUT Method +```json +{ + "name": "http_put_example", + "taskReferenceName": "put_example", + "inputParameters": { + "http_request": { + "uri": "https://jsonplaceholder.typicode.com/posts/1", + "method": "PUT", + "body": { + "title": "${get_example.output.response.body.title}", + "userId": "${get_example.output.response.body.userId}", + "action": "doSomethingDifferent" + } + } + }, + "type": "HTTP" +} +``` + +### DELETE Method +```json +{ + "name": "DELETE Example", + "taskReferenceName": "delete_example", + "inputParameters": { + "http_request": { + "uri": "https://jsonplaceholder.typicode.com/posts/1", + "method": "DELETE" + } + }, + "type": "HTTP" +} +``` + +## Isolation Groups +Why are my HTTP tasks not getting picked up? + +We might have too many HTTP tasks in the queue. There is a concept called Isolation Groups that you can rely on +for prioritizing certain HTTP tasks over others. + diff --git a/docs/documentation/configuration/workflowdef/systemtasks/human-task.md b/docs/documentation/configuration/workflowdef/systemtasks/human-task.md new file mode 100644 index 000000000..e50d54aff --- /dev/null +++ b/docs/documentation/configuration/workflowdef/systemtasks/human-task.md @@ -0,0 +1,48 @@ +# Human Task +```json +"type" : "HUMAN" +``` + +The `HUMAN` task is used when the workflow needs to be paused for an external signal to continue. It acts as a gate that +remains in the `IN_PROGRESS` state until marked as ```COMPLETED``` or ```FAILED``` by an external trigger. + +## Use Cases +The HUMAN is can be used when the workflow needs to pause and wait for human intervention, such as manual approval. +It can also be used with an event coming from external source such as Kafka, SQS or Conductor's internal queueing mechanism. + +## Configuration +No parameters are required + +## Completing +### Task Update API +To conclude a `HUMAN` task, the `POST {{ api_prefix }}/tasks` [API](../../../api/task.md) can be used. + +You'll need to provide the`taskId`, the task status (generally `COMPLETED` or `FAILED`), and the desired task output. + +### Event Handler +If SQS integration is enabled, the `HUMAN` task can also be resolved using the `{{ api_prefix }}/queue` API. + +You'll need the `workflowId` and `taskRefName` or `taskId`. + +2. POST `{{ api_prefix }}/queue/update/{workflowId}/{taskRefName}/{status}` +3. POST `{{ api_prefix }}/queue/update/{workflowId}/task/{taskId}/{status}` + +An [event handler](../../eventhandlers.md) using the `complete_task` action can also be configured. + +Any parameter that is sent in the body of the POST message will be repeated as the output of the task. For example, if we send a COMPLETED message as follows: + +```bash +curl -X "POST" "{{ server_host }}{{ api_prefix }}/queue/update/{workflowId}/waiting_around_ref/COMPLETED" -H 'Content-Type: application/json' -d '{"data_key":"somedatatoWait1","data_key2":"somedatatoWAit2"}' +``` + +The output of the task will be: + +```json +{ + "data_key":"somedatatoWait1", + "data_key2":"somedatatoWAit2" +} +``` + + + diff --git a/docs/documentation/configuration/workflowdef/systemtasks/index.md b/docs/documentation/configuration/workflowdef/systemtasks/index.md new file mode 100644 index 000000000..023e5ea0b --- /dev/null +++ b/docs/documentation/configuration/workflowdef/systemtasks/index.md @@ -0,0 +1,13 @@ +# System Tasks + +System Tasks (Workers) are built-in tasks that are general purpose and re-usable. They run on the Conductor servers. +Such tasks allow you to get started without having to write custom workers. + +| Task | Description | Use Case | +| :-------------------- | :----------------------------------- | :---------------------------------------------------------------------- | +| Event Publishing | [Event Task](event-task.md) | External eventing system integration. e.g. amqp, sqs, nats | +| HTTP | [HTTP Task](http-task.md) | Invoke any HTTP(S) endpoints | +| Inline Code Execution | [Inline Task](inline-task.md) | Execute arbitrary lightweight javascript code | +| JQ Transform | [JQ Task](json-jq-transform-task.md) | Use [JQ](https://github.com/stedolan/jq) to transform task input/output | +| Kafka Publish | [Kafka Task](kafka-publish-task.md) | Publish messages to Kafka | +| Wait | [Wait Task](wait-task.md) | Block until resolved | diff --git a/docs/documentation/configuration/workflowdef/systemtasks/inline-task.md b/docs/documentation/configuration/workflowdef/systemtasks/inline-task.md new file mode 100644 index 000000000..c5f1686bf --- /dev/null +++ b/docs/documentation/configuration/workflowdef/systemtasks/inline-task.md @@ -0,0 +1,67 @@ +# Inline Task + +```json +"type": "INLINE" +``` + +The `INLINE` task helps execute necessary logic at workflow runtime, +using an evaluator. There are two supported evaluators as of now: + +## Configuration +The `INLINE` task is configured by specifying the following keys inside `inputParameters`, along side any other input values required for the evaluation. + +### inputParameters +| Name | Type | Description | Notes | +| ------------- | ------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------ | +| evaluatorType | String | Type of the evaluator. Supported evaluators: `value-param`, `javascript` which evaluates javascript expression. | Must be non-empty. | +| expression | String | Expression associated with the type of evaluator. For `javascript` evaluator, Javascript evaluation engine is used to evaluate expression defined as a string. Must return a value. | Must be non-empty. | + +Besides `expression`, any value is accessible as `$.value` for the `expression` to evaluate. + +## Outputs + +| Name | Type | Description | +| ------ | ---- | ----------------------------------------------------------------------- | +| result | Map | Contains the output returned by the evaluator based on the `expression` | + +## Examples +### Example 1 +``` json +{ + "name": "INLINE_TASK", + "taskReferenceName": "inline_test", + "type": "INLINE", + "inputParameters": { + "inlineValue": "${workflow.input.inlineValue}", + "evaluatorType": "javascript", + "expression": "function scriptFun(){if ($.inlineValue == 1){ return {testvalue: true} } else { return + {testvalue: false} }} scriptFun();" + } +} +``` + +The task output can then be referenced in downstream tasks using an expression: +`"${inline_test.output.result.testvalue}"` + +!!!tip "Note" + The JavaScript evaluator accepts JS code written to the ECMAScript 5.1(ES5) standard + + +### Example 2 + +Perhaps a weather API sometimes returns Celcius, and sometimes returns Farenheit temperature values. This task ensures that the downstream tasks ONLY receive Celcius values: + +``` +{ + "name": "INLINE_TASK", + "taskReferenceName": "inline_test", + "type": "INLINE", + "inputParameters": { + "scale": "${workflow.input.tempScale}", + "temperature": "${workflow.input.temperature}", + "evaluatorType": "javascript", + "expression": "function SIvaluesOnly(){if ($.scale === "F"){ centigrade = ($.temperature -32)*5/9; return {temperature: centigrade} } else { return + {temperature: $.temperature} }} SIvaluesOnly();" + } +} +``` diff --git a/docs/docs/reference-docs/json-jq-transform-task.md b/docs/documentation/configuration/workflowdef/systemtasks/json-jq-transform-task.md similarity index 52% rename from docs/docs/reference-docs/json-jq-transform-task.md rename to docs/documentation/configuration/workflowdef/systemtasks/json-jq-transform-task.md index 794676a61..a37f5d72a 100644 --- a/docs/docs/reference-docs/json-jq-transform-task.md +++ b/docs/documentation/configuration/workflowdef/systemtasks/json-jq-transform-task.md @@ -1,21 +1,13 @@ ---- -sidebar_position: 1 ---- - # JSON JQ Transform Task ```json "type" : "JSON_JQ_TRANSFORM" ``` -### Introduction -JSON_JQ_TRANSFORM_TASK is a System task that allows processing of JSON data that is supplied to the task, by using the +`JSON_JQ_TRANSFORM` is a System task that allows processing of JSON data that is supplied to the task, by using the popular JQ processing tool’s query expression language. -Check the [JQ Manual](https://stedolan.github.io/jq/manual/v1.5/), and the -[JQ Playground](https://jqplay.org/) for more information on JQ - -### Use Cases +## Use Cases JSON is a popular format of choice for data-interchange. It is widely used in web and server applications, document storage, API I/O etc. It’s also used within Conductor to define workflow and task definitions and passing data and state @@ -23,19 +15,18 @@ between tasks and workflows. This makes a tool like JQ a natural fit for process usages within Conductor includes, working with HTTP task, JOIN tasks or standalone tasks that try to transform data from the output of one task to the input of another. -### Configuration +## Configuration +`queryExpression` is appended to the `inputParameters` of `JSON_JQ_TRANSFORM`, along side any other input values needed for the evaluation. +### inputParameters +| name | description | +|-----------------|---------------------| +| queryExpression | JQ query expression | -| Attribute | Description | -|-------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| name | Task Name. A unique name that is descriptive of the task function | -| taskReferenceName | Task Reference Name. A unique reference to this task. There can be multiple references of a task within the same workflow definition | -| type | Task Type. In this case, JSON_JQ_TRANSFORM | -| inputParameters | The input parameters that will be supplied to this task. The parameters will be a JSON object of atleast 2 attributes, one of which will be called queryExpression. The others are user named attributes. These attributes will be accessible by the JQ query processor | -| inputParameters/user-defined-key(s) | User defined key(s) along with values. | -| inputParameters/queryExpression | A JQ query expression | +### About JQ +Check out the [JQ Manual](https://stedolan.github.io/jq/manual/v1.5/). -#### Output Configuration +## Output | Attribute | Description | |------------|---------------------------------------------------------------------------| @@ -43,9 +34,8 @@ the output of one task to the input of another. | resultList | A List of results returned by the JQ expression | | error | An optional error message, indicating that the JQ query failed processing | -### Example - - +## Example +### Example 1 Here is an example of a _`JSON_JQ_TRANSFORM`_ task. The `inputParameters` attribute is expected to have a value object that has the following @@ -109,14 +99,10 @@ attribute along with a string message will be returned if there was an error pro } ``` -## Example JQ transforms - -### Cleaning up a JSON response - +### Example 2 A HTTP Task makes an API call to GitHub to request a list of "stargazers" (users who have starred a repository). The API response (for just one user) looks like: - -Snippet of ```${hundred_stargazers_ref.output}``` +Snippet of `${hundred_stargazers_ref.output}` ``` JSON @@ -145,33 +131,32 @@ Snippet of ```${hundred_stargazers_ref.output}``` } } ] - ``` -We only need the ```starred_at``` and ```login``` parameters for users who starred the repository AFTER a given date (provided as an input to the workflow ```${workflow.input.cutoff_date}```). We'll use the JQ Transform to simplify the output: +We only need the ```starred_at``` and ```login``` parameters for users who starred the repository AFTER a given date (provided as an input to the workflow `${workflow.input.cutoff_date}`). We'll use the JQ Transform to simplify the output: ```JSON { - "name": "jq_cleanup_stars", - "taskReferenceName": "jq_cleanup_stars_ref", - "inputParameters": { - "starlist": "${hundred_stargazers_ref.output.response.body}", - "queryExpression": "[.starlist[] | select (.starred_at > \"${workflow.input.cutoff_date}\") |{occurred_at:.starred_at, member: {github: .user.login}}]" - }, - "type": "JSON_JQ_TRANSFORM", - "decisionCases": {}, - "defaultCase": [], - "forkTasks": [], - "startDelay": 0, - "joinOn": [], - "optional": false, - "defaultExclusiveJoinTask": [], - "asyncComplete": false, - "loopOver": [] - } + "name": "jq_cleanup_stars", + "taskReferenceName": "jq_cleanup_stars_ref", + "inputParameters": { + "starlist": "${hundred_stargazers_ref.output.response.body}", + "queryExpression": "[.starlist[] | select (.starred_at > \"${workflow.input.cutoff_date}\") |{occurred_at:.starred_at, member: {github: .user.login}}]" + }, + "type": "JSON_JQ_TRANSFORM", + "decisionCases": {}, + "defaultCase": [], + "forkTasks": [], + "startDelay": 0, + "joinOn": [], + "optional": false, + "defaultExclusiveJoinTask": [], + "asyncComplete": false, + "loopOver": [] +} ``` -The JSON is stored in ```starlist```. The ```queryExpression``` reads in the JSON, selects only entries where the ```starred_at``` value meets the date criteria, and generates output JSON of the form: +The JSON is stored in `starlist`. The `queryExpression` reads in the JSON, selects only entries where the `starred_at` value meets the date criteria, and generates output JSON of the form: ```JSON { @@ -182,7 +167,7 @@ The JSON is stored in ```starlist```. The ```queryExpression``` reads in the JS } ``` -The entire expression is wrapped in [] to indicate that the response should be an array. +The entire expression is wrapped in `[]` to indicate that the response should be an array. diff --git a/docs/docs/reference-docs/kafka-publish-task.md b/docs/documentation/configuration/workflowdef/systemtasks/kafka-publish-task.md similarity index 84% rename from docs/docs/reference-docs/kafka-publish-task.md rename to docs/documentation/configuration/workflowdef/systemtasks/kafka-publish-task.md index d7cda5b9f..60832d2a8 100644 --- a/docs/docs/reference-docs/kafka-publish-task.md +++ b/docs/documentation/configuration/workflowdef/systemtasks/kafka-publish-task.md @@ -1,35 +1,35 @@ ---- -sidebar_position: 13 ---- - # Kafka Publish Task ```json "type" : "KAFKA_PUBLISH" ``` +The `KAFKA_PUBLISH` task is used to push messages to another microservice via Kafka. -### Introduction +## Configuration +The task expects a field named `kafka_request` as part of the task's `inputParameters`. -A Kafka Publish task is used to push messages to another microservice via Kafka. +### inputParameters +| name | description | +| ---------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| bootStrapServers | bootStrapServers for connecting to given kafka. | +| key | Key to be published | +| keySerializer | Serializer used for serializing the key published to kafka. One of the following can be set : 1. `org.apache.kafka.common.serialization.IntegerSerializer`
2. `org.apache.kafka.common.serialization.LongSerializer`
3. `org.apache.kafka.common.serialization.StringSerializer`.
Default is `StringSerializer` | +| value | Value published to kafka | +| requestTimeoutMs | Request timeout while publishing to kafka. If this value is not given the value is read from the property `kafka.publish.request.timeout.ms`. If the property is not set the value defaults to 100 ms | +| maxBlockMs | maxBlockMs while publishing to kafka. If this value is not given the value is read from the property `kafka.publish.max.block.ms`. If the property is not set the value defaults to 500 ms | +| headers | A map of additional kafka headers to be sent along with the request. | +| topic | Topic to publish | -### Configuration -The task expects an input parameter named ```kafka_request``` as part of the task's input with the following details: -| name | description | -|------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| bootStrapServers | bootStrapServers for connecting to given kafka. | -| key | Key to be published | -| keySerializer | Serializer used for serializing the key published to kafka. One of the following can be set :
1. org.apache.kafka.common.serialization.IntegerSerializer
2. org.apache.kafka.common.serialization.LongSerializer
3. org.apache.kafka.common.serialization.StringSerializer.
Default is String serializer | -| value | Value published to kafka | -| requestTimeoutMs | Request timeout while publishing to kafka. If this value is not given the value is read from the property `kafka.publish.request.timeout.ms`. If the property is not set the value defaults to 100 ms | -| maxBlockMs | maxBlockMs while publishing to kafka. If this value is not given the value is read from the property `kafka.publish.max.block.ms`. If the property is not set the value defaults to 500 ms | -| headers | A map of additional kafka headers to be sent along with the request. | -| topic | Topic to publish | +## Task Output -### Examples +Task status transitions to `COMPLETED` on success. -Sample Task +The task is marked as `FAILED` if the message could not be published to +the Kafka queue. +## Example + ```json { "name": "call_kafka", @@ -78,10 +78,3 @@ the cache size is 10 and expiry time is 120000 ms. To change the defaults following can be modified kafka.publish.producer.cache.size, kafka.publish.producer.cache.time.ms respectively. - -#### Kafka Task Output - -Task status transitions to `COMPLETED`. - -The task is marked as `FAILED` if the message could not be published to -the Kafka queue. diff --git a/docs/documentation/configuration/workflowdef/systemtasks/wait-task.md b/docs/documentation/configuration/workflowdef/systemtasks/wait-task.md new file mode 100644 index 000000000..c164ec00e --- /dev/null +++ b/docs/documentation/configuration/workflowdef/systemtasks/wait-task.md @@ -0,0 +1,52 @@ +# Wait Task +The WAIT task is a no-op task that will remain `IN_PROGRESS` until after a certain duration or timestamp, at which point it will be marked as `COMPLETED`. + +```json +"type" : "WAIT" +``` + +## Configuration +The `WAIT` task is configured using **either** `duration` **or** `until` in `inputParameters`. + +### inputParameters +| name | type | description | +| -------- | ------ | ----------------------- | +| duration | String | Duration to wait for | +| until | String | Timestamp to wait until | + +### Wait For time duration + +Format duration as ```XhYmZs```, using the `duration` key. + +```json +{ + "type": "WAIT", + "inputParameters": { + "duration": "10m20s" + } +} +``` + +### Wait until specific date/time + +Specify the timestamp using one of the formats, using the `until` key. + +1. ```yyyy-MM-dd HH:mm``` +2. ```yyyy-MM-dd HH:mm z``` +3. ```yyyy-MM-dd``` + +```json +{ + "type": "WAIT", + "inputParameters": { + "until": "2022-12-31 11:59" + } +} +``` +## External Triggers + +The task endpoint `POST {{ api_prefix }}/tasks` can be used to update the status of a task to COMPLETED prior to the configured timeout. This is +same technique as prescribed for the [HUMAN](human-task.md#completing) task. + +For cases where no timeout is necessary it is recommended that you use the [HUMAN](human-task.md) task directly. + diff --git a/docs/docs/metrics/client.md b/docs/documentation/metrics/client.md similarity index 100% rename from docs/docs/metrics/client.md rename to docs/documentation/metrics/client.md diff --git a/docs/docs/metrics/server.md b/docs/documentation/metrics/server.md similarity index 100% rename from docs/docs/metrics/server.md rename to docs/documentation/metrics/server.md diff --git a/docs/docs/img/tutorial/Switch_UPS.png b/docs/home/devex.png similarity index 100% rename from docs/docs/img/tutorial/Switch_UPS.png rename to docs/home/devex.png diff --git a/docs/docs/img/icons/brackets.svg b/docs/home/icons/brackets.svg similarity index 100% rename from docs/docs/img/icons/brackets.svg rename to docs/home/icons/brackets.svg diff --git a/docs/docs/img/favicon.svg b/docs/home/icons/conductor.svg similarity index 100% rename from docs/docs/img/favicon.svg rename to docs/home/icons/conductor.svg diff --git a/docs/docs/img/icons/modular.svg b/docs/home/icons/modular.svg similarity index 100% rename from docs/docs/img/icons/modular.svg rename to docs/home/icons/modular.svg diff --git a/docs/docs/img/icons/network.svg b/docs/home/icons/network.svg similarity index 100% rename from docs/docs/img/icons/network.svg rename to docs/home/icons/network.svg diff --git a/docs/docs/img/icons/osi.svg b/docs/home/icons/osi.svg similarity index 100% rename from docs/docs/img/icons/osi.svg rename to docs/home/icons/osi.svg diff --git a/docs/docs/img/icons/server.svg b/docs/home/icons/server.svg similarity index 100% rename from docs/docs/img/icons/server.svg rename to docs/home/icons/server.svg diff --git a/docs/docs/img/icons/shield.svg b/docs/home/icons/shield.svg similarity index 100% rename from docs/docs/img/icons/shield.svg rename to docs/home/icons/shield.svg diff --git a/docs/docs/img/icons/wrench.svg b/docs/home/icons/wrench.svg similarity index 100% rename from docs/docs/img/icons/wrench.svg rename to docs/home/icons/wrench.svg diff --git a/docs/home/redirect.html b/docs/home/redirect.html new file mode 100644 index 000000000..819b7fef6 --- /dev/null +++ b/docs/home/redirect.html @@ -0,0 +1,8 @@ + + + + + +

Redirecting ...

+ + \ No newline at end of file diff --git a/docs/docs/img/timeline.png b/docs/home/timeline.png similarity index 100% rename from docs/docs/img/timeline.png rename to docs/home/timeline.png diff --git a/docs/docs/img/workflow.svg b/docs/home/workflow.svg similarity index 100% rename from docs/docs/img/workflow.svg rename to docs/home/workflow.svg diff --git a/docs/docs/img/logo.svg b/docs/img/logo.svg similarity index 100% rename from docs/docs/img/logo.svg rename to docs/img/logo.svg diff --git a/docs/docs/img/netflix-oss.png b/docs/img/netflix-oss.png similarity index 100% rename from docs/docs/img/netflix-oss.png rename to docs/img/netflix-oss.png diff --git a/docs/docs/index.md b/docs/index.md similarity index 73% rename from docs/docs/index.md rename to docs/index.md index c1a0bd096..f9a4f8a6f 100644 --- a/docs/docs/index.md +++ b/docs/index.md @@ -1,28 +1,34 @@ -
+--- +hide: + - navigation + - toc +--- +
+
Scalable Workflow Orchestration
-
+
Conductor is a platform created by Netflix to orchestrate workflows that span across microservices. -
- +
- +
-
+
- Open Source + Open Source
Apache-2.0 license for commercial and non-commerical use. Freedom to deploy, modify and contribute back. @@ -30,7 +36,7 @@
- Modular + Modular
A fully abstracted backend enables you choose your own database persistence layer and queueing service. @@ -38,7 +44,7 @@
- Proven + Proven
Enterprise ready, Java Spring based platform that has been battle tested in production systems at Netflix and elsewhere. @@ -50,38 +56,38 @@
- Control + Control
-
+
Powerful flow control constructs including Decisions, Dynamic Fork-Joins and Subworkflows. Variables and templates are supported.
- Polyglot + Polyglot
-
+
Client libraries in multiple languages allows workers to be implemented in Java, Node JS, Python and C#.
- Scalable + Scalable
-
+
Distributed architecture for both orchestrator and workers scalable from a single workflow to millions of concurrent processes.
-
-
+
+
Developer Experience
-
+
  • Discover and visualize the process flows from the bundled UI
  • Integrated interface to create, refine and validate workflows
  • @@ -91,12 +97,12 @@
-
+
-
+
@@ -110,7 +116,7 @@
-
+
@@ -119,17 +125,15 @@
-
-

Why Conductor?

-
+

Why Conductor?

-
+

- Service Orchestration + Service Orchestration

-
+

Workflow definitions are decoupled from task implementations. This allows the creation of process flows in which each individual task can be implemented by an encapsulated microservice.

Designing a workflow orchestrator that is resilient and horizontally scalable is not a simple problem. At Netflix we have developed a solution in Conductor.

@@ -139,9 +143,9 @@

- Service Choreography + Service Choreography

-
+
Process flows are implicitly defined across multiple service implementations, often with tight peer-to-peer coupling between services. Multiple event buses and complex pub/sub models limit observability around process progress and capacity. @@ -151,3 +155,4 @@
+
\ No newline at end of file diff --git a/docs/kitchensink.json b/docs/kitchensink.json deleted file mode 100644 index 5e1e50de8..000000000 --- a/docs/kitchensink.json +++ /dev/null @@ -1,153 +0,0 @@ -{ - "name": "kitchensink", - "description": "kitchensink workflow", - "version": 1, - "tasks": [ - { - "name": "search_elasticsearch", - "taskReferenceName": "get_es_0", - "inputParameters": { - "http_request": { - "uri": "http://localhost:9200/conductor/workflow/_search?q=status:COMPLETED&size=10", - "method": "GET" - } - }, - "type": "HTTP" - }, - { - "name": "task_1", - "taskReferenceName": "task_1", - "inputParameters": { - "mod": "${workflow.input.mod}", - "oddEven": "${workflow.input.oddEven}" - }, - "type": "SIMPLE" - }, - { - "name": "dyntask", - "taskReferenceName": "task_2", - "inputParameters": { - "taskToExecute": "${workflow.input.task2Name}" - }, - "type": "DYNAMIC", - "dynamicTaskNameParam": "taskToExecute" - }, - { - "name": "oddEvenDecision", - "taskReferenceName": "oddEvenDecision", - "inputParameters": { - "oddEven": "${task_2.output.oddEven}" - }, - "type": "DECISION", - "caseValueParam": "oddEven", - "decisionCases": { - "0": [ - { - "name": "task_4", - "taskReferenceName": "task_4", - "inputParameters": { - "mod": "${task_2.output.mod}", - "oddEven": "${task_2.output.oddEven}" - }, - "type": "SIMPLE" - }, - { - "name": "dynamic_fanout", - "taskReferenceName": "fanout1", - "inputParameters": { - "dynamicTasks": "${task_4.output.dynamicTasks}", - "input": "${task_4.output.inputs}" - }, - "type": "FORK_JOIN_DYNAMIC", - "dynamicForkTasksParam": "dynamicTasks", - "dynamicForkTasksInputParamName": "input" - }, - { - "name": "dynamic_join", - "taskReferenceName": "join1", - "type": "JOIN" - } - ], - "1": [ - { - "name": "fork_join", - "taskReferenceName": "forkx", - "type": "FORK_JOIN", - "forkTasks": [ - [ - { - "name": "task_10", - "taskReferenceName": "task_10", - "type": "SIMPLE" - }, - { - "name": "sub_workflow_x", - "taskReferenceName": "wf3", - "inputParameters": { - "mod": "${task_1.output.mod}", - "oddEven": "${task_1.output.oddEven}" - }, - "type": "SUB_WORKFLOW", - "subWorkflowParam": { - "name": "sub_flow_1", - "version": 1 - } - } - ], - [ - { - "name": "task_11", - "taskReferenceName": "task_11", - "type": "SIMPLE" - }, - { - "name": "sub_workflow_x", - "taskReferenceName": "wf4", - "inputParameters": { - "mod": "${task_1.output.mod}", - "oddEven": "${task_1.output.oddEven}" - }, - "type": "SUB_WORKFLOW", - "subWorkflowParam": { - "name": "sub_flow_1", - "version": 1 - } - } - ] - ] - }, - { - "name": "join", - "taskReferenceName": "join2", - "type": "JOIN", - "joinOn": [ - "wf3", - "wf4" - ] - } - ] - } - }, - { - "name": "search_elasticsearch", - "taskReferenceName": "get_es_1", - "inputParameters": { - "http_request": { - "uri": "http://localhost:9200/conductor/workflow/_search?q=status:COMPLETED&size=10", - "method": "GET" - } - }, - "type": "HTTP" - }, - { - "name": "task_30", - "taskReferenceName": "task_30", - "inputParameters": { - "statuses": "${get_es_1.output...status}", - "fistWorkflowId": "${get_es_1.output.workflowId[0]}" - }, - "type": "SIMPLE" - } - ], - "schemaVersion": 2 -} \ No newline at end of file diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml deleted file mode 100644 index 80ba1f9b7..000000000 --- a/docs/mkdocs.yml +++ /dev/null @@ -1,114 +0,0 @@ -site_name: Conductor Documentation -site_description: Conductor is a platform created by Netflix to orchestrate workflows that span across microservices. -repo_url: https://github.com/Netflix/conductor -edit_uri: '' -strict: true -use_directory_urls: false - -nav: - - Getting Started: - - Running Conductor: - - From Source: gettingstarted/source.md - - Using Docker: gettingstarted/docker.md - - Hosted Solutions: gettingstarted/hosted.md - - Basic Concepts: gettingstarted/basicconcepts.md - - High Level Steps: gettingstarted/steps.md - - Using the Client: gettingstarted/client.md - - Start a Workflow: gettingstarted/startworkflow.md - - Why Conductor?: gettingstarted/intro.md - - How-Tos: - - Workflows: - - Debugging Workflows: how-tos/Workflows/debugging-workflows.md - - Handling Errors: how-tos/Workflows/handling-errors.md - - Searching Workflows: how-tos/Workflows/searching-workflows.md - - Starting Workflows: how-tos/Workflows/starting-workflows.md - - Updating Workflows: how-tos/Workflows/updating-workflows.md - - View Workflow Execution: how-tos/Workflows/view-workflow-executions.md - - Versioning Workflows: how-tos/Workflows/versioning-workflows.md - - Tasks: - - Creating Task Definitions: how-tos/Tasks/creating-tasks.md - - Dynamic vs Switch Tasks: how-tos/Tasks/dynamic-vs-switch-tasks.md - - Monitoring Task Queues: how-tos/Tasks/monitoring-task-queues.md - - Reusing Tasks: how-tos/Tasks/reusing-tasks.md - - Task Configurations: how-tos/Tasks/task-configurations.md - - Task Inputs: how-tos/Tasks/task-inputs.md - - Task Timeouts: how-tos/Tasks/task-timeouts.md - - Updating Task Definitions: how-tos/Tasks/updating-tasks.md - - Extending System Tasks: how-tos/Tasks/extending-system-tasks.md - - Workers: - - Build a Go Task Worker: how-tos/Workers/build-a-golang-task-worker.md - - Build a Java Task Worker: how-tos/Workers/build-a-java-task-worker.md - - Build a Python Task Worker: how-tos/Workers/build-a-python-task-worker.md - - Monitoring: - - Conductor Log Level: how-tos/Monitoring/Conductor-LogLevel.md - - Developer Labs: - - Beginner: labs/beginner.md - - A First Workflow: labs/running-first-workflow.md - - Events and Event Handlers: labs/eventhandlers.md - - Kitchen Sink: labs/kitchensink.md - - Documentation: - - Architecture: - - Overview: architecture/overview.md - - Task Lifecycle: architecture/tasklifecycle.md - - API Specification: apispec.md - - Configuration: - - Task Definition: configuration/taskdef.md - - Worker Definition: configuration/workerdef.md - - Workflow Definition: configuration/workflowdef.md - - System Tasks: configuration/systask.md - - System Operators: configuration/sysoperator.md - - Event Handlers: configuration/eventhandlers.md - - Task Domains: configuration/taskdomains.md - - Isolation Groups: configuration/isolationgroups.md - - Operators: - - Do-While: reference-docs/do-while-task.md - - Dynamic: reference-docs/dynamic-task.md - - Dynamic Fork: reference-docs/dynamic-fork-task.md - - Fork: reference-docs/fork-task.md - - Join: reference-docs/join-task.md - - Set Variable: reference-docs/set-variable-task.md - - Start Workflow: reference-docs/start-workflow-task.md - - Sub Workflow: reference-docs/sub-workflow-task.md - - Switch: reference-docs/switch-task.md - - Terminate: reference-docs/terminate-task.md - - Wait: reference-docs/wait-task.md - - System Tasks: - - Event Task: reference-docs/event-task.md - - HTTP Task: reference-docs/http-task.md - - Human Task: reference-docs/human-task.md - - Inline Task: reference-docs/inline-task.md - - JSON JQ Transform Task: reference-docs/json-jq-transform-task.md - - Kafka Publish Task: reference-docs/kafka-publish-task.md - - Conductor Metrics: - - Server Metrics: metrics/server.md - - Client Metrics: metrics/client.md - - Advanced Topics: - - Extending Conductor: extend.md - - Annotation Processor: reference-docs/annotation-processor.md - - Archival of Workflows: reference-docs/archival-of-workflows.md - - Azure Blob Storage: reference-docs/azureblob-storage.md - - External Payload Storage: externalpayloadstorage.md - - Redis: reference-docs/redis.md - - SDKs: - - CSharp SDK: how-tos/csharp-sdk.md - - Clojure SDK: how-tos/clojure-sdk.md - - Go SDK: how-tos/go-sdk.md - - Python SDK: how-tos/python-sdk.md - - - Best Practices: bestpractices.md - - FAQ: faq.md - - Directed Acyclic Graph: reference-docs/directed-acyclic-graph.md - - Technical Details: technicaldetails.md - - Resources: - - Contributing: resources/contributing.md - - Code of Conduct: resources/code-of-conduct.md - - Related Projects: resources/related.md - - License: resources/license.md -theme: - name: mkdocs - custom_dir: theme/ -extra_css: - - css/custom.css -markdown_extensions: -- admonition -- codehilite diff --git a/docs/docs/resources/contributing.md b/docs/resources/contributing.md similarity index 98% rename from docs/docs/resources/contributing.md rename to docs/resources/contributing.md index 89af84735..c80f6bbf6 100644 --- a/docs/docs/resources/contributing.md +++ b/docs/resources/contributing.md @@ -5,7 +5,7 @@ This guide helps to find the most efficient way to contribute, ask questions, an Code of conduct ----- -Please review our [code of conduct](code-of-conduct.md). +Please review our [code of conduct](https://orkes.io/orkes-conductor-community-code-of-conduct). I have a question! ----- diff --git a/docs/docs/resources/license.md b/docs/resources/license.md similarity index 94% rename from docs/docs/resources/license.md rename to docs/resources/license.md index 518de4064..4ae98902f 100644 --- a/docs/docs/resources/license.md +++ b/docs/resources/license.md @@ -1,6 +1,6 @@ # License -Copyright 2022 Netflix, Inc. +Copyright 2023 Conductor authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/docs/docs/resources/related.md b/docs/resources/related.md similarity index 100% rename from docs/docs/resources/related.md rename to docs/resources/related.md diff --git a/docs/theme/main.html b/docs/theme/main.html deleted file mode 100644 index c916e7dbc..000000000 --- a/docs/theme/main.html +++ /dev/null @@ -1,42 +0,0 @@ -{% extends "base.html" %} - -{% block extrahead %} - - - - - -{% endblock %} - - -{% block content %} -{% if page and page.is_homepage %} -
{% include "content.html" %}
-{% else %} -
{% include "toc.html" %}
-
{% include "content.html" %}
-{% endif %} -{% endblock %} - -{%- block next_prev %} -{%- endblock %} - -{% block footer %} - -{% endblock %} \ No newline at end of file diff --git a/docs/theme/toc-sub.html b/docs/theme/toc-sub.html deleted file mode 100644 index 25e9da180..000000000 --- a/docs/theme/toc-sub.html +++ /dev/null @@ -1,14 +0,0 @@ -{%- if not nav_item.children %} -
  • - {{ nav_item.title }} -
  • -{%- else %} -
  • - {{ nav_item.title }} -
      - {%- for nav_item in nav_item.children %} - {% include "toc-sub.html" %} - {%- endfor %} -
    -
  • -{%- endif %} diff --git a/docs/theme/toc.html b/docs/theme/toc.html deleted file mode 100644 index 240974784..000000000 --- a/docs/theme/toc.html +++ /dev/null @@ -1,7 +0,0 @@ -
    -
      - {%- for nav_item in nav %} - {% include "toc-sub.html" %} - {%- endfor %} -
    -
    \ No newline at end of file diff --git a/main.py b/main.py new file mode 100644 index 000000000..6ce763a26 --- /dev/null +++ b/main.py @@ -0,0 +1,84 @@ +import os +import re + +def define_env(env): + "Hook function" + + @env.macro + def insert_content(key = None): + key = key or env.page.title + filename = env.variables['extra']['additional_content'][key] + return include_file(filename) + + @env.macro + def include_file(filename): + prefix = env.variables['config']['docs_dir'] + full_filename = os.path.join(prefix, filename) + with open(full_filename, 'r') as f: + lines = f.readlines() + return ''.join(lines) + + + """ + def copy_markdown_images(tmpRoot, markdown): + # root = os.path.dirname(os.path.dirname(self.page.url)) + root = self.page.url + + paths = [] + + p = re.compile("!\[.*\]\((.*)\)") + it = p.finditer(markdown) + for match in it: + path = match.group(1) + paths.append(path) + + destinationPath = os.path.realpath(self.config['base_path'] + "/" + + root + "/gen_/" + path) + + if not os.path.isfile(destinationPath): + print("Copying image: " + path + " to " + destinationPath) + + os.makedirs(os.path.dirname(destinationPath), exist_ok=True) + shutil.copyfile(tmpRoot + "/" + path, destinationPath) + + for path in paths: + markdown = markdown.replace(path, "gen_/" + path) + + return markdown + """ + + @env.macro + def snippet(file_path, section_name, num_sections=1): + p = re.compile("^#+ ") + m = p.search(section_name) + if m: + section_level = m.span()[1] - 1 + root = env.variables['config']['docs_dir'] + full_path = os.path.join(root, file_path) + + content = "" + with open(full_path, 'r') as myfile: + content = myfile.read() + + p = re.compile("^" + section_name + "$", re.MULTILINE) + start = p.search(content) + start_span = start.span() + p = re.compile("^#{1," + str(section_level) + "} ", re.MULTILINE) + + result = "" + all = [x for x in p.finditer(content[start_span[1]:])] + + print (len(all)) + + if len(all) == 0 or (num_sections-1) >= len(all): + result = content[start_span[0]:] + else: + end = all[num_sections-1] + end_index = end.span()[0] + result = content[start_span[0]:end_index + start_span[1]] + + # If there are any images, find them, copy them + # result = copy_markdown_images(root, result) + return result + else: + return "Heading reference beginning in # is required" diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 000000000..195ce52ba --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,139 @@ +site_name: Conductor Documentation +site_description: Conductor is a platform created by Netflix to orchestrate workflows that span across microservices. +repo_url: https://github.com/conductor-oss/conductor +site_url: https://c4lm.github.io/conductor-docs +edit_uri: '' +strict: false +use_directory_urls: false + +nav: + - index.md + - Developer's Guide: + - Concepts: + - devguide/concepts/index.md + - devguide/concepts/workflows.md + - devguide/concepts/tasks.md + - devguide/concepts/workers.md + - devguide/concepts/why.md + - Running Conductor: + - devguide/running/source.md + - devguide/running/docker.md + - devguide/running/hosted.md + - Architecture: + - devguide/architecture/index.md + - devguide/architecture/tasklifecycle.md + - devguide/architecture/technicaldetails.md + - devguide/architecture/directed-acyclic-graph.md + - How Tos: + - Workflows: + - devguide/how-tos/Workflows/debugging-workflows.md + - devguide/how-tos/Workflows/handling-errors.md + - devguide/how-tos/Workflows/searching-workflows.md + - devguide/how-tos/Workflows/starting-workflows.md + - devguide/how-tos/Workflows/updating-workflows.md + - devguide/how-tos/Workflows/view-workflow-executions.md + - devguide/how-tos/Workflows/versioning-workflows.md + - Tasks: + - devguide/how-tos/Tasks/creating-tasks.md + - devguide/how-tos/Tasks/dynamic-vs-switch-tasks.md + - devguide/how-tos/Tasks/monitoring-task-queues.md + - devguide/how-tos/Tasks/reusing-tasks.md + - devguide/how-tos/Tasks/task-configurations.md + - devguide/how-tos/Tasks/task-inputs.md + - devguide/how-tos/Tasks/task-timeouts.md + - devguide/how-tos/Tasks/updating-tasks.md + - devguide/how-tos/Tasks/extending-system-tasks.md + - Workers: + - devguide/how-tos/Workers/build-a-golang-task-worker.md + - devguide/how-tos/Workers/build-a-java-task-worker.md + - devguide/how-tos/Workers/build-a-python-task-worker.md + - Monitoring: + - devguide/how-tos/Monitoring/Conductor-LogLevel.md + - Tutorial: + - devguide/labs/index.md + - devguide/labs/first-workflow.md + - devguide/labs/eventhandlers.md + - devguide/labs/kitchensink.md + - FAQ: devguide/faq.md + - Best Practices: devguide/bestpractices.md + - Documentation: + - API Specification: + - documentation/api/index.md + - documentation/api/metadata.md + - documentation/api/startworkflow.md + - documentation/api/workflow.md + - documentation/api/task.md + - documentation/api/taskdomains.md + - Configuration: + - Workflow Definition: + - documentation/configuration/workflowdef/index.md + - Operators: + - documentation/configuration/workflowdef/operators/index.md + - documentation/configuration/workflowdef/operators/do-while-task.md + - documentation/configuration/workflowdef/operators/dynamic-task.md + - documentation/configuration/workflowdef/operators/dynamic-fork-task.md + - documentation/configuration/workflowdef/operators/fork-task.md + - documentation/configuration/workflowdef/operators/join-task.md + - documentation/configuration/workflowdef/operators/set-variable-task.md + - documentation/configuration/workflowdef/operators/start-workflow-task.md + - documentation/configuration/workflowdef/operators/sub-workflow-task.md + - documentation/configuration/workflowdef/operators/switch-task.md + - documentation/configuration/workflowdef/operators/terminate-task.md + - System Tasks: + - documentation/configuration/workflowdef/systemtasks/index.md + - documentation/configuration/workflowdef/systemtasks/event-task.md + - documentation/configuration/workflowdef/systemtasks/http-task.md + - documentation/configuration/workflowdef/systemtasks/human-task.md + - documentation/configuration/workflowdef/systemtasks/inline-task.md + - documentation/configuration/workflowdef/systemtasks/json-jq-transform-task.md + - documentation/configuration/workflowdef/systemtasks/kafka-publish-task.md + - documentation/configuration/workflowdef/systemtasks/wait-task.md + - documentation/configuration/taskdef.md + - documentation/configuration/eventhandlers.md + - Conductor Metrics: + - Server Metrics: documentation/metrics/server.md + - Client Metrics: documentation/metrics/client.md + - Advanced Topics: + - documentation/advanced/extend.md + - documentation/advanced/isolationgroups.md + - documentation/advanced/annotation-processor.md + - documentation/advanced/archival-of-workflows.md + - documentation/advanced/azureblob-storage.md + - documentation/advanced/externalpayloadstorage.md + - documentation/advanced/redis.md + - Client SDKs: + - documentation/clientsdks/index.md + - documentation/clientsdks/java-sdk.md + - documentation/clientsdks/csharp-sdk.md + - documentation/clientsdks/clojure-sdk.md + - documentation/clientsdks/go-sdk.md + - documentation/clientsdks/python-sdk.md + - Resources: + - Contributing: resources/contributing.md + - Code of Conduct: resources/code-of-conduct.md + - Related Projects: resources/related.md + - License: resources/license.md +theme: + name: material + logo: img/logo.svg + features: + - navigation.tabs + - navigation.tabs.sticky + - navigation.indexes + - navigation.footer +extra_css: + - css/custom.css +plugins: + - search + - macros +markdown_extensions: + - admonition + - codehilite + - attr_list + - pymdownx.highlight: + anchor_linenums: true + - pymdownx.inlinehilite + - pymdownx.snippets + - pymdownx.superfences +copyright: Conductor authors + diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..070b90152 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +mkdocs +mkdocs-material +mkdocs-macros-plugin \ No newline at end of file