merge main

ghkdwlgns612 · May 16, 2022 · d0add13 · d0add13
2 parents b0925ac + bc9df66
commit d0add13
Show file tree

Hide file tree

Showing 381 changed files with 48,169 additions and 36,800 deletions.
diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -37,12 +37,12 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
         run: |
-          ./gradlew build -x :conductor-contribs:test --scan
+          ./gradlew build --scan
       - name: Build and Publish snapshot
         if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main'
         run: |
           echo "Running build for commit ${{ github.sha }}"
-          ./gradlew build -x :conductor-contribs:test snapshot --scan
+          ./gradlew build snapshot --scan
         env:
           NETFLIX_OSS_SIGNING_KEY: ${{ secrets.ORG_SIGNING_KEY }}
           NETFLIX_OSS_SIGNING_PASSWORD: ${{ secrets.ORG_SIGNING_PASSWORD }}

diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -30,7 +30,7 @@ jobs:
             ${{ runner.os }}-gradle-
       - name: Publish candidate
         if: startsWith(github.ref, 'refs/tags/v') && contains(github.ref, '-rc.')
-        run: ./gradlew -x :conductor-contribs:test -Prelease.useLastTag=true candidate --scan
+        run: ./gradlew -Prelease.useLastTag=true candidate --scan
         env:
           NETFLIX_OSS_SONATYPE_USERNAME: ${{ secrets.ORG_SONATYPE_USERNAME }}
           NETFLIX_OSS_SONATYPE_PASSWORD: ${{ secrets.ORG_SONATYPE_PASSWORD }}
@@ -40,11 +40,22 @@ jobs:
           NETFLIX_OSS_REPO_PASSWORD: ${{ secrets.ORG_NETFLIXOSS_PASSWORD }}
       - name: Publish release
         if: startsWith(github.ref, 'refs/tags/v') && (!contains(github.ref, '-rc.'))
-        run: ./gradlew -x :conductor-contribs:test -Prelease.useLastTag=true final --scan
+        run: ./gradlew -Prelease.useLastTag=true final --scan
         env:
           NETFLIX_OSS_SONATYPE_USERNAME: ${{ secrets.ORG_SONATYPE_USERNAME }}
           NETFLIX_OSS_SONATYPE_PASSWORD: ${{ secrets.ORG_SONATYPE_PASSWORD }}
           NETFLIX_OSS_SIGNING_KEY: ${{ secrets.ORG_SIGNING_KEY }}
           NETFLIX_OSS_SIGNING_PASSWORD: ${{ secrets.ORG_SIGNING_PASSWORD }}
           NETFLIX_OSS_REPO_USERNAME: ${{ secrets.ORG_NETFLIXOSS_USERNAME }}
           NETFLIX_OSS_REPO_PASSWORD: ${{ secrets.ORG_NETFLIXOSS_PASSWORD }}
+      - name: Publish tag to community repo
+        if: startsWith(github.ref, 'refs/tags/v')
+        run: |
+          export TAG=$(git describe --tags --abbrev=0)
+          echo "Current release version is $TAG"
+          echo "Triggering community build"
+          curl \
+            -H "Accept: application/vnd.github.v3+json" \
+            -H "Authorization: Bearer ${{ secrets.COMMUNITY_REPO_TRIGGER }}" \
+            -X POST https://api.github.com/repos/Netflix/conductor-community/dispatches \
+            -d '{"event_type": "publish_build","client_payload": {"tag":"'"$TAG"'"}}'
diff --git a/.github/workflows/update-gradle-wrapper.yml b/.github/workflows/update-gradle-wrapper.yml
@@ -10,9 +10,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - name: Set up JDK 1.8
+      - name: Set up Zulu JDK 11
         uses: actions/setup-java@v3
         with:
-          java-version: 1.8
+          distribution: 'zulu'
+          java-version: '11'
       - name: Update Gradle Wrapper
         uses: gradle-update/update-gradle-wrapper-action@v1
diff --git a/README.md b/README.md
@@ -14,6 +14,13 @@ The latest version is [![Github release](https://img.shields.io/github/v/release
 
 [2.31.8](https://github.com/Netflix/conductor/releases/tag/v2.31.8) is the **final** release of `2.31` branch. As of Feb 2022, `1.x` & `2.x` versions are no longer supported.
 
+## Community Contributions
+The modules contributed by the community are housed at [conductor-community](https://github.com/Netflix/conductor-community). Compatible versions of the community modules are released simultaneously with releases of the main modules.
+
+[Discussion Forum](https://github.com/Netflix/conductor/discussions) Please use the forum for questions and discussing ideas and join the community.
+
+[Access here other Conductor related projects made by the community!](/RELATED.md) - Backup tool, Cron like workflow starter, Docker containers...
+
 ## Getting Started - Building & Running Conductor
 ### Docker
 The easiest way to get started is with Docker containers. Please follow the instructions [here](https://github.com/Netflix/conductor/tree/main/docker). The server and UI can also be built from source separately.
@@ -53,28 +60,27 @@ To run the UI on the bundled development server, run `yarn run start`. Navigate
 ## Published Artifacts
 Binaries are available from [Netflix OSS Maven](https://artifacts.netflix.net/netflixoss/com/netflix/conductor/) repository, or the [Maven Central Repository](https://search.maven.org/search?q=g:com.netflix.conductor).
 
-| Artifact | Description |
-| ----------- | --------------- |
-| conductor-common | Common models used by various conductor modules |
-| conductor-core | Core Conductor module |
-| conductor-redis-persistence | Persistence and queue using Redis/Dynomite |
-| conductor-cassandra-persistence | Persistence using Cassandra |
-| conductor-mysql-persistence | Persistence and queue using MySQL |
-| conductor-postgres-persistence | Persistence and queue using Postgres |
-| conductor-es6-persistence | Indexing using Elasticsearch 6.X |
-| conductor-rest | Spring MVC resources for the core services |
-| conductor-ui | node.js based UI for Conductor |
-| conductor-contribs | Optional contrib package that holds extended workflow tasks and support for SQS, AMQP, etc|
-| conductor-client | Java client for Conductor that includes helpers for running worker tasks |
-| conductor-client-spring | Client starter kit for Spring |
-| conductor-server | Spring Boot Web Application |
-| conductor-azureblob-storage | External payload storage implementation using AzureBlob |
-| conductor-redis-lock | Workflow execution lock implementation using Redis |
-| conductor-zookeeper-lock | Workflow execution lock implementation using Zookeeper |
-| conductor-grpc | Protobuf models used by the server and client |
-| conductor-grpc-client | gRPC server Application |
-| conductor-grpc-server | gRPC client to interact with the gRPC server |
-| conductor-test-harness | Integration and regression tests |
+| Artifact                        | Description                                                                                     |
+|---------------------------------|-------------------------------------------------------------------------------------------------|
+| conductor-common                | Common models used by various conductor modules                                                 |
+| conductor-core                  | Core Conductor module                                                                           |
+| conductor-redis-persistence     | Persistence and queue using Redis/Dynomite                                                      |
+| conductor-cassandra-persistence | Persistence using Cassandra                                                                     |
+| conductor-es6-persistence       | Indexing using Elasticsearch 6.X                                                                |
+| conductor-rest                  | Spring MVC resources for the core services                                                      |
+| conductor-ui                    | node.js based UI for Conductor                                                                  |
+| conductor-client                | Java client for Conductor that includes helpers for running worker tasks                        |
+| conductor-client-spring         | Client starter kit for Spring                                                                   |
+| conductor-server                | Spring Boot Web Application                                                                     |
+| conductor-redis-lock            | Workflow execution lock implementation using Redis                                              |
+| conductor-awss3-storage         | External payload storage implementation using AWS S3                                            |
+| conductor-awssqs-event-queue    | Event queue implementation using AWS SQS                                                        |
+| conductor-http-task             | Workflow system task implementation to send make requests                                       |
+| conductor-json-jq-task          | Workflow system task implementation to evaluate JSON using [jq](https://stedolan.github.io/jq/) |
+| conductor-grpc                  | Protobuf models used by the server and client                                                   |
+| conductor-grpc-client           | gRPC server Application                                                                         |
+| conductor-grpc-server           | gRPC client to interact with the gRPC server                                                    |
+| conductor-test-harness          | Integration and regression tests                                                                |
 
 ## Database Requirements
 
@@ -86,11 +92,6 @@ Binaries are available from [Netflix OSS Maven](https://artifacts.netflix.net/ne
 * JDK 11+
 * UI requires Node 14 to build. Earlier Node versions may work but is untested.
 
-## Community
-[Discussion Forum](https://github.com/Netflix/conductor/discussions) Please use the forum for questions and discussing ideas and join the community.
-
-[Access here other Conductor related projects made by the community!](/RELATED.md) - Backup tool, Cron like workflow starter, Docker containers...
-
 ## Get Support
 Conductor is maintained by Media Workflow Infrastructure team at Netflix.  Use github issue tracking for filing issues and [Discussion Forum](https://github.com/Netflix/conductor/discussions) for any other questions, ideas or support requests. 
 

diff --git a/RELATED.md b/RELATED.md
@@ -1,5 +1,15 @@
 # Community projects related to Conductor
 
+## Client SDKs
+
+Further, all of the (non-Java) SDKs have a new GitHub home: the Conductor SDK repository is your new source for Conductor SDKs:
+
+* [Golang](https://github.com/conductor-sdk/conductor-go)
+* [Python](https://github.com/conductor-sdk/conductor-python)
+* [C#](https://github.com/conductor-sdk/conductor-csharp)
+* [Clojure](https://github.com/conductor-sdk/conductor-clojure)
+
+All contributions on the above client sdks can be made on [Conductor SDK](https://github.com/conductor-sdk) repository.
 
 ## Microservices operations
 
@@ -61,4 +71,4 @@ Docker image published to [Docker Hub](https://hub.docker.com/repository/docker/
 	* We should have a central dedicate service that can do scheduling for us and provide a trigger to a microservices at expected time.
 * It offers an additional module `io.github.jas34.scheduledwf.config.ScheduledWfServerModule` built on the existing core 
 of conductor and does not require deployment of any additional service.
-For more details refer: [Schedule Conductor Workflows](https://jas34.github.io/scheduledwf) and [Capability In Conductor To Schedule Workflows](https://github.com/Netflix/conductor/discussions/2256)
+For more details refer: [Schedule Conductor Workflows](https://jas34.github.io/scheduledwf) and [Capability In Conductor To Schedule Workflows](https://github.com/Netflix/conductor/discussions/2256)
diff --git a/annotations-processor/build.gradle b/annotations-processor/build.gradle
@@ -8,7 +8,7 @@ dependencies {
     api 'com.google.guava:guava:31.1-jre'
     api 'com.squareup:javapoet:1.13.+'
     api 'com.github.jknack:handlebars:4.3.+'
-    api 'com.google.protobuf:protobuf-java:3.20.0'
+    api 'com.google.protobuf:protobuf-java:3.20.1'
     api 'javax.annotation:javax.annotation-api:1.3.2'
     api gradleApi()
 

diff --git a/annotations-processor/dependencies.lock b/annotations-processor/dependencies.lock
@@ -12,7 +12,7 @@
             "locked": "31.1-jre"
         },
         "com.google.protobuf:protobuf-java": {
-            "locked": "3.20.0"
+            "locked": "3.20.1"
         },
         "com.netflix.conductor:conductor-annotations": {
             "project": true
@@ -87,7 +87,7 @@
             "locked": "31.1-jre"
         },
         "com.google.protobuf:protobuf-java": {
-            "locked": "3.20.0"
+            "locked": "3.20.1"
         },
         "com.netflix.conductor:conductor-annotations": {
             "project": true
@@ -137,7 +137,7 @@
             "locked": "31.1-jre"
         },
         "com.google.protobuf:protobuf-java": {
-            "locked": "3.20.0"
+            "locked": "3.20.1"
         },
         "com.netflix.conductor:conductor-annotations": {
             "project": true
@@ -181,7 +181,7 @@
             "locked": "31.1-jre"
         },
         "com.google.protobuf:protobuf-java": {
-            "locked": "3.20.0"
+            "locked": "3.20.1"
         },
         "com.netflix.conductor:conductor-annotations": {
             "project": true

diff --git a/awss3-storage/README.md b/awss3-storage/README.md
diff --git a/mysql-persistence/build.gradle → awss3-storage/build.gradle b/mysql-persistence/build.gradle → awss3-storage/build.gradle
@@ -1,5 +1,5 @@
 /*
- *  Copyright 2021 Netflix, Inc.
+ *  Copyright 2022 Netflix, Inc.
  *  <p>
  *  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
  *  the License. You may obtain a copy of the License at
@@ -16,25 +16,6 @@ dependencies {
     implementation project(':conductor-core')
     compileOnly 'org.springframework.boot:spring-boot-starter'
 
-    // SBMTODO: remove guava dep
-    implementation "com.google.guava:guava:${revGuava}"
-
-    implementation "com.fasterxml.jackson.core:jackson-databind"
-    implementation "com.fasterxml.jackson.core:jackson-core"
-
+    implementation "com.amazonaws:aws-java-sdk-s3:${revAwsSdk}"
     implementation "org.apache.commons:commons-lang3"
-
-    implementation "mysql:mysql-connector-java"
-    implementation "org.springframework.boot:spring-boot-starter-jdbc"
-    implementation "org.flywaydb:flyway-core"
-
-    testImplementation "org.testcontainers:mysql:${revTestContainer}"
-
-    testImplementation project(':conductor-core').sourceSets.test.output
-    testImplementation project(':conductor-common').sourceSets.test.output
-}
-
-test {
-    //the MySQL unit tests must run within the same JVM to share the same embedded DB
-    maxParallelForks = 1
 }