From 8c1d5f328cb72875554ef82c80b726bbf964906d Mon Sep 17 00:00:00 2001 From: c4lm Date: Fri, 5 Jan 2024 01:31:39 +0400 Subject: [PATCH 1/6] Update publishing flow --- .github/workflows/publish.yml | 88 +++++++++++++---------------------- deploy.gradle | 69 +++++++++++++++++++++++++++ 2 files changed, 102 insertions(+), 55 deletions(-) create mode 100644 deploy.gradle diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 3bf0960ae..dadfe1934 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,66 +1,44 @@ -name: Publish to NetflixOSS and Maven Central +name: Publish OSS Conductor + + on: release: types: - released - prereleased -permissions: - contents: read - jobs: - publish: + # This workflow contains a single job called "build" + build: + # The type of runner that the job will run on runs-on: ubuntu-latest - name: Gradle Build and Publish + + # Steps represent a sequence of tasks that will be executed as part of the job steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v3 - - name: Set up Zulu JDK 17 - uses: actions/setup-java@v3 - with: - distribution: 'zulu' - java-version: '17' - - name: Cache Gradle packages - uses: actions/cache@v3 - with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: | - ${{ runner.os }}-gradle- - - name: Publish candidate - if: startsWith(github.ref, 'refs/tags/v') && contains(github.ref, '-rc.') - run: ./gradlew -Prelease.useLastTag=true candidate --scan - env: - NETFLIX_OSS_SONATYPE_USERNAME: ${{ secrets.ORG_SONATYPE_USERNAME }} - NETFLIX_OSS_SONATYPE_PASSWORD: ${{ secrets.ORG_SONATYPE_PASSWORD }} - NETFLIX_OSS_SIGNING_KEY: ${{ secrets.ORG_SIGNING_KEY }} - NETFLIX_OSS_SIGNING_PASSWORD: ${{ secrets.ORG_SIGNING_PASSWORD }} - NETFLIX_OSS_REPO_USERNAME: ${{ secrets.ORG_NETFLIXOSS_USERNAME }} - NETFLIX_OSS_REPO_PASSWORD: ${{ secrets.ORG_NETFLIXOSS_PASSWORD }} - - name: Publish release - if: startsWith(github.ref, 'refs/tags/v') && (!contains(github.ref, '-rc.')) - run: ./gradlew -Prelease.useLastTag=true final --scan - env: - NETFLIX_OSS_SONATYPE_USERNAME: ${{ secrets.ORG_SONATYPE_USERNAME }} - NETFLIX_OSS_SONATYPE_PASSWORD: ${{ secrets.ORG_SONATYPE_PASSWORD }} - NETFLIX_OSS_SIGNING_KEY: ${{ secrets.ORG_SIGNING_KEY }} - NETFLIX_OSS_SIGNING_PASSWORD: ${{ secrets.ORG_SIGNING_PASSWORD }} - NETFLIX_OSS_REPO_USERNAME: ${{ secrets.ORG_NETFLIXOSS_USERNAME }} - NETFLIX_OSS_REPO_PASSWORD: ${{ secrets.ORG_NETFLIXOSS_PASSWORD }} - - name: Publish tag to community repo - if: startsWith(github.ref, 'refs/tags/v') + + # Checkout Conductor version tag and publish an initial release + - name: Build and Deploy Conductor Main run: | - export TAG=$(git describe --tags --abbrev=0) - echo "Current release version is $TAG" - echo "Triggering community build" - curl \ - -H "Accept: application/vnd.github.v3+json" \ - -H "Authorization: Bearer ${{ secrets.COMMUNITY_REPO_TRIGGER }}" \ - -X POST https://api.github.com/repos/Netflix/conductor-community/dispatches \ - -d '{"event_type": "publish_build","client_payload": {"tag":"'"$TAG"'"}}' - - name: Publish Test Report - uses: mikepenz/action-junit-report@v3 - if: always() # always run even if the previous step fails - with: - report_paths: '**/build/test-results/test/TEST-*.xml' + export TAG=$(git tag) + export CONDUCTOR_VERSION=`echo $TAG | cut -f2 -d"v"` + echo "CONDUCTOR_VERSION is $CONDUCTOR_VERSION" + echo "Tag to checkout and publish $TAG" + git clone -b $TAG https://github.com/conductor-oss/conductor.git + cd conductor + git checkout tags/$TAG + git describe --tags --abbrev=0 + ls -ltr + echo "Updating build.gradle - and printing its content" + cat ../deploy.gradle + cp ../deploy.gradle . + echo "apply from: "\"\$rootDir/deploy.gradle\""" >> build.gradle + cat build.gradle + cat deploy.gradle + ./gradlew publish -PmavenCentral -Pusername=${{ secrets.SONATYPE_USERNAME }} -Ppassword=${{ secrets.SONATYPE_PASSWORD }} + echo "All done" + env: + ORG_GRADLE_PROJECT_signingKeyId: ${{ secrets.SIGNING_KEY_ID }} + ORG_GRADLE_PROJECT_signingKey: ${{ secrets.SIGNING_KEY }} + ORG_GRADLE_PROJECT_signingPassword: ${{ secrets.SIGNING_PASSWORD }} diff --git a/deploy.gradle b/deploy.gradle new file mode 100644 index 000000000..a5fbe1a6e --- /dev/null +++ b/deploy.gradle @@ -0,0 +1,69 @@ + +allprojects { + + apply plugin: 'maven-publish' + apply plugin: 'java-library' + apply plugin: 'signing' + + group = 'org.conductoross' + def conductorVersion = System.getenv('CONDUCTOR_VERSION') + if (conductorVersion) { + println "Inferred version from env variable 'CONDUCTOR_VERSION': $conductorVersion" + version = conductorVersion + } + + publishing { + publications { + mavenJava(MavenPublication) { + from components.java + pom { + name = 'Conductor OSS' + description = 'Conductor OSS build.' + url = 'https://github.com/conductor-oss/conductor' + scm { + connection = 'scm:git:git://github.com/conductor-oss/conductor.git' + developerConnection = 'scm:git:ssh://github.com/conductor-oss/conductor.git' + url = 'https://github.com/conductor-oss/conductor' + } + licenses { + license { + name = 'The Apache License, Version 2.0' + url = 'http://www.apache.org/licenses/LICENSE-2.0.txt' + } + } + developers { + developer { + organization = 'Conductor OSS' + organizationUrl = 'https://conductor-oss.org/' + name = 'Conductor OSS' + } + } + } + } + } + + repositories { + maven { + println "Publishing to Sonatype Repository" + url = "https://s01.oss.sonatype.org/${project.version.endsWith('-SNAPSHOT') ? "content/repositories/snapshots/" : "service/local/staging/deploy/maven2/"}" + credentials { + username project.properties.username + password project.properties.password + } + } + } + } + + signing { + def signingKeyId = findProperty('signingKeyId') + def signingKey = findProperty('signingKey') + def signingPassword = findProperty('signingPassword') + System.out.println("signingKeyId: " + signingKeyId) + if (signingKeyId && signingKey && signingPassword) { + useInMemoryPgpKeys(signingKeyId, signingKey, signingPassword) + } + + sign publishing.publications + } + +} \ No newline at end of file From 3faeec6a0311c78ebc0f0cf94caf5c357631de82 Mon Sep 17 00:00:00 2001 From: Viren Baraiya Date: Thu, 4 Jan 2024 14:51:52 -0800 Subject: [PATCH 2/6] Update README.md --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 99e200624..eac404a6c 100644 --- a/README.md +++ b/README.md @@ -91,3 +91,9 @@ docker pull conductoross/conductor:3.15.0 ## Get Support There are several ways to get in touch with us: * [Slack Community](https://join.slack.com/t/orkes-conductor/shared_invite/zt-xyxqyseb-YZ3hwwAgHJH97bsrYRnSZg) + +## Contributors + + + + From a8f1260105f59d3603e056f999392bc020f07145 Mon Sep 17 00:00:00 2001 From: Viren Baraiya Date: Fri, 5 Jan 2024 14:13:30 -0800 Subject: [PATCH 3/6] deployment preparation --- .github/dependabot.yml | 6 +-- .github/pull_request_template.md | 2 +- .github/workflows/generate_gh_pages.yml | 4 +- .github/workflows/publish.yml | 56 ++++++++++++------------- .github/workflows/stale.yml | 28 ------------- annotations-processor/build.gradle | 4 +- build.gradle | 31 +++----------- dependencies.gradle | 2 +- dependencies.lock | 10 ----- deploy.gradle | 33 ++++++++------- es6-persistence/build.gradle | 2 +- 11 files changed, 60 insertions(+), 118 deletions(-) delete mode 100644 .github/workflows/stale.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml index ef87e0b82..fa854fcdc 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,9 +5,9 @@ updates: schedule: interval: "weekly" reviewers: - - "aravindanr" - - "jxu-nflx" - - "apanicker-nflx" + - "v1r3n" + - "boney9" + - "c4lm" - package-ecosystem: "github-actions" directory: "/" schedule: diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 4cd3eb440..8367fbc9a 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -3,7 +3,7 @@ Pull Request type - [ ] Bugfix - [ ] Feature - [ ] Refactoring (no functional changes, no api changes) -- [ ] Build related changes (Please run `./gradlew generateLock saveLock` to refresh dependencies) +- [ ] Build related changes - [ ] WHOSUSING.md - [ ] Other (please describe): diff --git a/.github/workflows/generate_gh_pages.yml b/.github/workflows/generate_gh_pages.yml index 93bffdfa4..8c429e1b8 100644 --- a/.github/workflows/generate_gh_pages.yml +++ b/.github/workflows/generate_gh_pages.yml @@ -1,8 +1,6 @@ name: Publish docs via GitHub Pages on: - push: - branches: - - main + workflow_dispatch jobs: build: diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index dadfe1934..51b514ffa 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,44 +1,40 @@ -name: Publish OSS Conductor - - +name: Publish Conductor OSS toMaven Central on: release: types: - released - prereleased +permissions: + contents: read + jobs: - # This workflow contains a single job called "build" - build: - # The type of runner that the job will run on + publish: runs-on: ubuntu-latest - - # Steps represent a sequence of tasks that will be executed as part of the job + name: Gradle Build and Publish steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v3 - - # Checkout Conductor version tag and publish an initial release - - name: Build and Deploy Conductor Main + - name: Set up Zulu JDK 17 + uses: actions/setup-java@v3 + with: + distribution: 'zulu' + java-version: '17' + - name: Cache Gradle packages + uses: actions/cache@v3 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} + restore-keys: | + ${{ runner.os }}-gradle- + - name: Publish release run: | - export TAG=$(git tag) - export CONDUCTOR_VERSION=`echo $TAG | cut -f2 -d"v"` - echo "CONDUCTOR_VERSION is $CONDUCTOR_VERSION" - echo "Tag to checkout and publish $TAG" - git clone -b $TAG https://github.com/conductor-oss/conductor.git - cd conductor - git checkout tags/$TAG - git describe --tags --abbrev=0 - ls -ltr - echo "Updating build.gradle - and printing its content" - cat ../deploy.gradle - cp ../deploy.gradle . - echo "apply from: "\"\$rootDir/deploy.gradle\""" >> build.gradle - cat build.gradle - cat deploy.gradle - ./gradlew publish -PmavenCentral -Pusername=${{ secrets.SONATYPE_USERNAME }} -Ppassword=${{ secrets.SONATYPE_PASSWORD }} - echo "All done" + export VERSION="${{github.ref_name}}" + export PUBLISH_VERSION=`echo ${VERSION:1}` + echo Publishing version $PUBLISH_VERSION + ./gradlew publish -Pversion=$PUBLISH_VERSION -Pusername=${{ secrets.SONATYPE_USERNAME }} -Ppassword=${{ secrets.SONATYPE_PASSWORD }} env: ORG_GRADLE_PROJECT_signingKeyId: ${{ secrets.SIGNING_KEY_ID }} ORG_GRADLE_PROJECT_signingKey: ${{ secrets.SIGNING_KEY }} - ORG_GRADLE_PROJECT_signingPassword: ${{ secrets.SIGNING_PASSWORD }} + ORG_GRADLE_PROJECT_signingPassword: ${{ secrets.SIGNING_PASSWORD }} \ No newline at end of file diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml deleted file mode 100644 index a90156f3b..000000000 --- a/.github/workflows/stale.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Close stale issues and pull requests - -on: - schedule: - - cron: "0 0 * * *" - -permissions: - contents: read - -jobs: - stale: - permissions: - issues: write # for actions/stale to close stale issues - pull-requests: write # for actions/stale to close stale PRs - runs-on: ubuntu-latest - steps: - - uses: actions/stale@v6 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - stale-issue-message: 'This issue is stale, because it has been open for 45 days with no activity. Remove the stale label or comment, or this will be closed in 7 days.' - close-issue-message: 'This issue was closed, because it has been stalled for 7 days with no activity.' - stale-pr-message: 'This PR is stale, because it has been open for 45 days with no activity. Remove the stale label or comment, or this will be closed in 7 days.' - close-pr-message: 'This PR was closed, because it has been stalled for 7 days with no activity.' - days-before-issue-stale: 45 - days-before-issue-close: 7 - days-before-pr-stale: 45 - days-before-pr-close: 7 - exempt-issue-labels: 'type: bug,enhancement,work_in_progress,help_wanted' diff --git a/annotations-processor/build.gradle b/annotations-processor/build.gradle index 5e0c28d3b..18844ab09 100644 --- a/annotations-processor/build.gradle +++ b/annotations-processor/build.gradle @@ -6,8 +6,8 @@ sourceSets { dependencies { implementation project(':conductor-annotations') api 'com.google.guava:guava:31.1-jre' - api 'com.squareup:javapoet:1.13.+' - api 'com.github.jknack:handlebars:4.3.+' + api 'com.squareup:javapoet:1.13.0' + api 'com.github.jknack:handlebars:4.3.1' api 'com.google.protobuf:protobuf-java:3.21.12' api 'jakarta.annotation:jakarta.annotation-api:2.1.1' api gradleApi() diff --git a/build.gradle b/build.gradle index 758dea44d..98fed4641 100644 --- a/build.gradle +++ b/build.gradle @@ -17,23 +17,12 @@ plugins { id 'io.spring.dependency-management' version '1.0.13.RELEASE' id 'java' id 'application' - id 'jacoco' - id 'nebula.netflixoss' version '10.6.0' + id 'maven-publish' + id 'signing' + id 'java-library' + id "com.diffplug.spotless" version "5.0.0" } -/* - * Copyright 2023 Conductor authors - *

- * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - // Establish version and status ext.githubProjectName = rootProject.name // Change if github project name is not the same as the root project's name @@ -43,9 +32,9 @@ subprojects { apply from: "$rootDir/dependencies.gradle" apply from: "$rootDir/springboot-bom-overrides.gradle" +apply from: "$rootDir/deploy.gradle" allprojects { - apply plugin: 'nebula.netflixoss' apply plugin: 'io.spring.dependency-management' apply plugin: 'java-library' apply plugin: 'project-report' @@ -53,7 +42,7 @@ allprojects { sourceCompatibility = JavaVersion.VERSION_17 targetCompatibility = JavaVersion.VERSION_17 - group = 'com.netflix.conductor' + group = 'org.conductoross' configurations.all { exclude group: 'ch.qos.logback', module: 'logback-classic' @@ -111,14 +100,6 @@ allprojects { } } -jacocoTestReport { - reports { - html.required = true - xml.required = true - csv.required = false - } -} - task server { dependsOn ':conductor-server:bootRun' } diff --git a/dependencies.gradle b/dependencies.gradle index 30625c0be..ed1c08fa1 100644 --- a/dependencies.gradle +++ b/dependencies.gradle @@ -31,7 +31,7 @@ ext { revGrpc = '1.57.2' revGuava = '30.0-jre' revHamcrestAllMatchers = '1.8' - revHealth = '1.1.+' + revHealth = '1.1.4' revProtoBuf = '3.21.12' revJakartaAnnotation = '2.1.1' revJAXB = '4.0.1' diff --git a/dependencies.lock b/dependencies.lock index fd0cb733b..545a15902 100644 --- a/dependencies.lock +++ b/dependencies.lock @@ -21,16 +21,6 @@ "locked": "2.20.0" } }, - "jacocoAgent": { - "org.jacoco:org.jacoco.agent": { - "locked": "0.8.8" - } - }, - "jacocoAnt": { - "org.jacoco:org.jacoco.ant": { - "locked": "0.8.8" - } - }, "runtimeClasspath": { "org.apache.logging.log4j:log4j-api": { "locked": "2.20.0" diff --git a/deploy.gradle b/deploy.gradle index a5fbe1a6e..20eec69ca 100644 --- a/deploy.gradle +++ b/deploy.gradle @@ -1,21 +1,24 @@ -allprojects { +subprojects { apply plugin: 'maven-publish' apply plugin: 'java-library' apply plugin: 'signing' group = 'org.conductoross' - def conductorVersion = System.getenv('CONDUCTOR_VERSION') - if (conductorVersion) { - println "Inferred version from env variable 'CONDUCTOR_VERSION': $conductorVersion" - version = conductorVersion - } publishing { publications { mavenJava(MavenPublication) { from components.java + versionMapping { + usage('java-api') { + fromResolutionOf('runtimeClasspath') + } + usage('java-runtime') { + fromResolutionResult() + } + } pom { name = 'Conductor OSS' description = 'Conductor OSS build.' @@ -44,8 +47,7 @@ allprojects { repositories { maven { - println "Publishing to Sonatype Repository" - url = "https://s01.oss.sonatype.org/${project.version.endsWith('-SNAPSHOT') ? "content/repositories/snapshots/" : "service/local/staging/deploy/maven2/"}" + url = "https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/" credentials { username project.properties.username password project.properties.password @@ -56,14 +58,17 @@ allprojects { signing { def signingKeyId = findProperty('signingKeyId') - def signingKey = findProperty('signingKey') - def signingPassword = findProperty('signingPassword') - System.out.println("signingKeyId: " + signingKeyId) - if (signingKeyId && signingKey && signingPassword) { - useInMemoryPgpKeys(signingKeyId, signingKey, signingPassword) + if (signingKeyId) { + println 'Signing artifacts with keys' + def signingKey = findProperty('signingKey') + def signingPassword = findProperty('signingPassword') + if (signingKeyId && signingKey && signingPassword) { + useInMemoryPgpKeys(signingKeyId, signingKey, signingPassword) + } + + sign publishing.publications } - sign publishing.publications } } \ No newline at end of file diff --git a/es6-persistence/build.gradle b/es6-persistence/build.gradle index c4a16081a..0db11f22a 100644 --- a/es6-persistence/build.gradle +++ b/es6-persistence/build.gradle @@ -37,6 +37,6 @@ dependencies { switch (org.gradle.internal.os.OperatingSystem.current()) { case org.gradle.internal.os.OperatingSystem.MAC_OS: - //tasks.forEach(task -> task.onlyIf { project.hasProperty('ES6Test') }) + tasks.forEach(task -> task.onlyIf { project.hasProperty('ES6Test') }) break; } From 64073f080f834dc580583367d6205c78b50f6088 Mon Sep 17 00:00:00 2001 From: Viren Baraiya Date: Fri, 5 Jan 2024 22:00:08 -0800 Subject: [PATCH 4/6] Test fixes (#35) * Update the tests --- .github/workflows/update-gradle-wrapper.yml | 19 ------------ .../application-integrationtest.properties | 2 ++ kafka/build.gradle | 1 + .../application-integrationtest.properties | 2 ++ mysql-persistence/build.gradle | 2 +- postgres-persistence/build.gradle | 1 + test-harness/build.gradle | 6 ++-- .../AbstractResiliencySpecification.groovy | 31 ++++++++++++++++++- .../test/base/AbstractSpecification.groovy | 8 +++-- .../WorkflowAndTaskConfigurationSpec.groovy | 7 +++++ .../integration/AbstractEndToEndTest.java | 8 +++-- .../grpc/AbstractGrpcEndToEndTest.java | 4 ++- .../http/AbstractHttpEndToEndTest.java | 3 +- .../application-integrationtest.properties | 2 ++ test-util/build.gradle | 4 ++- .../test/base/AbstractSpecification.groovy | 3 +- .../grpc/AbstractGrpcEndToEndTest.java | 2 ++ .../application-integrationtest.properties | 2 ++ workflow-event-listener/build.gradle | 2 ++ ...orkflowStatusPublisherIntegrationTest.java | 2 ++ .../application-integrationtest.properties | 2 ++ 21 files changed, 81 insertions(+), 32 deletions(-) delete mode 100644 .github/workflows/update-gradle-wrapper.yml diff --git a/.github/workflows/update-gradle-wrapper.yml b/.github/workflows/update-gradle-wrapper.yml deleted file mode 100644 index 831e6d5aa..000000000 --- a/.github/workflows/update-gradle-wrapper.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: Update Gradle Wrapper - -on: - schedule: - - cron: "0 0 * * *" - workflow_dispatch: - -jobs: - update-gradle-wrapper: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Set up Zulu JDK 17 - uses: actions/setup-java@v3 - with: - distribution: 'zulu' - java-version: '17' - - name: Update Gradle Wrapper - uses: gradle-update/update-gradle-wrapper-action@v1 diff --git a/java-sdk/src/test/resources/application-integrationtest.properties b/java-sdk/src/test/resources/application-integrationtest.properties index 6025e57a9..efb46697d 100644 --- a/java-sdk/src/test/resources/application-integrationtest.properties +++ b/java-sdk/src/test/resources/application-integrationtest.properties @@ -14,6 +14,8 @@ # conductor.db.type=memory +# disable trying to connect to redis and use in-memory +conductor.queue.type=xxx conductor.workflow-execution-lock.type=local_only conductor.external-payload-storage.type=mock conductor.indexing.enabled=false diff --git a/kafka/build.gradle b/kafka/build.gradle index 13339d752..847a9ba5d 100644 --- a/kafka/build.gradle +++ b/kafka/build.gradle @@ -26,5 +26,6 @@ dependencies { testImplementation project(':conductor-test-util') testImplementation project(':conductor-test-util').sourceSets.test.output + testImplementation "redis.clients:jedis:${revJedis}" } diff --git a/kafka/src/test/resources/application-integrationtest.properties b/kafka/src/test/resources/application-integrationtest.properties index 19e39ec47..a6432a1af 100644 --- a/kafka/src/test/resources/application-integrationtest.properties +++ b/kafka/src/test/resources/application-integrationtest.properties @@ -14,6 +14,8 @@ # conductor.db.type=memory +# disable trying to connect to redis and use in-memory +conductor.queue.type=xxx conductor.workflow-execution-lock.type=local_only conductor.external-payload-storage.type=dummy conductor.indexing.enabled=false diff --git a/mysql-persistence/build.gradle b/mysql-persistence/build.gradle index dccdfc32f..267dbee4f 100644 --- a/mysql-persistence/build.gradle +++ b/mysql-persistence/build.gradle @@ -33,7 +33,7 @@ dependencies { testImplementation project(':conductor-test-util').sourceSets.test.output testImplementation project(':conductor-common-persistence').sourceSets.test.output - + testImplementation "redis.clients:jedis:${revJedis}" } test { diff --git a/postgres-persistence/build.gradle b/postgres-persistence/build.gradle index 66155a5d6..acc925b51 100644 --- a/postgres-persistence/build.gradle +++ b/postgres-persistence/build.gradle @@ -29,6 +29,7 @@ dependencies { testImplementation project(':conductor-test-util').sourceSets.test.output testImplementation project(':conductor-common-persistence').sourceSets.test.output + testImplementation "redis.clients:jedis:${revJedis}" } diff --git a/test-harness/build.gradle b/test-harness/build.gradle index ea410db62..c250eddf6 100644 --- a/test-harness/build.gradle +++ b/test-harness/build.gradle @@ -7,7 +7,7 @@ dependencies { testImplementation project(':conductor-core') testImplementation project(':conductor-redis-persistence') testImplementation project(':conductor-cassandra-persistence') - testImplementation project(':conductor-es6-persistence') + testImplementation project(':conductor-es7-persistence') testImplementation project(':conductor-grpc-server') testImplementation project(':conductor-client') testImplementation project(':conductor-grpc-client') @@ -32,8 +32,8 @@ dependencies { testImplementation "org.spockframework:spock-core:${revSpock}" testImplementation "org.spockframework:spock-spring:${revSpock}" - testImplementation "org.elasticsearch.client:elasticsearch-rest-client:6.8.23" - testImplementation "org.elasticsearch.client:elasticsearch-rest-high-level-client:6.8.23" + testImplementation "org.elasticsearch.client:elasticsearch-rest-client:${revElasticSearch7}" + testImplementation "org.elasticsearch.client:elasticsearch-rest-high-level-client:${revElasticSearch7}" testImplementation "org.testcontainers:elasticsearch:${revTestContainer}" testImplementation('junit:junit:4.13.2') diff --git a/test-harness/src/test/groovy/com/netflix/conductor/test/base/AbstractResiliencySpecification.groovy b/test-harness/src/test/groovy/com/netflix/conductor/test/base/AbstractResiliencySpecification.groovy index 0ad2a7f87..de0385f75 100644 --- a/test-harness/src/test/groovy/com/netflix/conductor/test/base/AbstractResiliencySpecification.groovy +++ b/test-harness/src/test/groovy/com/netflix/conductor/test/base/AbstractResiliencySpecification.groovy @@ -14,14 +14,42 @@ package com.netflix.conductor.test.base import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty +import org.springframework.boot.test.context.SpringBootTest import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.ComponentScan import org.springframework.context.annotation.Configuration +import org.springframework.context.annotation.FilterType import org.springframework.context.annotation.Primary import org.springframework.test.context.TestPropertySource +import com.netflix.conductor.ConductorTestApp +import com.netflix.conductor.core.config.SchedulerConfiguration +import com.netflix.conductor.core.events.DefaultEventProcessor +import com.netflix.conductor.core.events.DefaultEventQueueManager +import com.netflix.conductor.core.events.queue.ConductorEventQueueProvider +import com.netflix.conductor.core.execution.mapper.DoWhileTaskMapper +import com.netflix.conductor.core.execution.mapper.EventTaskMapper +import com.netflix.conductor.core.execution.mapper.ForkJoinDynamicTaskMapper +import com.netflix.conductor.core.execution.mapper.ForkJoinTaskMapper +import com.netflix.conductor.core.execution.mapper.HumanTaskMapper +import com.netflix.conductor.core.execution.mapper.JoinTaskMapper +import com.netflix.conductor.core.execution.mapper.SimpleTaskMapper +import com.netflix.conductor.core.execution.mapper.SubWorkflowTaskMapper +import com.netflix.conductor.core.execution.mapper.SwitchTaskMapper +import com.netflix.conductor.core.execution.mapper.WaitTaskMapper +import com.netflix.conductor.core.execution.tasks.DoWhile +import com.netflix.conductor.core.execution.tasks.Event +import com.netflix.conductor.core.execution.tasks.ExclusiveJoin +import com.netflix.conductor.core.execution.tasks.Human +import com.netflix.conductor.core.execution.tasks.Inline +import com.netflix.conductor.core.execution.tasks.Join +import com.netflix.conductor.core.execution.tasks.SetVariable +import com.netflix.conductor.core.execution.tasks.SubWorkflow +import com.netflix.conductor.core.execution.tasks.Wait import com.netflix.conductor.dao.QueueDAO import com.netflix.conductor.redis.dao.DynoQueueDAO import com.netflix.conductor.redis.jedis.JedisMock +import com.netflix.conductor.tasks.json.JsonJqTransform import com.netflix.dyno.connectionpool.Host import com.netflix.dyno.queues.ShardSupplier import com.netflix.dyno.queues.redis.RedisQueues @@ -33,7 +61,8 @@ import spock.mock.DetachedMockFactory "conductor.system-task-workers.enabled=false", "conductor.workflow-repair-service.enabled=true", "conductor.workflow-reconciler.enabled=false", - "conductor.integ-test.queue-spy.enabled=true" + "conductor.integ-test.queue-spy.enabled=true", + "conductor.queue.type=xxx" ]) abstract class AbstractResiliencySpecification extends AbstractSpecification { diff --git a/test-harness/src/test/groovy/com/netflix/conductor/test/base/AbstractSpecification.groovy b/test-harness/src/test/groovy/com/netflix/conductor/test/base/AbstractSpecification.groovy index 974f30001..9aef149d9 100644 --- a/test-harness/src/test/groovy/com/netflix/conductor/test/base/AbstractSpecification.groovy +++ b/test-harness/src/test/groovy/com/netflix/conductor/test/base/AbstractSpecification.groovy @@ -16,6 +16,7 @@ import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.context.SpringBootTest import org.springframework.test.context.TestPropertySource +import com.netflix.conductor.ConductorTestApp import com.netflix.conductor.core.execution.AsyncSystemTaskExecutor import com.netflix.conductor.core.execution.StartWorkflowInput import com.netflix.conductor.core.execution.WorkflowExecutor @@ -27,8 +28,11 @@ import com.netflix.conductor.test.util.WorkflowTestUtil import spock.lang.Specification -@SpringBootTest -@TestPropertySource(locations = "classpath:application-integrationtest.properties") +@SpringBootTest(classes = ConductorTestApp.class) +@TestPropertySource(locations = "classpath:application-integrationtest.properties",properties = [ + "conductor.db.type=memory", + "conductor.queue.type=xxx" +]) abstract class AbstractSpecification extends Specification { @Autowired diff --git a/test-harness/src/test/groovy/com/netflix/conductor/test/integration/WorkflowAndTaskConfigurationSpec.groovy b/test-harness/src/test/groovy/com/netflix/conductor/test/integration/WorkflowAndTaskConfigurationSpec.groovy index d594fbfeb..fd27be739 100644 --- a/test-harness/src/test/groovy/com/netflix/conductor/test/integration/WorkflowAndTaskConfigurationSpec.groovy +++ b/test-harness/src/test/groovy/com/netflix/conductor/test/integration/WorkflowAndTaskConfigurationSpec.groovy @@ -13,7 +13,10 @@ package com.netflix.conductor.test.integration import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.test.context.TestPropertySource +import com.netflix.conductor.ConductorTestApp import com.netflix.conductor.common.metadata.tasks.Task import com.netflix.conductor.common.metadata.tasks.TaskDef import com.netflix.conductor.common.metadata.tasks.TaskResult @@ -31,6 +34,10 @@ import spock.lang.Shared import static com.netflix.conductor.test.util.WorkflowTestUtil.verifyPolledAndAcknowledgedTask +@TestPropertySource(properties = [ + "conductor.db.type=memory", + "conductor.queue.type=xxx" +]) class WorkflowAndTaskConfigurationSpec extends AbstractSpecification { @Autowired diff --git a/test-harness/src/test/java/com/netflix/conductor/test/integration/AbstractEndToEndTest.java b/test-harness/src/test/java/com/netflix/conductor/test/integration/AbstractEndToEndTest.java index c110e6730..66b883022 100644 --- a/test-harness/src/test/java/com/netflix/conductor/test/integration/AbstractEndToEndTest.java +++ b/test-harness/src/test/java/com/netflix/conductor/test/integration/AbstractEndToEndTest.java @@ -48,7 +48,11 @@ import static org.junit.Assert.assertNotNull; @TestPropertySource( - properties = {"conductor.indexing.enabled=true", "conductor.elasticsearch.version=6"}) + properties = { + "conductor.indexing.enabled=true", + "conductor.elasticsearch.version=7", + "conductor.queue.type=xxx" + }) public abstract class AbstractEndToEndTest { private static final Logger log = LoggerFactory.getLogger(AbstractEndToEndTest.class); @@ -62,7 +66,7 @@ public abstract class AbstractEndToEndTest { private static final ElasticsearchContainer container = new ElasticsearchContainer( DockerImageName.parse("docker.elastic.co/elasticsearch/elasticsearch-oss") - .withTag("6.8.17")); // this should match the client version + .withTag("7.10.2")); // this should match the client version private static RestClient restClient; diff --git a/test-harness/src/test/java/com/netflix/conductor/test/integration/grpc/AbstractGrpcEndToEndTest.java b/test-harness/src/test/java/com/netflix/conductor/test/integration/grpc/AbstractGrpcEndToEndTest.java index 96ea7783f..0c4de90a6 100644 --- a/test-harness/src/test/java/com/netflix/conductor/test/integration/grpc/AbstractGrpcEndToEndTest.java +++ b/test-harness/src/test/java/com/netflix/conductor/test/integration/grpc/AbstractGrpcEndToEndTest.java @@ -22,6 +22,7 @@ import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit4.SpringRunner; +import com.netflix.conductor.ConductorTestApp; import com.netflix.conductor.client.grpc.EventClient; import com.netflix.conductor.client.grpc.MetadataClient; import com.netflix.conductor.client.grpc.TaskClient; @@ -48,6 +49,7 @@ @RunWith(SpringRunner.class) @SpringBootTest( + classes = ConductorTestApp.class, properties = {"conductor.grpc-server.enabled=true", "conductor.grpc-server.port=8092"}) @TestPropertySource(locations = "classpath:application-integrationtest.properties") public abstract class AbstractGrpcEndToEndTest extends AbstractEndToEndTest { @@ -156,7 +158,7 @@ public void testAll() throws Exception { assertNotNull(polled); assertEquals(0, polled.size()); - polled = taskClient.batchPollTasksByTaskType(t0.getName(), "test", 1, 100); + polled = taskClient.batchPollTasksByTaskType(t0.getName(), "test", 1, 1000); assertNotNull(polled); assertEquals(1, polled.size()); assertEquals(t0.getName(), polled.get(0).getTaskDefName()); diff --git a/test-harness/src/test/java/com/netflix/conductor/test/integration/http/AbstractHttpEndToEndTest.java b/test-harness/src/test/java/com/netflix/conductor/test/integration/http/AbstractHttpEndToEndTest.java index b57066c01..279e38c5f 100644 --- a/test-harness/src/test/java/com/netflix/conductor/test/integration/http/AbstractHttpEndToEndTest.java +++ b/test-harness/src/test/java/com/netflix/conductor/test/integration/http/AbstractHttpEndToEndTest.java @@ -26,6 +26,7 @@ import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit4.SpringRunner; +import com.netflix.conductor.ConductorTestApp; import com.netflix.conductor.client.exception.ConductorClientException; import com.netflix.conductor.client.http.EventClient; import com.netflix.conductor.client.http.MetadataClient; @@ -55,7 +56,7 @@ import static org.junit.Assert.fail; @RunWith(SpringRunner.class) -@SpringBootTest(webEnvironment = WebEnvironment.RANDOM_PORT) +@SpringBootTest(webEnvironment = WebEnvironment.RANDOM_PORT, classes = ConductorTestApp.class) @TestPropertySource(locations = "classpath:application-integrationtest.properties") public abstract class AbstractHttpEndToEndTest extends AbstractEndToEndTest { diff --git a/test-harness/src/test/resources/application-integrationtest.properties b/test-harness/src/test/resources/application-integrationtest.properties index af1517070..b209c8054 100644 --- a/test-harness/src/test/resources/application-integrationtest.properties +++ b/test-harness/src/test/resources/application-integrationtest.properties @@ -14,6 +14,8 @@ # conductor.db.type=memory +# disable trying to connect to redis and use in-memory +conductor.queue.type=xxx conductor.workflow-execution-lock.type=local_only conductor.external-payload-storage.type=mock conductor.indexing.enabled=false diff --git a/test-util/build.gradle b/test-util/build.gradle index b9fe3754b..b3280ca51 100644 --- a/test-util/build.gradle +++ b/test-util/build.gradle @@ -6,7 +6,7 @@ dependencies { implementation project(':conductor-common') implementation project(':conductor-core') - implementation project(':conductor-server') + compileOnly project(':conductor-server') implementation project(':conductor-client') implementation project(':conductor-rest') implementation project(':conductor-grpc-server') @@ -18,6 +18,7 @@ dependencies { implementation "com.fasterxml.jackson.core:jackson-core" implementation "org.apache.commons:commons-lang3" implementation 'org.springframework.boot:spring-boot-starter-validation' + implementation 'org.springframework.retry:spring-retry' implementation "com.fasterxml.jackson.core:jackson-databind" implementation "com.fasterxml.jackson.core:jackson-core" @@ -46,6 +47,7 @@ dependencies { //In memory implementation "org.rarefiedredis.redis:redis-java:${revRarefiedRedis}" + } test { diff --git a/test-util/src/test/groovy/com/netflix/conductor/test/base/AbstractSpecification.groovy b/test-util/src/test/groovy/com/netflix/conductor/test/base/AbstractSpecification.groovy index 2fde6d4f6..2f59d9d60 100644 --- a/test-util/src/test/groovy/com/netflix/conductor/test/base/AbstractSpecification.groovy +++ b/test-util/src/test/groovy/com/netflix/conductor/test/base/AbstractSpecification.groovy @@ -12,6 +12,7 @@ */ package com.netflix.conductor.test.base +import com.netflix.conductor.ConductorTestApp import com.netflix.conductor.service.WorkflowService import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.context.SpringBootTest @@ -26,7 +27,7 @@ import com.netflix.conductor.test.util.WorkflowTestUtil import spock.lang.Specification -@SpringBootTest +@SpringBootTest(classes = ConductorTestApp.class) @TestPropertySource(locations = "classpath:application-integrationtest.properties") abstract class AbstractSpecification extends Specification { diff --git a/test-util/src/test/java/com/netflix/conductor/test/integration/grpc/AbstractGrpcEndToEndTest.java b/test-util/src/test/java/com/netflix/conductor/test/integration/grpc/AbstractGrpcEndToEndTest.java index 5f53592bb..5d43ef67d 100644 --- a/test-util/src/test/java/com/netflix/conductor/test/integration/grpc/AbstractGrpcEndToEndTest.java +++ b/test-util/src/test/java/com/netflix/conductor/test/integration/grpc/AbstractGrpcEndToEndTest.java @@ -20,6 +20,7 @@ import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit4.SpringRunner; +import com.netflix.conductor.ConductorTestApp; import com.netflix.conductor.client.grpc.EventClient; import com.netflix.conductor.client.grpc.MetadataClient; import com.netflix.conductor.client.grpc.TaskClient; @@ -46,6 +47,7 @@ @RunWith(SpringRunner.class) @SpringBootTest( + classes = ConductorTestApp.class, properties = {"conductor.grpc-server.enabled=true", "conductor.grpc-server.port=8092"}) @TestPropertySource(locations = "classpath:application-integrationtest.properties") public abstract class AbstractGrpcEndToEndTest extends AbstractEndToEndTest { diff --git a/test-util/src/test/resources/application-integrationtest.properties b/test-util/src/test/resources/application-integrationtest.properties index 3b9e396ae..2b0e07151 100644 --- a/test-util/src/test/resources/application-integrationtest.properties +++ b/test-util/src/test/resources/application-integrationtest.properties @@ -1,4 +1,6 @@ conductor.db.type=memory +# disable trying to connect to redis and use in-memory +conductor.queue.type=xxx conductor.workflow-execution-lock.type=local_only conductor.external-payload-storage.type=dummy conductor.indexing.enabled=true diff --git a/workflow-event-listener/build.gradle b/workflow-event-listener/build.gradle index b30e145ee..bf322be9a 100644 --- a/workflow-event-listener/build.gradle +++ b/workflow-event-listener/build.gradle @@ -22,4 +22,6 @@ dependencies { //In memory implementation "org.rarefiedredis.redis:redis-java:${revRarefiedRedis}" + testImplementation "redis.clients:jedis:${revJedis}" + } \ No newline at end of file diff --git a/workflow-event-listener/src/test/java/com/netflix/conductor/test/listener/WorkflowStatusPublisherIntegrationTest.java b/workflow-event-listener/src/test/java/com/netflix/conductor/test/listener/WorkflowStatusPublisherIntegrationTest.java index 3aa8b072b..33d909a84 100644 --- a/workflow-event-listener/src/test/java/com/netflix/conductor/test/listener/WorkflowStatusPublisherIntegrationTest.java +++ b/workflow-event-listener/src/test/java/com/netflix/conductor/test/listener/WorkflowStatusPublisherIntegrationTest.java @@ -29,6 +29,7 @@ import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit4.SpringRunner; +import com.netflix.conductor.ConductorTestApp; import com.netflix.conductor.common.metadata.tasks.Task; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskResult; @@ -51,6 +52,7 @@ @RunWith(SpringRunner.class) @SpringBootTest( + classes = ConductorTestApp.class, properties = { "conductor.db.type=memory", "conductor.workflow-status-listener.type=queue_publisher", diff --git a/workflow-event-listener/src/test/resources/application-integrationtest.properties b/workflow-event-listener/src/test/resources/application-integrationtest.properties index 19e39ec47..a6432a1af 100644 --- a/workflow-event-listener/src/test/resources/application-integrationtest.properties +++ b/workflow-event-listener/src/test/resources/application-integrationtest.properties @@ -14,6 +14,8 @@ # conductor.db.type=memory +# disable trying to connect to redis and use in-memory +conductor.queue.type=xxx conductor.workflow-execution-lock.type=local_only conductor.external-payload-storage.type=dummy conductor.indexing.enabled=false From f42a2798480436a7929f8b8d9791959fa8ff58f2 Mon Sep 17 00:00:00 2001 From: Viren Baraiya Date: Sat, 6 Jan 2024 11:06:19 -0800 Subject: [PATCH 5/6] update deployment properties --- deploy.gradle | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/deploy.gradle b/deploy.gradle index 20eec69ca..ebd0d240c 100644 --- a/deploy.gradle +++ b/deploy.gradle @@ -7,6 +7,11 @@ subprojects { group = 'org.conductoross' + java { + withSourcesJar() + withJavadocJar() + } + publishing { publications { mavenJava(MavenPublication) { @@ -59,13 +64,11 @@ subprojects { signing { def signingKeyId = findProperty('signingKeyId') if (signingKeyId) { - println 'Signing artifacts with keys' def signingKey = findProperty('signingKey') def signingPassword = findProperty('signingPassword') if (signingKeyId && signingKey && signingPassword) { useInMemoryPgpKeys(signingKeyId, signingKey, signingPassword) } - sign publishing.publications } From 2e309b2a0b3780937f9c56a02f81d0153504eb7d Mon Sep 17 00:00:00 2001 From: Hari R <37411837+haricane8133@users.noreply.github.com> Date: Tue, 9 Jan 2024 10:21:53 +0530 Subject: [PATCH 6/6] Fixes #7, another dormant bug (#9) * Fixes #7, another dormant bug * Review comment - Combine Regexes --- ui/src/components/NavLink.jsx | 3 ++- ui/src/plugins/fetch.js | 7 ++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/ui/src/components/NavLink.jsx b/ui/src/components/NavLink.jsx index 3fd7b4623..5763dc115 100644 --- a/ui/src/components/NavLink.jsx +++ b/ui/src/components/NavLink.jsx @@ -26,7 +26,8 @@ export default React.forwardRef((props, ref) => { ); } else { - const href = absolutePath ? url.toString() : cleanDuplicateSlash(getBasename() + url.toString()); + // Note: + '/' + is required here + const href = absolutePath ? url.toString() : cleanDuplicateSlash(getBasename() + '/' + url.toString()); return ( {rest.children} diff --git a/ui/src/plugins/fetch.js b/ui/src/plugins/fetch.js index 4467339ce..16761e771 100644 --- a/ui/src/plugins/fetch.js +++ b/ui/src/plugins/fetch.js @@ -41,6 +41,11 @@ export function fetchWithContext( }); } +/** + * @param {string} path + * @returns path with '/' not duplicated, except at :// + * + */ export function cleanDuplicateSlash(path) { - return path.replace(/([^:]\/)\/+/g, "$1"); + return path.replace(/(:\/\/)\/*|(\/)+/g, "$1$2"); }