Test fixes (#35)

* Update the tests

java-sdk/src/test/resources/application-integrationtest.properties

@@ -14,6 +14,8 @@
 #
 
 conductor.db.type=memory
+# disable trying to connect to redis and use in-memory
+conductor.queue.type=xxx
 conductor.workflow-execution-lock.type=local_only
 conductor.external-payload-storage.type=mock
 conductor.indexing.enabled=false

kafka/build.gradle

@@ -26,5 +26,6 @@ dependencies {
 
     testImplementation project(':conductor-test-util')
     testImplementation project(':conductor-test-util').sourceSets.test.output
+    testImplementation "redis.clients:jedis:${revJedis}"
 
 }

kafka/src/test/resources/application-integrationtest.properties

@@ -14,6 +14,8 @@
 #
 
 conductor.db.type=memory
+# disable trying to connect to redis and use in-memory
+conductor.queue.type=xxx
 conductor.workflow-execution-lock.type=local_only
 conductor.external-payload-storage.type=dummy
 conductor.indexing.enabled=false

mysql-persistence/build.gradle

@@ -33,7 +33,7 @@ dependencies {
 
     testImplementation project(':conductor-test-util').sourceSets.test.output
     testImplementation project(':conductor-common-persistence').sourceSets.test.output
-
+    testImplementation "redis.clients:jedis:${revJedis}"
 }
 
 test {

postgres-persistence/build.gradle

@@ -29,6 +29,7 @@ dependencies {
 
     testImplementation project(':conductor-test-util').sourceSets.test.output
     testImplementation project(':conductor-common-persistence').sourceSets.test.output
+    testImplementation "redis.clients:jedis:${revJedis}"
 
 }
 

test-harness/build.gradle

@@ -7,7 +7,7 @@ dependencies {
     testImplementation project(':conductor-core')
     testImplementation project(':conductor-redis-persistence')
     testImplementation project(':conductor-cassandra-persistence')
-    testImplementation project(':conductor-es6-persistence')
+    testImplementation project(':conductor-es7-persistence')
     testImplementation project(':conductor-grpc-server')
     testImplementation project(':conductor-client')
     testImplementation project(':conductor-grpc-client')
@@ -32,8 +32,8 @@ dependencies {
     testImplementation "org.spockframework:spock-core:${revSpock}"
     testImplementation "org.spockframework:spock-spring:${revSpock}"
 
-    testImplementation "org.elasticsearch.client:elasticsearch-rest-client:6.8.23"
-    testImplementation "org.elasticsearch.client:elasticsearch-rest-high-level-client:6.8.23"
+    testImplementation "org.elasticsearch.client:elasticsearch-rest-client:${revElasticSearch7}"
+    testImplementation "org.elasticsearch.client:elasticsearch-rest-high-level-client:${revElasticSearch7}"
 
     testImplementation "org.testcontainers:elasticsearch:${revTestContainer}"
     testImplementation('junit:junit:4.13.2')

test-harness/src/test/groovy/com/netflix/conductor/test/base/AbstractResiliencySpecification.groovy

@@ -14,14 +14,42 @@ package com.netflix.conductor.test.base
 
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty
+import org.springframework.boot.test.context.SpringBootTest
 import org.springframework.context.annotation.Bean
+import org.springframework.context.annotation.ComponentScan
 import org.springframework.context.annotation.Configuration
+import org.springframework.context.annotation.FilterType
 import org.springframework.context.annotation.Primary
 import org.springframework.test.context.TestPropertySource
 
+import com.netflix.conductor.ConductorTestApp
+import com.netflix.conductor.core.config.SchedulerConfiguration
+import com.netflix.conductor.core.events.DefaultEventProcessor
+import com.netflix.conductor.core.events.DefaultEventQueueManager
+import com.netflix.conductor.core.events.queue.ConductorEventQueueProvider
+import com.netflix.conductor.core.execution.mapper.DoWhileTaskMapper
+import com.netflix.conductor.core.execution.mapper.EventTaskMapper
+import com.netflix.conductor.core.execution.mapper.ForkJoinDynamicTaskMapper
+import com.netflix.conductor.core.execution.mapper.ForkJoinTaskMapper
+import com.netflix.conductor.core.execution.mapper.HumanTaskMapper
+import com.netflix.conductor.core.execution.mapper.JoinTaskMapper
+import com.netflix.conductor.core.execution.mapper.SimpleTaskMapper
+import com.netflix.conductor.core.execution.mapper.SubWorkflowTaskMapper
+import com.netflix.conductor.core.execution.mapper.SwitchTaskMapper
+import com.netflix.conductor.core.execution.mapper.WaitTaskMapper
+import com.netflix.conductor.core.execution.tasks.DoWhile
+import com.netflix.conductor.core.execution.tasks.Event
+import com.netflix.conductor.core.execution.tasks.ExclusiveJoin
+import com.netflix.conductor.core.execution.tasks.Human
+import com.netflix.conductor.core.execution.tasks.Inline
+import com.netflix.conductor.core.execution.tasks.Join
+import com.netflix.conductor.core.execution.tasks.SetVariable
+import com.netflix.conductor.core.execution.tasks.SubWorkflow
+import com.netflix.conductor.core.execution.tasks.Wait
 import com.netflix.conductor.dao.QueueDAO
 import com.netflix.conductor.redis.dao.DynoQueueDAO
 import com.netflix.conductor.redis.jedis.JedisMock
+import com.netflix.conductor.tasks.json.JsonJqTransform
 import com.netflix.dyno.connectionpool.Host
 import com.netflix.dyno.queues.ShardSupplier
 import com.netflix.dyno.queues.redis.RedisQueues
@@ -33,7 +61,8 @@ import spock.mock.DetachedMockFactory
         "conductor.system-task-workers.enabled=false",
         "conductor.workflow-repair-service.enabled=true",
         "conductor.workflow-reconciler.enabled=false",
-        "conductor.integ-test.queue-spy.enabled=true"
+        "conductor.integ-test.queue-spy.enabled=true",
+        "conductor.queue.type=xxx"
 ])
 abstract class AbstractResiliencySpecification extends AbstractSpecification {
 

test-harness/src/test/groovy/com/netflix/conductor/test/base/AbstractSpecification.groovy

@@ -16,6 +16,7 @@ import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.boot.test.context.SpringBootTest
 import org.springframework.test.context.TestPropertySource
 
+import com.netflix.conductor.ConductorTestApp
 import com.netflix.conductor.core.execution.AsyncSystemTaskExecutor
 import com.netflix.conductor.core.execution.StartWorkflowInput
 import com.netflix.conductor.core.execution.WorkflowExecutor
@@ -27,8 +28,11 @@ import com.netflix.conductor.test.util.WorkflowTestUtil
 
 import spock.lang.Specification
 
-@SpringBootTest
-@TestPropertySource(locations = "classpath:application-integrationtest.properties")
+@SpringBootTest(classes = ConductorTestApp.class)
+@TestPropertySource(locations = "classpath:application-integrationtest.properties",properties = [
+        "conductor.db.type=memory",
+        "conductor.queue.type=xxx"
+])
 abstract class AbstractSpecification extends Specification {
 
     @Autowired

test-harness/src/test/groovy/com/netflix/conductor/test/integration/WorkflowAndTaskConfigurationSpec.groovy

@@ -13,7 +13,10 @@
 package com.netflix.conductor.test.integration
 
 import org.springframework.beans.factory.annotation.Autowired
+import org.springframework.boot.test.context.SpringBootTest
+import org.springframework.test.context.TestPropertySource
 
+import com.netflix.conductor.ConductorTestApp
 import com.netflix.conductor.common.metadata.tasks.Task
 import com.netflix.conductor.common.metadata.tasks.TaskDef
 import com.netflix.conductor.common.metadata.tasks.TaskResult
@@ -31,6 +34,10 @@ import spock.lang.Shared
 
 import static com.netflix.conductor.test.util.WorkflowTestUtil.verifyPolledAndAcknowledgedTask
 
+@TestPropertySource(properties = [
+        "conductor.db.type=memory",
+        "conductor.queue.type=xxx"
+])
 class WorkflowAndTaskConfigurationSpec extends AbstractSpecification {
 
     @Autowired

test-harness/src/test/java/com/netflix/conductor/test/integration/AbstractEndToEndTest.java

@@ -48,7 +48,11 @@
 import static org.junit.Assert.assertNotNull;
 
 @TestPropertySource(
-        properties = {"conductor.indexing.enabled=true", "conductor.elasticsearch.version=6"})
+        properties = {
+            "conductor.indexing.enabled=true",
+            "conductor.elasticsearch.version=7",
+            "conductor.queue.type=xxx"
+        })
 public abstract class AbstractEndToEndTest {
 
     private static final Logger log = LoggerFactory.getLogger(AbstractEndToEndTest.class);
@@ -62,7 +66,7 @@ public abstract class AbstractEndToEndTest {
     private static final ElasticsearchContainer container =
             new ElasticsearchContainer(
                     DockerImageName.parse("docker.elastic.co/elasticsearch/elasticsearch-oss")
-                            .withTag("6.8.17")); // this should match the client version
+                            .withTag("7.10.2")); // this should match the client version
 
     private static RestClient restClient;
 

test-harness/src/test/java/com/netflix/conductor/test/integration/grpc/AbstractGrpcEndToEndTest.java

@@ -22,6 +22,7 @@
 import org.springframework.test.context.TestPropertySource;
 import org.springframework.test.context.junit4.SpringRunner;
 
+import com.netflix.conductor.ConductorTestApp;
 import com.netflix.conductor.client.grpc.EventClient;
 import com.netflix.conductor.client.grpc.MetadataClient;
 import com.netflix.conductor.client.grpc.TaskClient;
@@ -48,6 +49,7 @@
 
 @RunWith(SpringRunner.class)
 @SpringBootTest(
+        classes = ConductorTestApp.class,
         properties = {"conductor.grpc-server.enabled=true", "conductor.grpc-server.port=8092"})
 @TestPropertySource(locations = "classpath:application-integrationtest.properties")
 public abstract class AbstractGrpcEndToEndTest extends AbstractEndToEndTest {
@@ -156,7 +158,7 @@ public void testAll() throws Exception {
         assertNotNull(polled);
         assertEquals(0, polled.size());
 
-        polled = taskClient.batchPollTasksByTaskType(t0.getName(), "test", 1, 100);
+        polled = taskClient.batchPollTasksByTaskType(t0.getName(), "test", 1, 1000);
         assertNotNull(polled);
         assertEquals(1, polled.size());
         assertEquals(t0.getName(), polled.get(0).getTaskDefName());

test-harness/src/test/java/com/netflix/conductor/test/integration/http/AbstractHttpEndToEndTest.java

@@ -26,6 +26,7 @@
 import org.springframework.test.context.TestPropertySource;
 import org.springframework.test.context.junit4.SpringRunner;
 
+import com.netflix.conductor.ConductorTestApp;
 import com.netflix.conductor.client.exception.ConductorClientException;
 import com.netflix.conductor.client.http.EventClient;
 import com.netflix.conductor.client.http.MetadataClient;
@@ -55,7 +56,7 @@
 import static org.junit.Assert.fail;
 
 @RunWith(SpringRunner.class)
-@SpringBootTest(webEnvironment = WebEnvironment.RANDOM_PORT)
+@SpringBootTest(webEnvironment = WebEnvironment.RANDOM_PORT, classes = ConductorTestApp.class)
 @TestPropertySource(locations = "classpath:application-integrationtest.properties")
 public abstract class AbstractHttpEndToEndTest extends AbstractEndToEndTest {
 

test-harness/src/test/resources/application-integrationtest.properties

@@ -14,6 +14,8 @@
 #
 
 conductor.db.type=memory
+# disable trying to connect to redis and use in-memory
+conductor.queue.type=xxx
 conductor.workflow-execution-lock.type=local_only
 conductor.external-payload-storage.type=mock
 conductor.indexing.enabled=false

test-util/build.gradle

@@ -6,7 +6,7 @@ dependencies {
 
     implementation project(':conductor-common')
     implementation project(':conductor-core')
-    implementation project(':conductor-server')
+    compileOnly project(':conductor-server')
     implementation project(':conductor-client')
     implementation project(':conductor-rest')
     implementation project(':conductor-grpc-server')
@@ -18,6 +18,7 @@ dependencies {
     implementation "com.fasterxml.jackson.core:jackson-core"
     implementation "org.apache.commons:commons-lang3"
     implementation 'org.springframework.boot:spring-boot-starter-validation'
+    implementation 'org.springframework.retry:spring-retry'
 
     implementation "com.fasterxml.jackson.core:jackson-databind"
     implementation "com.fasterxml.jackson.core:jackson-core"
@@ -46,6 +47,7 @@ dependencies {
 
     //In memory
     implementation "org.rarefiedredis.redis:redis-java:${revRarefiedRedis}"
+
 }
 
 test {

test-util/src/test/groovy/com/netflix/conductor/test/base/AbstractSpecification.groovy

@@ -12,6 +12,7 @@
  */
 package com.netflix.conductor.test.base
 
+import com.netflix.conductor.ConductorTestApp
 import com.netflix.conductor.service.WorkflowService
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.boot.test.context.SpringBootTest
@@ -26,7 +27,7 @@ import com.netflix.conductor.test.util.WorkflowTestUtil
 
 import spock.lang.Specification
 
-@SpringBootTest
+@SpringBootTest(classes = ConductorTestApp.class)
 @TestPropertySource(locations = "classpath:application-integrationtest.properties")
 abstract class AbstractSpecification extends Specification {
 

test-util/src/test/java/com/netflix/conductor/test/integration/grpc/AbstractGrpcEndToEndTest.java

@@ -20,6 +20,7 @@
 import org.springframework.test.context.TestPropertySource;
 import org.springframework.test.context.junit4.SpringRunner;
 
+import com.netflix.conductor.ConductorTestApp;
 import com.netflix.conductor.client.grpc.EventClient;
 import com.netflix.conductor.client.grpc.MetadataClient;
 import com.netflix.conductor.client.grpc.TaskClient;
@@ -46,6 +47,7 @@
 
 @RunWith(SpringRunner.class)
 @SpringBootTest(
+        classes = ConductorTestApp.class,
         properties = {"conductor.grpc-server.enabled=true", "conductor.grpc-server.port=8092"})
 @TestPropertySource(locations = "classpath:application-integrationtest.properties")
 public abstract class AbstractGrpcEndToEndTest extends AbstractEndToEndTest {

test-util/src/test/resources/application-integrationtest.properties

@@ -1,4 +1,6 @@
 conductor.db.type=memory
+# disable trying to connect to redis and use in-memory
+conductor.queue.type=xxx
 conductor.workflow-execution-lock.type=local_only
 conductor.external-payload-storage.type=dummy
 conductor.indexing.enabled=true

workflow-event-listener/build.gradle

@@ -22,4 +22,6 @@ dependencies {
 
     //In memory
     implementation "org.rarefiedredis.redis:redis-java:${revRarefiedRedis}"
+    testImplementation "redis.clients:jedis:${revJedis}"
+
 }

workflow-event-listener/src/test/java/com/netflix/conductor/test/listener/WorkflowStatusPublisherIntegrationTest.java

@@ -29,6 +29,7 @@
 import org.springframework.test.context.TestPropertySource;
 import org.springframework.test.context.junit4.SpringRunner;
 
+import com.netflix.conductor.ConductorTestApp;
 import com.netflix.conductor.common.metadata.tasks.Task;
 import com.netflix.conductor.common.metadata.tasks.TaskDef;
 import com.netflix.conductor.common.metadata.tasks.TaskResult;
@@ -51,6 +52,7 @@
 
 @RunWith(SpringRunner.class)
 @SpringBootTest(
+        classes = ConductorTestApp.class,
         properties = {
             "conductor.db.type=memory",
             "conductor.workflow-status-listener.type=queue_publisher",

workflow-event-listener/src/test/resources/application-integrationtest.properties

@@ -14,6 +14,8 @@
 #
 
 conductor.db.type=memory
+# disable trying to connect to redis and use in-memory
+conductor.queue.type=xxx
 conductor.workflow-execution-lock.type=local_only
 conductor.external-payload-storage.type=dummy
 conductor.indexing.enabled=false