Elasticsearch 7

dependencies.gradle

@@ -61,4 +61,5 @@ ext {
     revKafka = '2.6.0'
     revMicrometer = '1.6.2'
     revPrometheus = '0.9.0'
+    revElasticSearch7 = '7.17.13'
 }

es7-persistence/README.md

@@ -0,0 +1,95 @@
+# ES7 Persistence
+
+This module provides ES7 persistence when indexing workflows and tasks.
+
+### ES Breaking changes
+
+From ES6 to ES7 there were significant breaking changes which affected ES7-persistence module implementation.
+* Mapping type deprecation
+* Templates API
+* TransportClient deprecation
+
+More information can be found here: https://www.elastic.co/guide/en/elasticsearch/reference/current/breaking-changes-7.0.html
+
+
+## Build
+
+1. In order to use the ES7, you must change the following files from ES6 to ES7:
+
+https://github.com/Netflix/conductor/blob/main/build.gradle
+https://github.com/Netflix/conductor/blob/main/server/src/main/resources/application.properties
+
+In file:
+
+- /build.gradle
+
+change ext['elasticsearch.version'] from revElasticSearch6 to revElasticSearch7
+
+
+In file:
+
+- /server/src/main/resources/application.properties
+
+change conductor.elasticsearch.version from 6 to 7
+
+Also you need to recreate dependencies.lock files with ES7 dependencies. To do that delete all dependencies.lock files and then run: 
+
+```
+./gradlew generateLock updateLock saveLock
+```
+
+
+2. To use the ES7 for all modules include test-harness, you must change also the following files:
+
+https://github.com/Netflix/conductor/blob/main/test-harness/build.gradle
+https://github.com/Netflix/conductor/blob/main/test-harness/src/test/java/com/netflix/conductor/test/integration/AbstractEndToEndTest.java
+
+In file:
+
+- /test-harness/build.gradle
+
+* change module inclusion from 'es6-persistence' to 'es7-persistence'
+
+In file:
+
+- /test-harness/src/test/java/com/netflix/conductor/test/integration/AbstractEndToEndTest.java
+
+* change conductor.elasticsearch.version from 6 to 7
+* change DockerImageName.parse("docker.elastic.co/elasticsearch/elasticsearch-oss").withTag("6.8.12") to DockerImageName.parse("docker.elastic.co/elasticsearch/elasticsearch-oss").withTag("7.6.2")
+
+
+
+### Configuration
+(Default values shown below)
+
+This module uses the following configuration options:
+```properties
+# A comma separated list of schema/host/port of the ES nodes to communicate with.
+# Schema can be `http` or `https`. If schema is ignored then `http` transport will be used;
+# Since ES deprecated TransportClient, conductor will use only the  REST transport protocol.
+conductor.elasticsearch.url=
+
+#The name of the workflow and task index. 
+conductor.elasticsearch.indexPrefix=conductor
+
+#Worker Queue size used in executor service for async methods in IndexDao.
+conductor.elasticsearch.asyncWorkerQueueSize=100
+
+#Maximum thread pool size in executor service for async methods in IndexDao
+conductor.elasticsearch.asyncMaxPoolSize=12
+
+#Timeout (in seconds) for the in-memory to be flushed if not explicitly indexed
+conductor.elasticsearch.asyncBufferFlushTimeout=10
+```
+
+
+### BASIC Authentication
+If you need to pass user/password to connect to ES, add the following properties to your config file
+* conductor.elasticsearch.username
+* conductor.elasticsearch.password
+
+Example
+```
+conductor.elasticsearch.username=someusername
+conductor.elasticsearch.password=somepassword
+```

es7-persistence/build.gradle

@@ -0,0 +1,54 @@
+plugins {
+    id 'com.github.johnrengelman.shadow' version '7.0.0'
+    id 'java'
+}
+
+configurations {
+    // Prevent shaded dependencies from being published, while keeping them available to tests
+    shadow.extendsFrom compileOnly
+    testRuntime.extendsFrom compileOnly
+}
+
+ext['elasticsearch.version'] = revElasticSearch7
+
+dependencies {
+
+    implementation project(':conductor-common')
+    implementation project(':conductor-core')
+    implementation project(':conductor-common-persistence')
+
+    compileOnly 'org.springframework.boot:spring-boot-starter'
+    compileOnly 'org.springframework.retry:spring-retry'
+
+    implementation "commons-io:commons-io:${revCommonsIo}"
+    implementation "org.apache.commons:commons-lang3"
+    implementation "com.google.guava:guava:${revGuava}"
+
+    implementation "com.fasterxml.jackson.core:jackson-databind"
+    implementation "com.fasterxml.jackson.core:jackson-core"
+
+    implementation "org.elasticsearch.client:elasticsearch-rest-client:${revElasticSearch7}"
+    implementation "org.elasticsearch.client:elasticsearch-rest-high-level-client:${revElasticSearch7}"
+
+    testImplementation "net.java.dev.jna:jna:5.7.0"
+    testImplementation "org.awaitility:awaitility:${revAwaitility}"
+    testImplementation "org.testcontainers:elasticsearch:${revTestContainer}"
+    testImplementation project(':conductor-test-util').sourceSets.test.output
+    testImplementation 'org.springframework.retry:spring-retry'
+
+}
+
+// Drop the classifier and delete jar task actions to replace the regular jar artifact with the shadow artifact
+shadowJar {
+    configurations = [project.configurations.shadow]
+    classifier = null
+
+    // Service files are not included by default.
+    mergeServiceFiles {
+        include 'META-INF/services/*'
+        include 'META-INF/maven/*'
+    }
+}
+
+jar.enabled = false
+jar.dependsOn shadowJar