From 84da27665a7e947ff00166d4a06b17a0e20f66d7 Mon Sep 17 00:00:00 2001 From: marko-bekhta Date: Thu, 1 Dec 2022 17:48:53 +0100 Subject: [PATCH 01/20] HSEARCH-4674 Use test containers for Elasticsearch/OpenSearch --- .../elasticsearch/env-7.10.properties | 5 - .../elasticsearch/env-7.11.properties | 5 - .../elasticsearch/env-7.12.properties | 5 - .../elasticsearch/env-7.13.properties | 5 - .../elasticsearch/env-7.14.properties | 5 - .../elasticsearch/env-7.15.properties | 5 - .../elasticsearch/env-7.16.properties | 7 - .../elasticsearch/env-7.17.properties | 7 - .../elasticsearch/env-8.0.properties | 8 - .../elasticsearch/env-8.1.properties | 8 - .../elasticsearch/env-8.10.properties | 10 - .../elasticsearch/env-8.2.properties | 8 - .../elasticsearch/env-8.3.properties | 8 - .../elasticsearch/env-8.4.properties | 8 - .../elasticsearch/env-8.5.properties | 8 - .../elasticsearch/env-8.6.properties | 8 - .../elasticsearch/env-8.7.properties | 8 - .../elasticsearch/env-8.8.properties | 10 - .../elasticsearch/env-8.9.properties | 10 - build/parents/build/pom.xml | 11 ++ build/parents/integrationtest/pom.xml | 177 ------------------ build/parents/springtest/pom.xml | 7 + ...searchTestHostConnectionConfiguration.java | 9 +- .../elasticsearch/SearchBackendContainer.java | 131 +++++++++++++ util/internal/integrationtest/common/pom.xml | 5 + 25 files changed, 162 insertions(+), 316 deletions(-) delete mode 100644 build/container/elasticsearch/env-7.10.properties delete mode 100644 build/container/elasticsearch/env-7.11.properties delete mode 100644 build/container/elasticsearch/env-7.12.properties delete mode 100644 build/container/elasticsearch/env-7.13.properties delete mode 100644 build/container/elasticsearch/env-7.14.properties delete mode 100644 build/container/elasticsearch/env-7.15.properties delete mode 100644 build/container/elasticsearch/env-7.16.properties delete mode 100644 build/container/elasticsearch/env-7.17.properties delete mode 100644 build/container/elasticsearch/env-8.0.properties delete mode 100644 build/container/elasticsearch/env-8.1.properties delete mode 100644 build/container/elasticsearch/env-8.10.properties delete mode 100644 build/container/elasticsearch/env-8.2.properties delete mode 100644 build/container/elasticsearch/env-8.3.properties delete mode 100644 build/container/elasticsearch/env-8.4.properties delete mode 100644 build/container/elasticsearch/env-8.5.properties delete mode 100644 build/container/elasticsearch/env-8.6.properties delete mode 100644 build/container/elasticsearch/env-8.7.properties delete mode 100644 build/container/elasticsearch/env-8.8.properties delete mode 100644 build/container/elasticsearch/env-8.9.properties create mode 100644 util/internal/integrationtest/backend/elasticsearch/src/main/java/org/hibernate/search/util/impl/integrationtest/backend/elasticsearch/SearchBackendContainer.java diff --git a/build/container/elasticsearch/env-7.10.properties b/build/container/elasticsearch/env-7.10.properties deleted file mode 100644 index 2543c13954f..00000000000 --- a/build/container/elasticsearch/env-7.10.properties +++ /dev/null @@ -1,5 +0,0 @@ -# Disable some features that are not needed in our tests and just slow down startup -indices.lifecycle.history_index_enabled=false -stack.templates.enabled=false -xpack.ml.enabled=false -xpack.watcher.enabled=false \ No newline at end of file diff --git a/build/container/elasticsearch/env-7.11.properties b/build/container/elasticsearch/env-7.11.properties deleted file mode 100644 index 2543c13954f..00000000000 --- a/build/container/elasticsearch/env-7.11.properties +++ /dev/null @@ -1,5 +0,0 @@ -# Disable some features that are not needed in our tests and just slow down startup -indices.lifecycle.history_index_enabled=false -stack.templates.enabled=false -xpack.ml.enabled=false -xpack.watcher.enabled=false \ No newline at end of file diff --git a/build/container/elasticsearch/env-7.12.properties b/build/container/elasticsearch/env-7.12.properties deleted file mode 100644 index 2543c13954f..00000000000 --- a/build/container/elasticsearch/env-7.12.properties +++ /dev/null @@ -1,5 +0,0 @@ -# Disable some features that are not needed in our tests and just slow down startup -indices.lifecycle.history_index_enabled=false -stack.templates.enabled=false -xpack.ml.enabled=false -xpack.watcher.enabled=false \ No newline at end of file diff --git a/build/container/elasticsearch/env-7.13.properties b/build/container/elasticsearch/env-7.13.properties deleted file mode 100644 index 2543c13954f..00000000000 --- a/build/container/elasticsearch/env-7.13.properties +++ /dev/null @@ -1,5 +0,0 @@ -# Disable some features that are not needed in our tests and just slow down startup -indices.lifecycle.history_index_enabled=false -stack.templates.enabled=false -xpack.ml.enabled=false -xpack.watcher.enabled=false \ No newline at end of file diff --git a/build/container/elasticsearch/env-7.14.properties b/build/container/elasticsearch/env-7.14.properties deleted file mode 100644 index 2543c13954f..00000000000 --- a/build/container/elasticsearch/env-7.14.properties +++ /dev/null @@ -1,5 +0,0 @@ -# Disable some features that are not needed in our tests and just slow down startup -indices.lifecycle.history_index_enabled=false -stack.templates.enabled=false -xpack.ml.enabled=false -xpack.watcher.enabled=false \ No newline at end of file diff --git a/build/container/elasticsearch/env-7.15.properties b/build/container/elasticsearch/env-7.15.properties deleted file mode 100644 index 2543c13954f..00000000000 --- a/build/container/elasticsearch/env-7.15.properties +++ /dev/null @@ -1,5 +0,0 @@ -# Disable some features that are not needed in our tests and just slow down startup -indices.lifecycle.history_index_enabled=false -stack.templates.enabled=false -xpack.ml.enabled=false -xpack.watcher.enabled=false \ No newline at end of file diff --git a/build/container/elasticsearch/env-7.16.properties b/build/container/elasticsearch/env-7.16.properties deleted file mode 100644 index 66a62a9e8d8..00000000000 --- a/build/container/elasticsearch/env-7.16.properties +++ /dev/null @@ -1,7 +0,0 @@ -# Disable some features that are not needed in our tests and just slow down startup -cluster.deprecation_indexing.enabled=false -indices.lifecycle.history_index_enabled=false -slm.history_index_enabled=false -stack.templates.enabled=false -xpack.ml.enabled=false -xpack.watcher.enabled=false \ No newline at end of file diff --git a/build/container/elasticsearch/env-7.17.properties b/build/container/elasticsearch/env-7.17.properties deleted file mode 100644 index 66a62a9e8d8..00000000000 --- a/build/container/elasticsearch/env-7.17.properties +++ /dev/null @@ -1,7 +0,0 @@ -# Disable some features that are not needed in our tests and just slow down startup -cluster.deprecation_indexing.enabled=false -indices.lifecycle.history_index_enabled=false -slm.history_index_enabled=false -stack.templates.enabled=false -xpack.ml.enabled=false -xpack.watcher.enabled=false \ No newline at end of file diff --git a/build/container/elasticsearch/env-8.0.properties b/build/container/elasticsearch/env-8.0.properties deleted file mode 100644 index d89332da80b..00000000000 --- a/build/container/elasticsearch/env-8.0.properties +++ /dev/null @@ -1,8 +0,0 @@ -# Disable some features that are not needed in our tests and just slow down startup -cluster.deprecation_indexing.enabled=false -indices.lifecycle.history_index_enabled=false -slm.history_index_enabled=false -stack.templates.enabled=false -xpack.ml.enabled=false -xpack.monitoring.templates.enabled=false -xpack.watcher.enabled=false \ No newline at end of file diff --git a/build/container/elasticsearch/env-8.1.properties b/build/container/elasticsearch/env-8.1.properties deleted file mode 100644 index d89332da80b..00000000000 --- a/build/container/elasticsearch/env-8.1.properties +++ /dev/null @@ -1,8 +0,0 @@ -# Disable some features that are not needed in our tests and just slow down startup -cluster.deprecation_indexing.enabled=false -indices.lifecycle.history_index_enabled=false -slm.history_index_enabled=false -stack.templates.enabled=false -xpack.ml.enabled=false -xpack.monitoring.templates.enabled=false -xpack.watcher.enabled=false \ No newline at end of file diff --git a/build/container/elasticsearch/env-8.10.properties b/build/container/elasticsearch/env-8.10.properties deleted file mode 100644 index b6d16ea0c68..00000000000 --- a/build/container/elasticsearch/env-8.10.properties +++ /dev/null @@ -1,10 +0,0 @@ -# Disable some features that are not needed in our tests and just slow down startup -cluster.deprecation_indexing.enabled=false -indices.lifecycle.history_index_enabled=false -slm.history_index_enabled=false -stack.templates.enabled=false -xpack.ent_search.enabled=false -xpack.ml.enabled=false -xpack.monitoring.templates.enabled=false -xpack.profiling.enabled=false -xpack.watcher.enabled=false \ No newline at end of file diff --git a/build/container/elasticsearch/env-8.2.properties b/build/container/elasticsearch/env-8.2.properties deleted file mode 100644 index d89332da80b..00000000000 --- a/build/container/elasticsearch/env-8.2.properties +++ /dev/null @@ -1,8 +0,0 @@ -# Disable some features that are not needed in our tests and just slow down startup -cluster.deprecation_indexing.enabled=false -indices.lifecycle.history_index_enabled=false -slm.history_index_enabled=false -stack.templates.enabled=false -xpack.ml.enabled=false -xpack.monitoring.templates.enabled=false -xpack.watcher.enabled=false \ No newline at end of file diff --git a/build/container/elasticsearch/env-8.3.properties b/build/container/elasticsearch/env-8.3.properties deleted file mode 100644 index d89332da80b..00000000000 --- a/build/container/elasticsearch/env-8.3.properties +++ /dev/null @@ -1,8 +0,0 @@ -# Disable some features that are not needed in our tests and just slow down startup -cluster.deprecation_indexing.enabled=false -indices.lifecycle.history_index_enabled=false -slm.history_index_enabled=false -stack.templates.enabled=false -xpack.ml.enabled=false -xpack.monitoring.templates.enabled=false -xpack.watcher.enabled=false \ No newline at end of file diff --git a/build/container/elasticsearch/env-8.4.properties b/build/container/elasticsearch/env-8.4.properties deleted file mode 100644 index d89332da80b..00000000000 --- a/build/container/elasticsearch/env-8.4.properties +++ /dev/null @@ -1,8 +0,0 @@ -# Disable some features that are not needed in our tests and just slow down startup -cluster.deprecation_indexing.enabled=false -indices.lifecycle.history_index_enabled=false -slm.history_index_enabled=false -stack.templates.enabled=false -xpack.ml.enabled=false -xpack.monitoring.templates.enabled=false -xpack.watcher.enabled=false \ No newline at end of file diff --git a/build/container/elasticsearch/env-8.5.properties b/build/container/elasticsearch/env-8.5.properties deleted file mode 100644 index d89332da80b..00000000000 --- a/build/container/elasticsearch/env-8.5.properties +++ /dev/null @@ -1,8 +0,0 @@ -# Disable some features that are not needed in our tests and just slow down startup -cluster.deprecation_indexing.enabled=false -indices.lifecycle.history_index_enabled=false -slm.history_index_enabled=false -stack.templates.enabled=false -xpack.ml.enabled=false -xpack.monitoring.templates.enabled=false -xpack.watcher.enabled=false \ No newline at end of file diff --git a/build/container/elasticsearch/env-8.6.properties b/build/container/elasticsearch/env-8.6.properties deleted file mode 100644 index d89332da80b..00000000000 --- a/build/container/elasticsearch/env-8.6.properties +++ /dev/null @@ -1,8 +0,0 @@ -# Disable some features that are not needed in our tests and just slow down startup -cluster.deprecation_indexing.enabled=false -indices.lifecycle.history_index_enabled=false -slm.history_index_enabled=false -stack.templates.enabled=false -xpack.ml.enabled=false -xpack.monitoring.templates.enabled=false -xpack.watcher.enabled=false \ No newline at end of file diff --git a/build/container/elasticsearch/env-8.7.properties b/build/container/elasticsearch/env-8.7.properties deleted file mode 100644 index d89332da80b..00000000000 --- a/build/container/elasticsearch/env-8.7.properties +++ /dev/null @@ -1,8 +0,0 @@ -# Disable some features that are not needed in our tests and just slow down startup -cluster.deprecation_indexing.enabled=false -indices.lifecycle.history_index_enabled=false -slm.history_index_enabled=false -stack.templates.enabled=false -xpack.ml.enabled=false -xpack.monitoring.templates.enabled=false -xpack.watcher.enabled=false \ No newline at end of file diff --git a/build/container/elasticsearch/env-8.8.properties b/build/container/elasticsearch/env-8.8.properties deleted file mode 100644 index b6d16ea0c68..00000000000 --- a/build/container/elasticsearch/env-8.8.properties +++ /dev/null @@ -1,10 +0,0 @@ -# Disable some features that are not needed in our tests and just slow down startup -cluster.deprecation_indexing.enabled=false -indices.lifecycle.history_index_enabled=false -slm.history_index_enabled=false -stack.templates.enabled=false -xpack.ent_search.enabled=false -xpack.ml.enabled=false -xpack.monitoring.templates.enabled=false -xpack.profiling.enabled=false -xpack.watcher.enabled=false \ No newline at end of file diff --git a/build/container/elasticsearch/env-8.9.properties b/build/container/elasticsearch/env-8.9.properties deleted file mode 100644 index b6d16ea0c68..00000000000 --- a/build/container/elasticsearch/env-8.9.properties +++ /dev/null @@ -1,10 +0,0 @@ -# Disable some features that are not needed in our tests and just slow down startup -cluster.deprecation_indexing.enabled=false -indices.lifecycle.history_index_enabled=false -slm.history_index_enabled=false -stack.templates.enabled=false -xpack.ent_search.enabled=false -xpack.ml.enabled=false -xpack.monitoring.templates.enabled=false -xpack.profiling.enabled=false -xpack.watcher.enabled=false \ No newline at end of file diff --git a/build/parents/build/pom.xml b/build/parents/build/pom.xml index 33d90fae960..199de858e92 100644 --- a/build/parents/build/pom.xml +++ b/build/parents/build/pom.xml @@ -119,6 +119,8 @@ 5.10.0 1.10.0 6.0.0 + 1.19.1 + 2.2 5.7.0 @@ -684,6 +686,13 @@ pom import + + org.testcontainers + testcontainers-bom + ${version.testcontainers} + pom + import + org.mockito @@ -1186,6 +1195,8 @@ ${project.version} ${test.performance.enable} ${test.elasticsearch.distribution} + ${test.elasticsearch.distribution}:${test.elasticsearch.version} + ${test.elasticsearch.version} diff --git a/build/parents/integrationtest/pom.xml b/build/parents/integrationtest/pom.xml index 31528a32737..0dcbcdafb85 100644 --- a/build/parents/integrationtest/pom.xml +++ b/build/parents/integrationtest/pom.xml @@ -80,97 +80,6 @@ ${test.containers.run.skip} true - - ${test.elasticsearch.run.elastic.image.name}:${test.elasticsearch.run.elastic.image.tag} - elasticsearch - - ${test.elasticsearch.run.elastic.skip} - - WARN - single-node - - - false - - -Xms1g -Xmx1g - - false - - ${rootProject.directory}/build/container/elasticsearch/env-${parsed-version.test.elasticsearch.version.majorVersion}.${parsed-version.test.elasticsearch.version.minorVersion}.properties - /var/lib/elasticsearch - - 9200:9200 - - - Elasticsearch: - default - cyan - - - - http://localhost:9200 - GET - 200 - - - - - - - ${test.elasticsearch.run.opensearch.image.name}:${test.elasticsearch.run.opensearch.image.tag} - opensearch - - ${test.elasticsearch.run.opensearch.skip} - - WARN - single-node - - true - - true - - false - - false - - - 9200:9200 - - - OpenSearch: - default - cyan - - - - http://localhost:9200 - GET - 200 - - - - - ${test.database.run.postgres.image.name}:${test.database.run.postgres.image.tag} postgres @@ -450,53 +359,8 @@ - - - - elasticsearch-run - - - - !test.elasticsearch.connection.uris - - - - http://localhost:9200 - - - - - elasticsearch-do-not-run - - - - test.elasticsearch.connection.uris - - - - true - - - - - elasticsearch-test-skip - - - - test.elasticsearch.skip - true - - - - true - - - aws @@ -516,47 +380,6 @@ - - opensearch - - - test.elasticsearch.distribution - opensearch - - - - ${test.elasticsearch.run.skip} - true - ${version.org.opensearch.latest} - - - - - amazon-opensearch-serverless - - - test.elasticsearch.distribution - amazon-opensearch-serverless - - - - - - - - ${test.elasticsearch.run.skip} - true - ${version.org.opensearch.latest} - - ${version.org.elasticsearch.latest} - - - diff --git a/build/parents/springtest/pom.xml b/build/parents/springtest/pom.xml index 3c451495474..2f67b9e15e0 100644 --- a/build/parents/springtest/pom.xml +++ b/build/parents/springtest/pom.xml @@ -57,6 +57,13 @@ pom import + + org.testcontainers + testcontainers-bom + ${version.testcontainers} + pom + import + org.springframework.boot spring-boot-dependencies diff --git a/util/internal/integrationtest/backend/elasticsearch/src/main/java/org/hibernate/search/util/impl/integrationtest/backend/elasticsearch/ElasticsearchTestHostConnectionConfiguration.java b/util/internal/integrationtest/backend/elasticsearch/src/main/java/org/hibernate/search/util/impl/integrationtest/backend/elasticsearch/ElasticsearchTestHostConnectionConfiguration.java index fea81fa08cc..ee1f52bf7c2 100644 --- a/util/internal/integrationtest/backend/elasticsearch/src/main/java/org/hibernate/search/util/impl/integrationtest/backend/elasticsearch/ElasticsearchTestHostConnectionConfiguration.java +++ b/util/internal/integrationtest/backend/elasticsearch/src/main/java/org/hibernate/search/util/impl/integrationtest/backend/elasticsearch/ElasticsearchTestHostConnectionConfiguration.java @@ -35,7 +35,14 @@ public static ElasticsearchTestHostConnectionConfiguration get() { private final String awsCredentialsSecretAccessKey; private ElasticsearchTestHostConnectionConfiguration() { - this.uris = System.getProperty( "test.elasticsearch.connection.uris" ); + String uris = System.getProperty( "test.elasticsearch.connection.uris" ); + if ( uris == null || uris.trim().isEmpty() ) { + // need to start a container: + String host = SearchBackendContainer.host(); + int port = SearchBackendContainer.mappedPort( 9200 ); + uris = host + ":" + port; + } + this.uris = uris; this.username = System.getProperty( "test.elasticsearch.connection.username" ); this.password = System.getProperty( "test.elasticsearch.connection.password" ); String enabledAsString = System.getProperty( "test.elasticsearch.connection.aws.signing.enabled" ); diff --git a/util/internal/integrationtest/backend/elasticsearch/src/main/java/org/hibernate/search/util/impl/integrationtest/backend/elasticsearch/SearchBackendContainer.java b/util/internal/integrationtest/backend/elasticsearch/src/main/java/org/hibernate/search/util/impl/integrationtest/backend/elasticsearch/SearchBackendContainer.java new file mode 100644 index 00000000000..af35b05f56d --- /dev/null +++ b/util/internal/integrationtest/backend/elasticsearch/src/main/java/org/hibernate/search/util/impl/integrationtest/backend/elasticsearch/SearchBackendContainer.java @@ -0,0 +1,131 @@ +/* + * Hibernate Search, full-text search for your domain model + * + * License: GNU Lesser General Public License (LGPL), version 2.1 or later + * See the lgpl.txt file in the root directory or . + */ +package org.hibernate.search.util.impl.integrationtest.backend.elasticsearch; + +import java.time.Duration; + +import org.hibernate.search.backend.elasticsearch.ElasticsearchDistributionName; +import org.hibernate.search.backend.elasticsearch.ElasticsearchVersion; + +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.HttpWaitStrategy; +import org.testcontainers.utility.DockerImageName; + +public final class SearchBackendContainer { + + private SearchBackendContainer() { + } + + private static final GenericContainer SEARCH_CONTAINER; + + static { + String distribution = System.getProperty( "org.hibernate.search.integrationtest.backend.elasticsearch.distribution" ); + String tag = System.getProperty( "org.hibernate.search.integrationtest.backend.elasticsearch.tag" ); + + ElasticsearchVersion version = ElasticsearchVersion.of( ElasticsearchDistributionName.of( distribution ), tag ); + + SEARCH_CONTAINER = "elastic".equalsIgnoreCase( distribution ) ? elasticsearch( tag, version ) : opensearch( tag ); + } + + public static int mappedPort(int port) { + startIfNeeded(); + return SEARCH_CONTAINER.getMappedPort( port ); + } + + public static String host() { + startIfNeeded(); + return SEARCH_CONTAINER.getHost(); + } + + private static void startIfNeeded() { + if ( !SEARCH_CONTAINER.isRunning() ) { + SEARCH_CONTAINER.start(); + } + } + + private static GenericContainer elasticsearch(String imageTag, ElasticsearchVersion version) { + GenericContainer container = common( "elastic/elasticsearch", imageTag ) + .withEnv( "logger.level", "WARN" ) + .withEnv( "discovery.type", "single-node" ) + // Older images require HTTP authentication for all requests; + // it's not practical for testing, so we disable that: + .withEnv( "xpack.security.enabled", "false" ) + // Limit the RAM usage. + // Recent versions of ES limit themselves to 50% of the total available RAM, + // but on CI this is sometimes too much, as we also have the Maven JVM + // and the JVM that runs tests taking up a significant amount of RAM, + // leaving too little for filesystem caches and resulting in freezes: + .withEnv( "ES_JAVA_OPTS", "-Xms1g -Xmx1g" ) + // Disable disk-based shard allocation thresholds: on large, relatively full disks (>90% used), + // it will lead to index creation to get stuck waiting for other nodes to join the cluster, + // which will never happen since we only have one node. + // See https://www.elastic.co/guide/en/elasticsearch/reference/7.17/modules-cluster.html#disk-based-shard-allocation + .withEnv( "cluster.routing.allocation.disk.threshold_enabled", "false" ); + + // Disable a few features that we don't use and that just slow up container startup. + if ( version.majorOptional().orElse( Integer.MIN_VALUE ) == 8 ) { + if ( version.minor().orElse( Integer.MAX_VALUE ) > 7 ) { + container.withEnv( "cluster.deprecation_indexing.enabled", "false" ) + .withEnv( "xpack.profiling.enabled", "false" ) + .withEnv( "xpack.ent_search.enabled", "false" ); + } + return container.withEnv( "cluster.deprecation_indexing.enabled", "false" ) + .withEnv( "indices.lifecycle.history_index_enabled", "false" ) + .withEnv( "slm.history_index_enabled", "false" ) + .withEnv( "stack.templates.enabled", "false" ) + .withEnv( "xpack.ml.enabled", "false" ) + .withEnv( "xpack.monitoring.templates.enabled", "false" ) + .withEnv( "xpack.watcher.enabled", "false" ); + } + + if ( version.majorOptional().orElse( Integer.MIN_VALUE ) == 7 ) { + if ( version.minor().orElse( Integer.MAX_VALUE ) > 15 ) { + container.withEnv( "cluster.deprecation_indexing.enabled", "false" ) + .withEnv( "slm.history_index_enabled", "false" ); + } + return container.withEnv( "indices.lifecycle.history_index_enabled", "false" ) + .withEnv( "stack.templates.enabled", "false" ) + .withEnv( "xpack.ml.enabled", "false" ) + .withEnv( "xpack.watcher.enabled", "false" ); + } + + return container; + } + + private static GenericContainer opensearch(String imageTag) { + return common( "opensearchproject/opensearch", imageTag ) + .withEnv( "logger.level", "WARN" ) + .withEnv( "discovery.type", "single-node" ) + // Prevent swapping + .withEnv( "bootstrap.memory_lock", "true" ) + // OpenSearch expects SSL and uses a self-signed certificate by default; + // it's not practical for testing, so we disable that. + .withEnv( "plugins.security.disabled", "true" ) + // ISM floods the logs with errors, and we don't need it. + // See https://docs-beta.opensearch.org/im-plugin/ism/settings/ + .withEnv( "plugins.index_state_management.enabled", "false" ) + // Disable disk-based shard allocation thresholds: on large, relatively full disks (>90% used), + // it will lead to index creation to get stuck waiting for other nodes to join the cluster, + // which will never happen since we only have one node. + // See https://www.elastic.co/guide/en/elasticsearch/reference/7.17/modules-cluster.html#disk-based-shard-allocation + // See https://opensearch.org/docs/latest/opensearch/popular-api/#change-disk-watermarks-or-other-cluster-settings + .withEnv( "cluster.routing.allocation.disk.threshold_enabled", "false" ); + } + + /* + * Suppress "Resource leak: '' is never closed". Testcontainers take care of closing + * this resource in the end. + */ + @SuppressWarnings("resource") + private static GenericContainer common(String image, String tag) { + return new GenericContainer<>( DockerImageName.parse( image ).withTag( tag ) ) + .withExposedPorts( 9200, 9300 ) + .waitingFor( new HttpWaitStrategy().forPort( 9200 ).forStatusCode( 200 ) ) + .withStartupTimeout( Duration.ofMinutes( 5 ) ) + .withReuse( true ); + } +} diff --git a/util/internal/integrationtest/common/pom.xml b/util/internal/integrationtest/common/pom.xml index 213406dd800..8e4059e3817 100644 --- a/util/internal/integrationtest/common/pom.xml +++ b/util/internal/integrationtest/common/pom.xml @@ -22,6 +22,11 @@ org.hibernate.search hibernate-search-util-internal-test-common + + + org.testcontainers + testcontainers + From 51f0eb9768a583ec224c4e30cf30ec62f2795f11 Mon Sep 17 00:00:00 2001 From: marko-bekhta Date: Thu, 1 Dec 2022 20:24:02 +0100 Subject: [PATCH 02/20] HSEARCH-4674 Use test containers for databases --- build/parents/integrationtest/pom.xml | 347 ++--------------- .../integrationtest/mapper/orm/pom.xml | 4 + .../mapper/orm/DatabaseContainer.java | 368 ++++++++++++++++++ .../mapper/orm/OrmSetupHelper.java | 8 +- 4 files changed, 403 insertions(+), 324 deletions(-) create mode 100644 util/internal/integrationtest/mapper/orm/src/main/java/org/hibernate/search/util/impl/integrationtest/mapper/orm/DatabaseContainer.java diff --git a/build/parents/integrationtest/pom.xml b/build/parents/integrationtest/pom.xml index 0dcbcdafb85..f9272592f6c 100644 --- a/build/parents/integrationtest/pom.xml +++ b/build/parents/integrationtest/pom.xml @@ -26,19 +26,6 @@ ${java-version.test.compiler.java_home} ${java-version.test.compiler} generate-sources - - - -Dhibernate.dialect=${db.dialect} - -Dhibernate.connection.driver_class=${jdbc.driver} - -Dhibernate.connection.url=${jdbc.url} - -Dhibernate.connection.username=${jdbc.user} - -Dhibernate.connection.password=${jdbc.pass} - -Dhibernate.connection.isolation=${jdbc.isolation} - false - - ${test.elasticsearch.run.skip} @@ -72,257 +54,6 @@ - - io.fabric8 - docker-maven-plugin - ${version.docker.maven.plugin} - - ${test.containers.run.skip} - true - - - ${test.database.run.postgres.image.name}:${test.database.run.postgres.image.tag} - postgres - - ${test.database.run.postgres.skip} - - hibernate_orm_test - hibernate_orm_test - hibernate_orm_test - - - 5432:5432 - - - PostgreSQL: - default - blue - - - - - (?s)ready to accept connections.*ready to accept connections - - - - - ${test.database.run.mariadb.image.name}:${test.database.run.mariadb.image.tag} - mariadb - - ${test.database.run.mariadb.skip} - --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci - - hibernate_orm_test - hibernate_orm_test - hibernate_orm_test - true - - - 3306:3306 - - - MariaDB: - default - blue - - - /var/lib/mysql - - - - mysqladmin ping -h localhost -u hibernate_orm_test -phibernate_orm_test - - - - - - ${test.database.run.mysql.image.name}:${test.database.run.mysql.image.tag} - mysql - - ${test.database.run.mysql.skip} - - hibernate_orm_test - hibernate_orm_test - hibernate_orm_test - true - - - 3306:3306 - - - MySQL: - default - blue - - - /var/lib/mysql - - - - mysqladmin ping -h localhost -u hibernate_orm_test -phibernate_orm_test - - - - - - ${test.database.run.db2.image.name}:${test.database.run.db2.image.tag} - db2 - - ${test.database.run.db2.skip} - - hreact - hreact - hreact - accept - - false - false - false - - - 50005:50000 - - - DB2: - default - blue - - - bridge - - true - - - .*INSTANCE.* - - - - 300000 - - - - - ${test.database.run.oracle.image.name}:${test.database.run.oracle.image.tag} - oracle - - ${test.database.run.oracle.skip} - - hibernate_orm_test - - - 1521:1521 - - - Oracle Database: - default - blue - - - - - - DATABASE IS READY TO USE! - - - - - ${test.database.run.mssql.image.name}:${test.database.run.mssql.image.tag} - mssql - - ${test.database.run.mssql.skip} - - Y - ActuallyRequired11Complexity - - - 1433:1433 - - - MS SQL Server: - default - blue - - - SQL Server is now ready for client connections - - - - - - - ${test.database.run.cockroachdb.image.name}:${test.database.run.cockroachdb.image.tag} - cockroachdb - - ${test.database.run.cockroachdb.skip} - - - start-single-node - --insecure - - --store=type=mem,size=.5 - - --max-offset=100ms - --log=sinks: {stderr: {channels: [OPS]}} - - - - - nofile - 1956 - 1956 - - - - hibernate_orm_test - hibernate_orm_test - hibernate_orm_test - - - 26257:26257 - - - CockroachDB: - default - blue - - - CockroachDB node starting at - - - - - - - - docker-start - pre-integration-test - - - stop - start - - - - docker-stop - post-integration-test - - stop - - - - @@ -393,14 +124,11 @@ - org.hibernate.dialect.H2Dialect com.h2database h2 - org.h2.Driver - jdbc:h2:mem:db1;DB_CLOSE_DELAY=-1 - sa - sa - + + -Dorg.hibernate.search.integrationtest.orn.database=h2 + @@ -409,15 +137,11 @@ ci-postgresql - ${test.database.run.skip} - org.hibernate.dialect.PostgreSQLDialect org.postgresql postgresql - org.postgresql.Driver - jdbc:postgresql://localhost:5432/hibernate_orm_test - hibernate_orm_test - hibernate_orm_test - + + -Dorg.hibernate.search.integrationtest.orn.database=postgresql + @@ -426,15 +150,11 @@ ci-mariadb - ${test.database.run.skip} - org.hibernate.dialect.MariaDBDialect org.mariadb.jdbc mariadb-java-client - org.mariadb.jdbc.Driver - jdbc:mariadb://localhost/hibernate_orm_test - hibernate_orm_test - hibernate_orm_test - + + -Dorg.hibernate.search.integrationtest.orn.database=mariadb + @@ -443,15 +163,11 @@ ci-mysql - ${test.database.run.skip} - org.hibernate.dialect.MySQLDialect com.mysql mysql-connector-j - com.mysql.jdbc.Driver - jdbc:mysql://localhost/hibernate_orm_test - hibernate_orm_test - hibernate_orm_test - + + -Dorg.hibernate.search.integrationtest.orn.database=mysql + @@ -460,15 +176,11 @@ ci-db2 - ${test.database.run.skip} - org.hibernate.dialect.DB2Dialect com.ibm.db2 jcc - com.ibm.db2.jcc.DB2Driver - jdbc:db2://localhost:50005/hreact - hreact - hreact - + + -Dorg.hibernate.search.integrationtest.orn.database=db2 + @@ -477,15 +189,11 @@ ci-oracle - ${test.database.run.skip} - org.hibernate.dialect.OracleDialect com.oracle.database.jdbc ojdbc11 - oracle.jdbc.OracleDriver - jdbc:oracle:thin:@localhost:1521/XE - SYSTEM - hibernate_orm_test - + + -Dorg.hibernate.search.integrationtest.orn.database=oracle + -Doracle.jdbc.timezoneAsRegion=false @@ -496,14 +204,11 @@ ci-mssql - ${test.database.run.skip} - org.hibernate.dialect.SQLServerDialect com.microsoft.sqlserver mssql-jdbc - com.microsoft.sqlserver.jdbc.SQLServerDriver - jdbc:sqlserver://localhost:1433;databaseName=tempdb;encrypt=false - sa - ActuallyRequired11Complexity + + -Dorg.hibernate.search.integrationtest.orn.database=mssql + @@ -512,16 +217,12 @@ ci-cockroachdb - ${test.database.run.skip} - org.hibernate.dialect.CockroachDialect org.postgresql postgresql - org.postgresql.Driver - jdbc:postgresql://localhost:26257/defaultdb?sslmode=disable - root - - + + -Dorg.hibernate.search.integrationtest.orn.database=cockroachdb + diff --git a/util/internal/integrationtest/mapper/orm/pom.xml b/util/internal/integrationtest/mapper/orm/pom.xml index a5d23c37874..25d8018196d 100644 --- a/util/internal/integrationtest/mapper/orm/pom.xml +++ b/util/internal/integrationtest/mapper/orm/pom.xml @@ -49,6 +49,10 @@ org.hibernate.orm hibernate-testing + + org.testcontainers + jdbc + diff --git a/util/internal/integrationtest/mapper/orm/src/main/java/org/hibernate/search/util/impl/integrationtest/mapper/orm/DatabaseContainer.java b/util/internal/integrationtest/mapper/orm/src/main/java/org/hibernate/search/util/impl/integrationtest/mapper/orm/DatabaseContainer.java new file mode 100644 index 00000000000..012578afea6 --- /dev/null +++ b/util/internal/integrationtest/mapper/orm/src/main/java/org/hibernate/search/util/impl/integrationtest/mapper/orm/DatabaseContainer.java @@ -0,0 +1,368 @@ +/* + * Hibernate Search, full-text search for your domain model + * + * License: GNU Lesser General Public License (LGPL), version 2.1 or later + * See the lgpl.txt file in the root directory or . + */ +package org.hibernate.search.util.impl.integrationtest.mapper.orm; + +import java.time.Duration; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.concurrent.Future; + +import com.github.dockerjava.api.model.HostConfig; +import com.github.dockerjava.api.model.Ulimit; + +import org.testcontainers.containers.JdbcDatabaseContainer; +import org.testcontainers.utility.DockerImageName; + +/* + * Suppress "Resource leak: '' is never closed". Testcontainers take care of closing + * these resources in the end. + */ +@SuppressWarnings("resource") +public final class DatabaseContainer { + + private static final Duration REGULAR_TIMEOUT = Duration.ofMinutes( 5 ); + private static final Duration EXTENDED_TIMEOUT = Duration.ofMinutes( 10 ); + + private DatabaseContainer() { + } + + private static final SupportedDatabase DATABASE; + private static final HibernateSearchJdbcDatabaseContainer DATABASE_CONTAINER; + + + static { + String database = System.getProperty( "org.hibernate.search.integrationtest.orn.database" ); + DATABASE = SupportedDatabase.from( database ); + + DATABASE_CONTAINER = DATABASE.container(); + } + + + public static Configuration configuration() { + if ( !SupportedDatabase.H2.equals( DATABASE ) ) { + DATABASE_CONTAINER.start(); + } + Configuration configuration = DATABASE.configuration( DATABASE_CONTAINER ); + configuration.addAsSystemProperties(); + return configuration; + } + + private enum SupportedDatabase { + H2 { + @Override + String dialect() { + return org.hibernate.dialect.H2Dialect.class.getName(); + } + + @Override + Configuration configuration(JdbcDatabaseContainer container) { + return new Configuration( + dialect(), + "org.h2.Driver", + "jdbc:h2:mem:db1;DB_CLOSE_DELAY=-1", + "sa", + "sa", + "" + ); + } + + @Override + HibernateSearchJdbcDatabaseContainer container() { + return null; + } + }, + POSTGRESQL { + @Override + String dialect() { + return org.hibernate.dialect.PostgreSQLDialect.class.getName(); + } + + @Override + HibernateSearchJdbcDatabaseContainer container() { + return new HibernateSearchJdbcDatabaseContainer( + DockerImageName.parse( "postgres" ).withTag( "13.1" ), + "org.postgresql.Driver", + "jdbc:postgresql://%s:%d/hibernate_orm_test", + 5432, + "hibernate_orm_test", + "hibernate_orm_test", + "select 1" + ).withEnv( "POSTGRES_USER", "hibernate_orm_test" ) + .withEnv( "POSTGRES_PASSWORD", "hibernate_orm_test" ) + .withEnv( "POSTGRES_DB", "hibernate_orm_test" ); + } + }, + MARIADB { + @Override + String dialect() { + return org.hibernate.dialect.MariaDBDialect.class.getName(); + } + + @Override + HibernateSearchJdbcDatabaseContainer container() { + return new HibernateSearchJdbcDatabaseContainer( + DockerImageName.parse( "mariadb" ).withTag( "10.5.8" ), + "org.mariadb.jdbc.Driver", + "jdbc:mariadb://%s:%d/hibernate_orm_test", + 3306, + "hibernate_orm_test", + "hibernate_orm_test", + "select 1" + ).withCommand( "--character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci" ) + .withEnv( "MYSQL_USER", "hibernate_orm_test" ) + .withEnv( "MYSQL_PASSWORD", "hibernate_orm_test" ) + .withEnv( "MYSQL_DATABASE", "hibernate_orm_test" ) + .withEnv( "MYSQL_RANDOM_ROOT_PASSWORD", "true" ) + .withTmpFs( Collections.singletonMap( "/var/lib/mysql", "" ) ); + } + }, + MYSQL { + @Override + String dialect() { + return org.hibernate.dialect.MySQLDialect.class.getName(); + } + + @Override + HibernateSearchJdbcDatabaseContainer container() { + return new HibernateSearchJdbcDatabaseContainer( + DockerImageName.parse( "mysql" ).withTag( "8.0.22" ), + "com.mysql.jdbc.Driver", + "jdbc:mysql://%s:%d/hibernate_orm_test", + 3306, + "hibernate_orm_test", + "hibernate_orm_test", + "select 1" + ).withCommand( "--character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci" ) + .withEnv( "MYSQL_USER", "hibernate_orm_test" ) + .withEnv( "MYSQL_PASSWORD", "hibernate_orm_test" ) + .withEnv( "MYSQL_DATABASE", "hibernate_orm_test" ) + .withEnv( "MYSQL_RANDOM_ROOT_PASSWORD", "true" ) + .withTmpFs( Collections.singletonMap( "/var/lib/mysql", "" ) ); + } + }, + DB2 { + @Override + String dialect() { + return org.hibernate.dialect.DB2Dialect.class.getName(); + } + + @Override + HibernateSearchJdbcDatabaseContainer container() { + return new HibernateSearchJdbcDatabaseContainer( + DockerImageName.parse( "ibmcom/db2" ).withTag( "11.5.8.0" ), + "com.ibm.db2.jcc.DB2Driver", + "jdbc:db2://%s:%d/hreact", + 50000, + "hreact", + "hreact", + "SELECT 1 FROM SYSIBM.SYSDUMMY1" + + ).withNetworkMode( "bridge" ) + .withEnv( "DB2INSTANCE", "hreact" ) + .withEnv( "DB2INST1_PASSWORD", "hreact" ) + .withEnv( "DBNAME", "hreact" ) + .withEnv( "LICENSE", "accept" ) + // These help the DB2 container start faster + .withEnv( "AUTOCONFIG", "false" ) + .withEnv( "ARCHIVE_LOGS", "false" ) + .withEnv( "PERSISTENT_HOME", "false" ) + // because it takes ages to start and select query will just get tired to retry + // this is a JDBCContainer specific setting + .withStartupTimeoutSeconds( 600 ) + .withStartupTimeout( EXTENDED_TIMEOUT ); + } + }, + ORACLE { + @Override + String dialect() { + return org.hibernate.dialect.OracleDialect.class.getName(); + } + + @Override + HibernateSearchJdbcDatabaseContainer container() { + return new HibernateSearchJdbcDatabaseContainer( + DockerImageName.parse( "gvenzl/oracle-xe" ).withTag( "21-slim-faststart" ), + "oracle.jdbc.OracleDriver", + "jdbc:oracle:thin:@%s:%d/XE", + 1521, + "SYSTEM", + "hibernate_orm_test", + "select 1 from dual" + ).withEnv( "ORACLE_PASSWORD", "hibernate_orm_test" ) + .withStartupTimeout( EXTENDED_TIMEOUT ); + } + }, + MSSQL { + @Override + String dialect() { + return org.hibernate.dialect.SQLServerDialect.class.getName(); + } + + @Override + HibernateSearchJdbcDatabaseContainer container() { + return new HibernateSearchJdbcDatabaseContainer( + DockerImageName.parse( "mcr.microsoft.com/mssql/server" ).withTag( "2019-CU8-ubuntu-16.04" ), + "com.microsoft.sqlserver.jdbc.SQLServerDriver", + "jdbc:sqlserver://%s:%d;databaseName=tempdb", + 1433, + "SA", + "ActuallyRequired11Complexity", + "select 1" + ).withEnv( "ACCEPT_EULA", "Y" ) + .withEnv( "SA_PASSWORD", "ActuallyRequired11Complexity" ) + .withStartupTimeout( EXTENDED_TIMEOUT ); + } + }, + COCKROACHDB { + @Override + String dialect() { + return org.hibernate.dialect.CockroachDialect.class.getName(); + } + + @Override + HibernateSearchJdbcDatabaseContainer container() { + return new HibernateSearchJdbcDatabaseContainer( + DockerImageName.parse( "cockroachdb/cockroach" ).withTag( "v22.1.4" ), + "org.postgresql.Driver", + "jdbc:postgresql://%s:%d/defaultdb?sslmode=disable", + 26257, + "root", + "", + "select 1" + ).withCommand( "start-single-node --insecure" ) + .withStartupTimeout( EXTENDED_TIMEOUT ) + .withCreateContainerCmdModifier( cmd -> { + HostConfig hostConfig = cmd.getHostConfig(); + if ( hostConfig == null ) { + throw new IllegalStateException( "Host config is `null`. Cannot redefine the ulimits!" ); + } + hostConfig.withUlimits( List.of( new Ulimit( "nofile", 1956L, 1956L ) ) ); + } ); + } + }; + + Configuration configuration(JdbcDatabaseContainer container) { + return new Configuration( + dialect(), + container.getDriverClassName(), + container.getJdbcUrl(), + container.getUsername(), + container.getPassword(), + "" + ); + } + + abstract String dialect(); + + abstract HibernateSearchJdbcDatabaseContainer container(); + + static SupportedDatabase from(String name) { + for ( SupportedDatabase database : values() ) { + if ( database.name().equalsIgnoreCase( name ) ) { + return database; + } + } + throw new IllegalStateException( "Unsupported database requested: " + name ); + } + } + + private static class HibernateSearchJdbcDatabaseContainer + extends JdbcDatabaseContainer { + + private final String driverClassName; + private final String jdbcUrlPattern; + private final int port; + private final String username; + private final String password; + private final String testQueryString; + + public HibernateSearchJdbcDatabaseContainer(Future image, String driverClassName, String jdbcUrlPattern, + int port, String username, String password, String testQueryString) { + super( image ); + this.driverClassName = driverClassName; + this.jdbcUrlPattern = jdbcUrlPattern; + this.port = port; + this.username = username; + this.password = password; + this.testQueryString = testQueryString; + posInit(); + } + + public HibernateSearchJdbcDatabaseContainer(DockerImageName dockerImageName, String driverClassName, + String jdbcUrlPattern, int port, String username, String password, String testQueryString) { + super( dockerImageName ); + this.driverClassName = driverClassName; + this.jdbcUrlPattern = jdbcUrlPattern; + this.port = port; + this.username = username; + this.password = password; + this.testQueryString = testQueryString; + posInit(); + } + + private void posInit() { + withExposedPorts( port ); + withReuse( true ); + withStartupTimeout( REGULAR_TIMEOUT ); + } + + @Override + public String getDriverClassName() { + return driverClassName; + } + + @Override + public String getJdbcUrl() { + return String.format( Locale.ROOT, jdbcUrlPattern, getHost(), getMappedPort( port ) ); + } + + @Override + public String getUsername() { + return username; + } + + @Override + public String getPassword() { + return password; + } + + @Override + protected String getTestQueryString() { + return testQueryString; + } + } + + + public static class Configuration { + private final String dialect; + private final String driver; + private final String url; + private final String user; + private final String pass; + private final String isolation; + + public Configuration(String dialect, String driver, String url, String user, String pass, String isolation) { + this.dialect = dialect; + this.driver = driver; + this.url = url; + this.user = user; + this.pass = pass; + this.isolation = isolation; + } + + private void addAsSystemProperties() { + System.setProperty( "hibernate.dialect", this.dialect ); + System.setProperty( "hibernate.connection.driver_class", this.driver ); + System.setProperty( "hibernate.connection.url", this.url ); + System.setProperty( "hibernate.connection.username", this.user ); + System.setProperty( "hibernate.connection.password", this.pass ); + System.setProperty( "hibernate.connection.isolation", this.isolation ); + } + } + +} diff --git a/util/internal/integrationtest/mapper/orm/src/main/java/org/hibernate/search/util/impl/integrationtest/mapper/orm/OrmSetupHelper.java b/util/internal/integrationtest/mapper/orm/src/main/java/org/hibernate/search/util/impl/integrationtest/mapper/orm/OrmSetupHelper.java index 9267230387a..138ada8c7fc 100644 --- a/util/internal/integrationtest/mapper/orm/src/main/java/org/hibernate/search/util/impl/integrationtest/mapper/orm/OrmSetupHelper.java +++ b/util/internal/integrationtest/mapper/orm/src/main/java/org/hibernate/search/util/impl/integrationtest/mapper/orm/OrmSetupHelper.java @@ -46,6 +46,13 @@ public class OrmSetupHelper private static final Map DEFAULT_PROPERTIES; static { + Map defaults = new LinkedHashMap<>(); + + String jdbcUrl = System.getProperty( "hibernate.connection.url" ); + if ( jdbcUrl == null || jdbcUrl.trim().isEmpty() ) { + DatabaseContainer.configuration(); + } + // we don't need a ServiceLoader using a general-purpose aggregated class loader, // since we expect the service impl in the direct dependent test module. ServiceLoader serviceLoader = ServiceLoader.load( OrmSetupHelperConfig.class ); @@ -53,7 +60,6 @@ public class OrmSetupHelper if ( iterator.hasNext() ) { OrmSetupHelperConfig next = iterator.next(); DEFAULT_COORDINATION_STRATEGY_EXPECTATIONS = next.coordinationStrategyExpectations(); - Map defaults = new LinkedHashMap<>(); next.overrideHibernateSearchDefaults( defaults::put ); DEFAULT_PROPERTIES = Collections.unmodifiableMap( defaults ); } From 455a68cdc2bcddd5c39a15be4ae7554c2c36ac19 Mon Sep 17 00:00:00 2001 From: marko-bekhta Date: Thu, 1 Dec 2022 20:24:24 +0100 Subject: [PATCH 03/20] HSEARCH-4674 Remove maven docker plugin --- .github/workflows/build.yml | 6 +---- CONTRIBUTING.md | 10 ++----- Jenkinsfile | 24 ----------------- build/parents/build/pom.xml | 3 --- build/parents/integrationtest/pom.xml | 27 ------------------- documentation/pom.xml | 4 --- integrationtest/backend/elasticsearch/pom.xml | 4 --- .../java/modules/orm-elasticsearch/pom.xml | 4 --- .../java/modules/orm-lucene/pom.xml | 4 --- .../orm-outbox-polling-elasticsearch/pom.xml | 4 --- .../pojo-standalone-elasticsearch/pom.xml | 4 --- .../modules/pojo-standalone-lucene/pom.xml | 4 --- integrationtest/mapper/orm-cdi/pom.xml | 5 ---- integrationtest/mapper/orm-envers/pom.xml | 5 ---- .../mapper/orm-jakarta-batch/pom.xml | 5 ---- .../mapper/orm-outbox-polling/pom.xml | 4 --- .../mapper/orm-realbackend/pom.xml | 4 --- integrationtest/mapper/orm-spring/pom.xml | 4 --- integrationtest/mapper/orm/pom.xml | 5 ---- .../pojo-standalone-realbackend/pom.xml | 4 --- .../performance/backend/elasticsearch/pom.xml | 4 --- integrationtest/showcase/library/pom.xml | 4 --- integrationtest/v5migrationhelper/orm/pom.xml | 5 ---- pom.xml | 6 ----- 24 files changed, 3 insertions(+), 150 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e875fff9a9d..64efadc8873 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -66,10 +66,6 @@ jobs: # We can't start Linux containers on GitHub Actions' Windows VMs, # so we can't run Elasticsearch tests. # See https://github.com/actions/runner-images/issues/1143#issuecomment-972929995 - # For some reason docker-maven-plugin will error out with "All pipe instances are busy" - # on GitHub Actions' Windows VMs (when using Bash?), - # so we also explicitly disable docker-maven-plugin. - # See https://github.com/fabric8io/docker-maven-plugin/issues/548#issuecomment-255477600 - { name: "Windows JDK 17", runs-on: 'windows-latest', @@ -77,7 +73,7 @@ jobs: version: 17 }, maven: { - args: '-Dtest.elasticsearch.skip=true -Dtest.containers.run.skip=true' + args: '-Dtest.elasticsearch.skip=true' } } steps: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1fbe28dca22..61dbb1cb87e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -223,8 +223,8 @@ and run unit tests and integration tests. ``` Note: on Windows, you will need a Docker install able to run Linux containers. -If you don't have that, you can skip the Elasticsearch tests and all container startups: -`./mvnw clean install -Dtest.elasticsearch.skip=true -Dtest.containers.run.skip=true`. +If you don't have that, you can skip the Elasticsearch tests and only run tests against H2 database (without using DB containers): +`./mvnw clean install -Dtest.elasticsearch.skip=true`. Note: the produced JARs are compatible with Java 8 and later, regardless of the JDK used to build Hibernate Search. @@ -346,12 +346,6 @@ you can skip all Elasticsearch tests (and thus the Elasticsearch container start ./mvnw clean install -Dtest.elasticsearch.skip=true ``` -On Windows, you might need to also skip all docker container features: - -```bash -./mvnw clean install -Dtest.elasticsearch.skip=true -Dtest.containers.run.skip=true -``` - Alternatively, you can prevent the build from launching an Elasticsearch server automatically and run Elasticsearch-related tests against your own server using the `test.elasticsearch.connection.uris` property: diff --git a/Jenkinsfile b/Jenkinsfile index c5efb672179..175a1b2bef4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -540,30 +540,6 @@ stage('Non-default environments') { ''' String mavenDockerArgs = "" def startedContainers = false - // DB2 setup is super slow (~5 to 15 minutes). - // We can't afford to do that once per module, - // so we start DB2 here, once and for all. - if ( buildEnv.dbName == 'db2' ) { - // Prevent the actual build from starting the DB container - mavenBuildAdditionalArgs += " -Dtest.database.run.db2.skip=true" - // Pick a module that doesn't normally execute the docker-maven-plugin, - // but that has the configuration necessary to start the DB container: - // that way, the maven-docker-plugin won't try to remove our container - // when we execute maven another time to run the tests. - // (This works because maven-docker-plugin filters containers to stop - // based on the Maven GAV coordinates of the Maven project that started that container, - // which are attached to the container thanks to a container label). - mavenDockerArgs = """ \ - -pl build/parents/integrationtest \ - -P$buildEnv.mavenProfile \ - -Dtest.database.run.db2.skip=false \ - """ - // Cleanup just in case some containers were left over from a previous build. - sh "mvn docker:stop $mavenDockerArgs" - pullContainerImages mavenDockerArgs - sh "mvn docker:start $mavenDockerArgs" - startedContainers = true - } try { mavenNonDefaultBuild buildEnv, """ \ -Pdist \ diff --git a/build/parents/build/pom.xml b/build/parents/build/pom.xml index 199de858e92..cc5100c6b31 100644 --- a/build/parents/build/pom.xml +++ b/build/parents/build/pom.xml @@ -185,9 +185,6 @@ - - false - diff --git a/build/parents/integrationtest/pom.xml b/build/parents/integrationtest/pom.xml index f9272592f6c..7b93cc32a71 100644 --- a/build/parents/integrationtest/pom.xml +++ b/build/parents/integrationtest/pom.xml @@ -59,33 +59,6 @@ - - - skippingAllTests - - - skipTests - - - - - true - - - - skippingIntegrationTests - - - skipITs - - - - - true - - - diff --git a/documentation/pom.xml b/documentation/pom.xml index 51c1fa8c84c..208605c1d13 100644 --- a/documentation/pom.xml +++ b/documentation/pom.xml @@ -103,10 +103,6 @@ - - io.fabric8 - docker-maven-plugin - org.apache.maven.plugins maven-compiler-plugin diff --git a/integrationtest/backend/elasticsearch/pom.xml b/integrationtest/backend/elasticsearch/pom.xml index c8cafe310a7..28cb2e14dbe 100644 --- a/integrationtest/backend/elasticsearch/pom.xml +++ b/integrationtest/backend/elasticsearch/pom.xml @@ -76,10 +76,6 @@ - - io.fabric8 - docker-maven-plugin - diff --git a/integrationtest/java/modules/orm-elasticsearch/pom.xml b/integrationtest/java/modules/orm-elasticsearch/pom.xml index ff8f331278c..10583373e83 100644 --- a/integrationtest/java/modules/orm-elasticsearch/pom.xml +++ b/integrationtest/java/modules/orm-elasticsearch/pom.xml @@ -77,10 +77,6 @@ - - io.fabric8 - docker-maven-plugin - diff --git a/integrationtest/java/modules/orm-lucene/pom.xml b/integrationtest/java/modules/orm-lucene/pom.xml index 42099857889..e03186a34b0 100644 --- a/integrationtest/java/modules/orm-lucene/pom.xml +++ b/integrationtest/java/modules/orm-lucene/pom.xml @@ -72,10 +72,6 @@ - - io.fabric8 - docker-maven-plugin - diff --git a/integrationtest/java/modules/orm-outbox-polling-elasticsearch/pom.xml b/integrationtest/java/modules/orm-outbox-polling-elasticsearch/pom.xml index feb1563d6da..7ad416cd8a9 100644 --- a/integrationtest/java/modules/orm-outbox-polling-elasticsearch/pom.xml +++ b/integrationtest/java/modules/orm-outbox-polling-elasticsearch/pom.xml @@ -95,10 +95,6 @@ - - io.fabric8 - docker-maven-plugin - diff --git a/integrationtest/java/modules/pojo-standalone-elasticsearch/pom.xml b/integrationtest/java/modules/pojo-standalone-elasticsearch/pom.xml index e3af263297e..011fe6e0a7a 100644 --- a/integrationtest/java/modules/pojo-standalone-elasticsearch/pom.xml +++ b/integrationtest/java/modules/pojo-standalone-elasticsearch/pom.xml @@ -69,10 +69,6 @@ - - io.fabric8 - docker-maven-plugin - diff --git a/integrationtest/java/modules/pojo-standalone-lucene/pom.xml b/integrationtest/java/modules/pojo-standalone-lucene/pom.xml index dff13b01c0b..2e9bca3d4c1 100644 --- a/integrationtest/java/modules/pojo-standalone-lucene/pom.xml +++ b/integrationtest/java/modules/pojo-standalone-lucene/pom.xml @@ -64,10 +64,6 @@ - - io.fabric8 - docker-maven-plugin - diff --git a/integrationtest/mapper/orm-cdi/pom.xml b/integrationtest/mapper/orm-cdi/pom.xml index 6448c700cda..41a34ebd72e 100644 --- a/integrationtest/mapper/orm-cdi/pom.xml +++ b/integrationtest/mapper/orm-cdi/pom.xml @@ -68,11 +68,6 @@ - - - io.fabric8 - docker-maven-plugin - diff --git a/integrationtest/mapper/orm-envers/pom.xml b/integrationtest/mapper/orm-envers/pom.xml index a7d59523da2..ae0dd7d3b5c 100644 --- a/integrationtest/mapper/orm-envers/pom.xml +++ b/integrationtest/mapper/orm-envers/pom.xml @@ -58,11 +58,6 @@ - - - io.fabric8 - docker-maven-plugin - diff --git a/integrationtest/mapper/orm-jakarta-batch/pom.xml b/integrationtest/mapper/orm-jakarta-batch/pom.xml index 14687a9729e..15f5fc3c784 100644 --- a/integrationtest/mapper/orm-jakarta-batch/pom.xml +++ b/integrationtest/mapper/orm-jakarta-batch/pom.xml @@ -173,11 +173,6 @@ - - - io.fabric8 - docker-maven-plugin - diff --git a/integrationtest/mapper/orm-outbox-polling/pom.xml b/integrationtest/mapper/orm-outbox-polling/pom.xml index 0e5d5b5b732..678e0806ec0 100644 --- a/integrationtest/mapper/orm-outbox-polling/pom.xml +++ b/integrationtest/mapper/orm-outbox-polling/pom.xml @@ -99,10 +99,6 @@ - - io.fabric8 - docker-maven-plugin - diff --git a/integrationtest/mapper/orm-realbackend/pom.xml b/integrationtest/mapper/orm-realbackend/pom.xml index 5cd28d45422..bf3bbd6ebcb 100644 --- a/integrationtest/mapper/orm-realbackend/pom.xml +++ b/integrationtest/mapper/orm-realbackend/pom.xml @@ -138,10 +138,6 @@ - - io.fabric8 - docker-maven-plugin - diff --git a/integrationtest/mapper/orm-spring/pom.xml b/integrationtest/mapper/orm-spring/pom.xml index cb43f641a58..32ee44ea8c7 100644 --- a/integrationtest/mapper/orm-spring/pom.xml +++ b/integrationtest/mapper/orm-spring/pom.xml @@ -114,10 +114,6 @@ - - io.fabric8 - docker-maven-plugin - diff --git a/integrationtest/mapper/orm/pom.xml b/integrationtest/mapper/orm/pom.xml index 5f137c99b5a..3b9938201f0 100644 --- a/integrationtest/mapper/orm/pom.xml +++ b/integrationtest/mapper/orm/pom.xml @@ -65,11 +65,6 @@ - - - io.fabric8 - docker-maven-plugin - diff --git a/integrationtest/mapper/pojo-standalone-realbackend/pom.xml b/integrationtest/mapper/pojo-standalone-realbackend/pom.xml index 48963149289..a90d60ac77c 100644 --- a/integrationtest/mapper/pojo-standalone-realbackend/pom.xml +++ b/integrationtest/mapper/pojo-standalone-realbackend/pom.xml @@ -103,10 +103,6 @@ - - io.fabric8 - docker-maven-plugin - diff --git a/integrationtest/performance/backend/elasticsearch/pom.xml b/integrationtest/performance/backend/elasticsearch/pom.xml index 0b150dbcef0..cad2fbddb72 100644 --- a/integrationtest/performance/backend/elasticsearch/pom.xml +++ b/integrationtest/performance/backend/elasticsearch/pom.xml @@ -65,10 +65,6 @@ - - io.fabric8 - docker-maven-plugin - org.bsc.maven maven-processor-plugin diff --git a/integrationtest/showcase/library/pom.xml b/integrationtest/showcase/library/pom.xml index 4739a2f9d3a..392cffe94cc 100644 --- a/integrationtest/showcase/library/pom.xml +++ b/integrationtest/showcase/library/pom.xml @@ -178,10 +178,6 @@ - - io.fabric8 - docker-maven-plugin - diff --git a/integrationtest/v5migrationhelper/orm/pom.xml b/integrationtest/v5migrationhelper/orm/pom.xml index a2cef0402d2..93d0a086f72 100755 --- a/integrationtest/v5migrationhelper/orm/pom.xml +++ b/integrationtest/v5migrationhelper/orm/pom.xml @@ -108,11 +108,6 @@ - - - io.fabric8 - docker-maven-plugin - diff --git a/pom.xml b/pom.xml index 74f834901fb..b2ec77c7c9f 100644 --- a/pom.xml +++ b/pom.xml @@ -291,7 +291,6 @@ 9.6 0.8.11 2.0.8 - 0.43.4 1.1.0 3.10.0.2594 3.0.0 @@ -927,11 +926,6 @@ - - io.fabric8 - docker-maven-plugin - ${version.docker.maven.plugin} - org.jacoco jacoco-maven-plugin From 34377317c7fedaa0dde4e7b6fc40c44cb4e64403 Mon Sep 17 00:00:00 2001 From: marko-bekhta Date: Fri, 2 Dec 2022 15:28:37 +0100 Subject: [PATCH 04/20] HSEARCH-4674 Update groovy script to list containers --- build/parents/integrationtest/pom.xml | 80 +++++++++++++++++-- pom.xml | 49 ++++++------ .../mapper/orm/DatabaseContainer.java | 2 +- 3 files changed, 96 insertions(+), 35 deletions(-) diff --git a/build/parents/integrationtest/pom.xml b/build/parents/integrationtest/pom.xml index 7b93cc32a71..43851bcd319 100644 --- a/build/parents/integrationtest/pom.xml +++ b/build/parents/integrationtest/pom.xml @@ -65,6 +65,34 @@ + + + elasticsearch-run + + + + !test.elasticsearch.connection.uris + + + + ${test.elasticsearch.run.elastic.image.name} + ${test.elasticsearch.run.elastic.image.tag} + + + + + elasticsearch-do-not-run + + + + test.elasticsearch.connection.uris + + + + false + + + aws @@ -84,6 +112,20 @@ + + opensearch + + + test.elasticsearch.distribution + opensearch + + + + ${test.elasticsearch.run.opensearch.image.name} + ${test.elasticsearch.run.opensearch.image.tag} + + + @@ -100,7 +142,7 @@ com.h2database h2 - -Dorg.hibernate.search.integrationtest.orn.database=h2 + -Dorg.hibernate.search.integrationtest.orm.database=h2 @@ -112,8 +154,12 @@ org.postgresql postgresql + true + ${test.database.run.postgres.image.name} + ${test.database.run.postgres.image.tag} - -Dorg.hibernate.search.integrationtest.orn.database=postgresql + -Dorg.hibernate.search.integrationtest.orm.database=postgresql + -Dorg.hibernate.search.integrationtest.orm.database.image.name=${test.database.run.image.tag} @@ -125,8 +171,11 @@ org.mariadb.jdbc mariadb-java-client + true + ${test.database.run.mariadb.image.name} + ${test.database.run.mariadb.image.tag} - -Dorg.hibernate.search.integrationtest.orn.database=mariadb + -Dorg.hibernate.search.integrationtest.orm.database=mariadb @@ -138,8 +187,11 @@ com.mysql mysql-connector-j + true + ${test.database.run.mysql.image.name} + ${test.database.run.mysql.image.tag} - -Dorg.hibernate.search.integrationtest.orn.database=mysql + -Dorg.hibernate.search.integrationtest.orm.database=mysql @@ -151,8 +203,11 @@ com.ibm.db2 jcc + true + ${test.database.run.db2.image.name} + ${test.database.run.db2.image.tag} - -Dorg.hibernate.search.integrationtest.orn.database=db2 + -Dorg.hibernate.search.integrationtest.orm.database=db2 @@ -164,8 +219,11 @@ com.oracle.database.jdbc ojdbc11 + true + ${test.database.run.oracle.image.name} + ${test.database.run.oracle.image.tag} - -Dorg.hibernate.search.integrationtest.orn.database=oracle + -Dorg.hibernate.search.integrationtest.orm.database=oracle -Doracle.jdbc.timezoneAsRegion=false @@ -179,8 +237,11 @@ com.microsoft.sqlserver mssql-jdbc + true + ${test.database.run.mssql.image.name} + ${test.database.run.mssql.image.tag} - -Dorg.hibernate.search.integrationtest.orn.database=mssql + -Dorg.hibernate.search.integrationtest.orm.database=mssql @@ -193,8 +254,11 @@ org.postgresql postgresql + true + ${test.database.run.cockroachdb.image.name} + ${test.database.run.cockroachdb.image.tag} - -Dorg.hibernate.search.integrationtest.orn.database=cockroachdb + -Dorg.hibernate.search.integrationtest.orm.database=cockroachdb diff --git a/pom.xml b/pom.xml index b2ec77c7c9f..3f0d650d038 100644 --- a/pom.xml +++ b/pom.xml @@ -387,6 +387,10 @@ docker.io/opensearchproject/opensearch ${test.elasticsearch.version} + + testcontainers/ryuk + 0.5.1 + true @@ -425,6 +429,17 @@ docker.io/cockroachdb/cockroach v23.1.4 + + false + h2 + + + true + + + + -Xmx512m -Xms128m @@ -980,33 +995,15 @@ -->