HSEARCH-4674 Rework Elasticsearch/OpenSearch container integration

hibernate · Oct 24, 2023 · 0fd2fe8 · 0fd2fe8
1 parent 75cd843
commit 0fd2fe8
Show file tree

Hide file tree

Showing 7 changed files with 84 additions and 55 deletions.
diff --git a/build/container/amazon-opensearch-serverless.Dockerfile b/build/container/amazon-opensearch-serverless.Dockerfile
@@ -0,0 +1,3 @@
+# OpenSearch
+# See https://hub.docker.com/r/opensearchproject/opensearch/tags
+FROM docker.io/opensearchproject/opensearch:2.11.0
diff --git a/build/container/elasticsearch.Dockerfile → build/container/elastic.Dockerfile b/build/container/elasticsearch.Dockerfile → build/container/elastic.Dockerfile
diff --git a/build/parents/build/pom.xml b/build/parents/build/pom.xml
@@ -218,7 +218,6 @@
              because we will be able to change the configuration without re-compiling the configuration file
              (which is located in a dependency of the integration tests modules).
          -->
-        <test.elasticsearch.distribution>elastic</test.elasticsearch.distribution>
         <test.elasticsearch.connection.uris></test.elasticsearch.connection.uris>
         <test.elasticsearch.connection.username></test.elasticsearch.connection.username>
         <test.elasticsearch.connection.password></test.elasticsearch.connection.password>
@@ -1196,10 +1195,9 @@
                             <org.hibernate.search.version>${project.version}</org.hibernate.search.version>
                             <org.hibernate.search.enable_performance_tests>${test.performance.enable}</org.hibernate.search.enable_performance_tests>
                             <org.hibernate.search.integrationtest.backend.elasticsearch.distribution>${test.elasticsearch.distribution}</org.hibernate.search.integrationtest.backend.elasticsearch.distribution>
-                            <org.hibernate.search.integrationtest.backend.elasticsearch.name>${test.searchengine.run.image.name}</org.hibernate.search.integrationtest.backend.elasticsearch.name>
-                            <org.hibernate.search.integrationtest.backend.elasticsearch.tag>${test.searchengine.run.image.tag}</org.hibernate.search.integrationtest.backend.elasticsearch.tag>
+                            <org.hibernate.search.integrationtest.backend.elasticsearch.version>${test.elasticsearch.version}</org.hibernate.search.integrationtest.backend.elasticsearch.version>
                             <org.hibernate.search.integrationtest.orm.database.kind>${test.database.run.kind}</org.hibernate.search.integrationtest.orm.database.kind>
-                            <org.hibernate.search.integrationtest.orm.project.root.directory>${rootProject.directory}</org.hibernate.search.integrationtest.orm.project.root.directory>
+                            <org.hibernate.search.integrationtest.project.root.directory>${rootProject.directory}</org.hibernate.search.integrationtest.project.root.directory>
                         </systemPropertyVariables>
                     </configuration>
                 </plugin>

diff --git a/build/parents/integrationtest/pom.xml b/build/parents/integrationtest/pom.xml
@@ -96,31 +96,32 @@
         <!--
              To run tests against a different version of Elasticsearch, see CONTRIBUTING.md.
           -->
-        <!-- Profile enabled when an Elasticsearch instance must be pulled -->
+        <!-- Profile enabled when an Elasticsearch/OpenSearch container must NOT be started as a test container, but when an URL to an "external" service is supplied! -->
         <profile>
-            <id>elasticsearch-run</id>
+            <id>search-container-do-not-start</id>
             <activation>
-                <!-- Activate by default, i.e. if test.elasticsearch.connection.uris has not been defined explicitly -->
+                <!-- Activate if test.elasticsearch.connection.uris has been defined explicitly -->
                 <property>
-                    <name>!test.elasticsearch.connection.uris</name>
+                    <name>test.elasticsearch.connection.uris</name>
                 </property>
             </activation>
             <properties>
-                <test.searchengine.run.image.name>${test.elasticsearch.run.elastic.image.name}</test.searchengine.run.image.name>
-                <test.searchengine.run.image.tag>${test.elasticsearch.run.elastic.image.tag}</test.searchengine.run.image.tag>
+                <test.searchengine.run.image.pull>false</test.searchengine.run.image.pull>
             </properties>
         </profile>
-        <!-- Profile enabled when an Elasticsearch instance must NOT be run by Maven -->
+
+        <!-- Profile enabled when Elasticsearch tests are skipped -->
         <profile>
-            <id>elasticsearch-do-not-run</id>
+            <id>elasticsearch-test-skip</id>
             <activation>
                 <!-- Activate if test.elasticsearch.connection.uris has been defined explicitly -->
                 <property>
-                    <name>test.elasticsearch.connection.uris</name>
+                    <name>test.elasticsearch.skip</name>
+                    <value>true</value>
                 </property>
             </activation>
             <properties>
-                <test.searchengine.run.image.pull>false</test.searchengine.run.image.pull>
+                <test.elasticsearch.run.skip.forRelevantModules>true</test.elasticsearch.run.skip.forRelevantModules>
             </properties>
         </profile>
 
@@ -143,20 +144,6 @@
             </properties>
         </profile>
 
-        <profile>
-            <id>opensearch</id>
-            <activation>
-                <property>
-                    <name>test.elasticsearch.distribution</name>
-                    <value>opensearch</value>
-                </property>
-            </activation>
-            <properties>
-                <test.searchengine.run.image.name>${test.elasticsearch.run.opensearch.image.name}</test.searchengine.run.image.name>
-                <test.searchengine.run.image.tag>${test.elasticsearch.run.opensearch.image.tag}</test.searchengine.run.image.tag>
-            </properties>
-        </profile>
-
         <!-- =============================== -->
         <!-- Database profiles               -->
         <!-- =============================== -->

diff --git a/pom.xml b/pom.xml
@@ -377,24 +377,14 @@
         <!-- Container images for various integration tests -->
         <!-- The latest version of Elasticsearch tested against by default -->
         <version.org.elasticsearch.latest>8.10.4</version.org.elasticsearch.latest>
-        <test.elasticsearch.version>${version.org.elasticsearch.latest}</test.elasticsearch.version>
-
-        <test.elasticsearch.run.elastic.skip>true</test.elasticsearch.run.elastic.skip>
-        <test.elasticsearch.run.elastic.image.name>docker.io/elastic/elasticsearch</test.elasticsearch.run.elastic.image.name>
-        <test.elasticsearch.run.elastic.image.tag>${test.elasticsearch.version}</test.elasticsearch.run.elastic.image.tag>
-
-        <test.elasticsearch.run.opensearch.skip>true</test.elasticsearch.run.opensearch.skip>
-        <test.elasticsearch.run.opensearch.image.name>docker.io/opensearchproject/opensearch</test.elasticsearch.run.opensearch.image.name>
-        <test.elasticsearch.run.opensearch.image.tag>${test.elasticsearch.version}</test.elasticsearch.run.opensearch.image.tag>
+        <test.elasticsearch.version></test.elasticsearch.version>
+        <test.elasticsearch.distribution>elastic</test.elasticsearch.distribution>
 
         <!-- Docker images to be pulled. To be redefined in specific profiles of Elasticsearch/OpenSearch \
              or database specific ones. -->
         <test.database.run.kind>h2</test.database.run.kind>
 
         <test.searchengine.run.image.pull>true</test.searchengine.run.image.pull>
-        <test.searchengine.run.image.name></test.searchengine.run.image.name>
-        <test.searchengine.run.image.tag></test.searchengine.run.image.tag>
-
 
         <!-- Set empty default values to avoid Maven leaving property references (${...}) when it doesn't find a value -->
 
@@ -967,8 +957,14 @@
                                     if ( isNotBlankString( '${test.database.run.kind}' ) && '${test.database.run.kind}' != 'h2') {
                                         images += parseImage( './build/container/${test.database.run.kind}.Dockerfile' )
                                     }
-                                    if ( isTrueString( '${test.searchengine.run.image.pull}' ) && isNotBlankString( '${test.searchengine.run.image.name}' ) ) {
-                                        images += '${test.searchengine.run.image.name}:${test.searchengine.run.image.tag}'
+                                    if ( isTrueString( '${test.searchengine.run.image.pull}' ) ) {
+                                        def fromFile = parseImage( './build/container/${test.elasticsearch.distribution}.Dockerfile')
+                                        def suppliedVersion = '${test.elasticsearch.version}'
+                                        if (isNotBlankString(suppliedVersion)) {
+                                            images += fromFile.substring(0, fromFile.lastIndexOf(':')+1) + suppliedVersion
+                                        } else {
+                                            images += fromFile
+                                        }
                                     }
                                     // Exclude images from non-dockerhub repositories
                                     new File('${containerImagesListFile}').append( images.findAll{ref -> !(ref ==~ /^(?!((.*\.)?docker\.io))[\w\d\.]+\/[\w\d\.]+\/[\w\d\.]+(:.+)?$/)}.join('\n') + '\n' )

diff --git a/...org/hibernate/search/util/impl/integrationtest/backend/elasticsearch/SearchContainer.java b/...org/hibernate/search/util/impl/integrationtest/backend/elasticsearch/SearchContainer.java
@@ -6,7 +6,12 @@
  */
 package org.hibernate.search.util.impl.integrationtest.backend.elasticsearch;
 
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.time.Duration;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import org.hibernate.search.backend.elasticsearch.ElasticsearchDistributionName;
 import org.hibernate.search.backend.elasticsearch.ElasticsearchVersion;
@@ -23,12 +28,30 @@ private SearchContainer() {
 	private static final GenericContainer<?> SEARCH_CONTAINER;
 
 	static {
-		String name = System.getProperty( "org.hibernate.search.integrationtest.backend.elasticsearch.name", "" );
-		String tag = System.getProperty( "org.hibernate.search.integrationtest.backend.elasticsearch.tag" );
-
-		SEARCH_CONTAINER = name.contains( "elastic" )
-				? elasticsearch( name, tag, ElasticsearchVersion.of( ElasticsearchDistributionName.ELASTIC, tag ) )
-				: opensearch( name, tag );
+		ElasticsearchDistributionName distributionName = ElasticsearchDistributionName
+				.of( System.getProperty( "org.hibernate.search.integrationtest.backend.elasticsearch.distribution", "" ) );
+		String tag = System.getProperty( "org.hibernate.search.integrationtest.backend.elasticsearch.version" );
+		Path root = Path.of( System.getProperty( "org.hibernate.search.integrationtest.project.root.directory", "" ) );
+
+		try {
+			DockerImageName dockerImageName = parseDockerImageName( root.resolve( "build" ).resolve( "container" )
+					.resolve( distributionName.externalRepresentation() + ".Dockerfile" ), tag );
+			switch ( distributionName ) {
+				case ELASTIC:
+					SEARCH_CONTAINER = elasticsearch( dockerImageName );
+					break;
+				case OPENSEARCH:
+				case AMAZON_OPENSEARCH_SERVERLESS:
+					SEARCH_CONTAINER = opensearch( dockerImageName );
+					break;
+				default:
+					throw new IllegalStateException( "Unknown distribution " + distributionName );
+			}
+		}
+		catch (IOException e) {
+			throw new IllegalStateException(
+					"Unable to initialize a Search Engine container [" + distributionName + ", " + tag + ", " + root + "]", e );
+		}
 	}
 
 	public static int mappedPort(int port) {
@@ -60,8 +83,8 @@ private static void startIfNeeded() {
 		}
 	}
 
-	private static GenericContainer<?> elasticsearch(String name, String tag, ElasticsearchVersion version) {
-		GenericContainer<?> container = common( name, tag )
+	private static GenericContainer<?> elasticsearch(DockerImageName dockerImageName) {
+		GenericContainer<?> container = common( dockerImageName )
 				.withEnv( "logger.level", "WARN" )
 				.withEnv( "discovery.type", "single-node" )
 				// Older images require HTTP authentication for all requests;
@@ -79,6 +102,9 @@ private static GenericContainer<?> elasticsearch(String name, String tag, Elasti
 				// See https://www.elastic.co/guide/en/elasticsearch/reference/7.17/modules-cluster.html#disk-based-shard-allocation
 				.withEnv( "cluster.routing.allocation.disk.threshold_enabled", "false" );
 
+		ElasticsearchVersion version =
+				ElasticsearchVersion.of( ElasticsearchDistributionName.ELASTIC, dockerImageName.getVersionPart() );
+
 		if ( version.majorOptional().orElse( Integer.MIN_VALUE ) == 8 ) {
 			if ( version.minor().orElse( Integer.MAX_VALUE ) > 7 ) {
 				container.withEnv( "cluster.deprecation_indexing.enabled", "false" )
@@ -112,8 +138,8 @@ private static GenericContainer<?> elasticsearch(String name, String tag, Elasti
 		return container;
 	}
 
-	private static GenericContainer<?> opensearch(String name, String tag) {
-		return common( name, tag )
+	private static GenericContainer<?> opensearch(DockerImageName dockerImageName) {
+		return common( dockerImageName )
 				.withEnv( "logger.level", "WARN" )
 				.withEnv( "discovery.type", "single-node" )
 				// Prevent swapping
@@ -137,11 +163,30 @@ private static GenericContainer<?> opensearch(String name, String tag) {
 	 * this resource in the end.
 	 */
 	@SuppressWarnings("resource")
-	private static GenericContainer<?> common(String image, String tag) {
-		return new GenericContainer<>( DockerImageName.parse( image ).withTag( tag ) )
+	private static GenericContainer<?> common(DockerImageName dockerImageName) {
+		return new GenericContainer<>( dockerImageName )
 				.withExposedPorts( 9200, 9300 )
 				.waitingFor( new HttpWaitStrategy().forPort( 9200 ).forStatusCode( 200 ) )
 				.withStartupTimeout( Duration.ofMinutes( 5 ) )
 				.withReuse( true );
 	}
+
+
+	private static DockerImageName parseDockerImageName(Path dockerfile, String tag) throws IOException {
+		Pattern DOCKERFILE_FROM_LINE_PATTERN = Pattern.compile( "FROM (.+)" );
+
+		DockerImageName dockerImageName = Files.lines( dockerfile )
+				.map( DOCKERFILE_FROM_LINE_PATTERN::matcher )
+				.filter( Matcher::matches )
+				.map( m -> m.group( 1 ) )
+				.map( DockerImageName::parse )
+				.findAny().orElseThrow( () -> new IllegalStateException(
+						"Dockerfile " + dockerfile + " has unexpected structure. It *must* contain a single FROM line." ) );
+
+		if ( tag != null && !tag.trim().isEmpty() ) {
+			return dockerImageName.withTag( tag );
+		}
+
+		return dockerImageName;
+	}
 }
diff --git a/...ain/java/org/hibernate/search/util/impl/integrationtest/mapper/orm/DatabaseContainer.java b/...ain/java/org/hibernate/search/util/impl/integrationtest/mapper/orm/DatabaseContainer.java
@@ -35,7 +35,7 @@ private DatabaseContainer() {
 
 	static {
 		String name = System.getProperty( "org.hibernate.search.integrationtest.orm.database.kind", "" );
-		Path root = Path.of( System.getProperty( "org.hibernate.search.integrationtest.orm.project.root.directory", "" ) );
+		Path root = Path.of( System.getProperty( "org.hibernate.search.integrationtest.project.root.directory", "" ) );
 		DATABASE = SupportedDatabase.from( name );
 
 		DATABASE_CONTAINER = DATABASE.container(