HSEARCH-4674 Use test containers

.github/dependabot.yml

@@ -4,6 +4,16 @@
 # https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
 
 version: 2
+registries:
+  dockerhub:
+    type: docker-registry
+    url: https://registry.hub.docker.com
+    username: ${{secrets.DOCKERHUB_USERNAME}}
+    password: ${{secrets.DOCKERHUB_TOKEN}}
+    replaces-base: true
+  ibm:
+    type: docker-registry
+    url: https://icr.io
 updates:
   - package-ecosystem: "maven"
     directory: "/"
@@ -99,3 +109,56 @@ updates:
       # See https://github.com/dependabot/dependabot-core/issues/4028
       # And, additionally, it is only for performance testing.
       - dependency-name: "org.bsc.maven:maven-processor-plugin"
+  - package-ecosystem: "docker"
+    registries:
+      - dockerhub
+      - ibm
+    # For dependabot to find Docker files they all should be in the same directory.
+    # Dependabot is picking any files that has `dockerfile` in them.
+    #
+    # This includes only Ryuk container, hence we want it to be checked/updated at the same time when our regular java dependencies are updated
+    directory: "/build/container/"
+    schedule:
+      interval: "weekly"
+      day: "tuesday"
+    assignees: ["yrodiere"]
+    groups:
+      # This group combines all build containers dependencies.
+      build-containers:
+        patterns:
+          # Include all:
+          - "*"
+  - package-ecosystem: "docker"
+    registries:
+      - dockerhub
+    # For dependabot to find Docker files they all should be in the same directory.
+    # Dependabot is picking any files that has `dockerfile` in them.
+    #
+    # This will only include database containers, hence we check for updates only once a month:
+    directory: "/build/container/database"
+    schedule:
+      interval: "monthly"
+    assignees: ["yrodiere"]
+    groups:
+      # This group combines all database containers dependencies.
+      database-containers:
+        patterns:
+          # Include all:
+          - "*"
+  - package-ecosystem: "docker"
+    registries:
+      - dockerhub
+    # For dependabot to find Docker files they all should be in the same directory.
+    # Dependabot is picking any files that has `dockerfile` in them.
+    directory: "/build/container/search-backend"
+    schedule:
+      interval: "daily"
+    assignees: ["yrodiere"]
+    groups:
+      # This group combines all search backends containers dependencies.
+      # We will still need to create specific tickets to address these updates and do some code changes,
+      # so we just want to get a notification from the bot that the new versions are available.
+      search-backend-containers:
+        patterns:
+          # Include all:
+          - "*"

.github/workflows/build.yml

@@ -33,6 +33,8 @@ on:
       # we'll reject such dependant PRs and send a PR ourselves.
       - '!dependabot/**'
       - 'dependabot/maven/build-dependencies-**'
+      - 'dependabot/docker/build-containers-**'
+      - 'dependabot/docker/database-containers-**'
 
 concurrency:
   group: "workflow = ${{ github.workflow }}, ref = ${{ github.event.ref }}, pr = ${{ github.event.pull_request.id }}"
@@ -60,24 +62,21 @@ jobs:
               version: 17
             },
             maven: {
-              args: ''
+              # Since we only start an Elasticsearch container on Linux we add the profile to enable container reuse here:
+              args: '-Pci-build'
             }
           }
           # We can't start Linux containers on GitHub Actions' Windows VMs,
           # so we can't run Elasticsearch tests.
           # See https://github.com/actions/runner-images/issues/1143#issuecomment-972929995
-          # For some reason docker-maven-plugin will error out with "All pipe instances are busy"
-          # on GitHub Actions' Windows VMs (when using Bash?),
-          # so we also explicitly disable docker-maven-plugin.
-          # See https://github.com/fabric8io/docker-maven-plugin/issues/548#issuecomment-255477600
           - {
             name: "Windows JDK 17",
             runs-on: 'windows-latest',
             java: {
               version: 17
             },
             maven: {
-              args: '-Dtest.elasticsearch.skip=true -Dtest.containers.run.skip=true'
+              args: '-Dtest.elasticsearch.skip=true'
             }
           }
     steps:

CONTRIBUTING.md

@@ -223,8 +223,8 @@ and run unit tests and integration tests.
 ```
 
 Note: on Windows, you will need a Docker install able to run Linux containers.
-If you don't have that, you can skip the Elasticsearch tests and all container startups:
-`./mvnw clean install -Dtest.elasticsearch.skip=true -Dtest.containers.run.skip=true`.
+If you don't have that, you can skip the Elasticsearch tests and only run tests against H2 database (without using DB containers):
+`./mvnw clean install -Dtest.elasticsearch.skip=true`.
 
 Note: the produced JARs are compatible with Java 8 and later,
 regardless of the JDK used to build Hibernate Search.
@@ -346,12 +346,6 @@ you can skip all Elasticsearch tests (and thus the Elasticsearch container start
 ./mvnw clean install -Dtest.elasticsearch.skip=true
 ```
 
-On Windows, you might need to also skip all docker container features:
-
-```bash
-./mvnw clean install -Dtest.elasticsearch.skip=true -Dtest.containers.run.skip=true
-```
-
 Alternatively, you can prevent the build from launching an Elasticsearch server automatically
 and run Elasticsearch-related tests against your own server using the
 `test.elasticsearch.connection.uris` property:
@@ -403,6 +397,22 @@ Or more simply, if your AWS credentials are already stored in `~/.aws/credential
         -Dtest.elasticsearch.connection.aws.region=<Your AWS region ID>
 ```
 
+### Testcontainers
+
+Hibernate Search uses Testcontainers in its integration tests. 
+By default, testcontainers are not reusable, i.e. a container is started at the beginning of an executed test suite,
+and then stopped; each execution of failsafe plugin will start/stop their own containers as well as each test module.
+With [reusable testcontainers](https://java.testcontainers.org/features/reuse/) required containers will start as needed
+but will not be terminated, staying available between different failsafe executions and test modules
+and even after the maven build is finished.
+Reusable containers must be stopped manually, if needed (e.g. with `docker stop <container name>`
+or `docker kill <container name>`).
+
+There are a few ways to enable reusable testcontainers:
+
+1. Enable reusable testcontainers in `~/.testcontainers.properties`, by adding `testcontainers.reuse.enable=true`
+2. Build with `ci-build` Maven profile: `./mvnw clean install -Pci-build`
+
 ### JQAssistant
 
 You can request static analysis and sanity checks with the `jqassistant` profile.

Jenkinsfile

@@ -431,6 +431,7 @@ stage('Default build') {
 					${commonMavenArgs} \
 					-DskipSurefireTests \
 					-Pskip-checks \
+					-Pci-build \
 					-Pci-rebuild \
 					${incrementalBuild ? """ \
 							-Dincremental \
@@ -540,30 +541,6 @@ stage('Non-default environments') {
 					'''
 					String mavenDockerArgs = ""
 					def startedContainers = false
-					// DB2 setup is super slow (~5 to 15 minutes).
-					// We can't afford to do that once per module,
-					// so we start DB2 here, once and for all.
-					if ( buildEnv.dbName == 'db2' ) {
-						// Prevent the actual build from starting the DB container
-						mavenBuildAdditionalArgs += " -Dtest.database.run.db2.skip=true"
-						// Pick a module that doesn't normally execute the docker-maven-plugin,
-						// but that has the configuration necessary to start the DB container:
-						// that way, the maven-docker-plugin won't try to remove our container
-						// when we execute maven another time to run the tests.
-						// (This works because maven-docker-plugin filters containers to stop
-						// based on the Maven GAV coordinates of the Maven project that started that container,
-						// which are attached to the container thanks to a container label).
-						mavenDockerArgs = """ \
-								-pl build/parents/integrationtest \
-								-P$buildEnv.mavenProfile \
-								-Dtest.database.run.db2.skip=false \
-						"""
-						// Cleanup just in case some containers were left over from a previous build.
-						sh "mvn docker:stop $mavenDockerArgs"
-						pullContainerImages mavenDockerArgs
-						sh "mvn docker:start $mavenDockerArgs"
-						startedContainers = true
-					}
 					try {
 						mavenNonDefaultBuild buildEnv, """ \
 								-Pdist \
@@ -962,7 +939,7 @@ void mavenNonDefaultBuild(BuildEnvironment buildEnv, String args, List<String> a
 	def testSuffix = buildEnv.tag.replaceAll('[^a-zA-Z0-9_\\-+]+', '_')
 
 	sh """ \
-			mvn clean install -Dsurefire.environment=$testSuffix \
+			mvn clean install -Pci-build -Dsurefire.environment=$testSuffix \
 					${toTestJdkArg(buildEnv)} \
 					--fail-at-end \
 					$args \

build/container/database/cockroachdb.Dockerfile

@@ -0,0 +1,3 @@
+# CockroachDB
+# See https://hub.docker.com/r/cockroachdb/cockroach/tags
+FROM docker.io/cockroachdb/cockroach:v23.1.11

build/container/database/db2.Dockerfile

@@ -0,0 +1,3 @@
+# DB2
+# See https://icr.io/v2/db2_community/db2/tags/list
+FROM icr.io/db2_community/db2:11.5.8.0

build/container/database/mariadb.Dockerfile

@@ -0,0 +1,3 @@
+# MariaDB
+# See https://hub.docker.com/_/mariadb/tags
+FROM docker.io/mariadb:11.1.2

build/container/database/mssql.Dockerfile

@@ -0,0 +1,3 @@
+# MS SQL Server
+# See https://hub.docker.com/_/microsoft-mssql-server
+FROM docker.io/mcr.microsoft.com/mssql/server:2022-CU9-ubuntu-20.04

build/container/database/mysql.Dockerfile

@@ -0,0 +1,3 @@
+# MySQL
+# See https://hub.docker.com/_/mysql/tags
+FROM docker.io/mysql:8.1.0

build/container/database/oracle.Dockerfile

@@ -0,0 +1,3 @@
+# Oracle
+# See https://hub.docker.com/r/gvenzl/oracle-free/tags
+FROM docker.io/gvenzl/oracle-free:23.3-slim-faststart

build/container/database/postgresql.Dockerfile

@@ -0,0 +1,3 @@
+# PostgreSQL
+# See https://hub.docker.com/_/postgres
+FROM docker.io/postgres:16.0