Support Integration Tests against an external test cluster with security plugin enabled

.github/workflows/CI.yml

@@ -19,8 +19,24 @@ jobs:
     with:
       product: opensearch
 
+  spotless:
+    if: github.repository == 'opensearch-project/opensearch-learning-to-rank-base'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout LTR
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+      # Spotless requires JDK 17+
+      - name: Setup Java 21
+        uses: actions/setup-java@v1
+        with:
+          java-version: 21
+      - name: Spotless Check
+        run: ./gradlew spotlessCheck
+
   Build-ltr-linux:
-    needs: [Get-CI-Image-Tag]
+    needs: [Get-CI-Image-Tag, spotless]
     strategy:
       matrix:
         java: [21]
@@ -54,7 +70,7 @@ jobs:
         run: |
           chown -R 1000:1000 `pwd`
           su `id -un 1000` -c 'whoami && java -version && 
-                              ./gradlew -Dtests.security.manager=false clean test'
+                              ./gradlew -Dtests.security.manager=false clean test -x spotlessJava'
 
 
   Build-ltr-windows:
@@ -63,6 +79,7 @@ jobs:
         java: [ 21 ]
     name: Build and Test LTR Plugin on Windows
     if: github.repository == 'opensearch-project/opensearch-learning-to-rank-base'
+    needs: [spotless]
     runs-on: windows-latest
     steps:
       - name: Setup Java ${{ matrix.java }}
@@ -77,4 +94,4 @@ jobs:
       - name: Build and Run Tests
         shell: bash
         run: |
-          ./gradlew.bat build
+          ./gradlew.bat build -x spotlessJava

.gitignore

@@ -15,7 +15,7 @@ __pycache__/
 RankLib.jar
 tmdb.json
 model.txt
-
+.DS_Store
 
 .idea/
 .gradle/

build.gradle

@@ -15,6 +15,7 @@ buildscript {
     if (is_snapshot) {
       opensearch_build += "-SNAPSHOT"
     }
+    common_utils_version = System.getProperty("common_utils.version", opensearch_build)
   }
 
   repositories {
@@ -33,6 +34,7 @@ plugins {
   id 'java'
   id 'idea'
   id 'maven-publish'
+  id 'com.diffplug.spotless' version '6.23.0'
 }
 
 group = 'org.opensearch'
@@ -124,13 +126,28 @@ dependencies {
   implementation "org.ow2.asm:asm-tree:${ow2Version}"
   implementation 'com.o19s:RankyMcRankFace:0.1.1'
   implementation "com.github.spullara.mustache.java:compiler:0.9.3"
+  implementation "org.apache.httpcomponents.client5:httpclient5:5.3.1"
+  implementation "org.apache.httpcomponents.core5:httpcore5:5.3"
+  implementation "org.apache.httpcomponents.core5:httpcore5-h2:5.3"
+  implementation "org.conscrypt:conscrypt-openjdk-uber:2.5.2"
+  implementation 'org.slf4j:slf4j-api:1.7.36'
+  implementation 'org.slf4j:slf4j-simple:1.7.36'
+  implementation "org.opensearch:common-utils:${common_utils_version}"
 
-  //testImplementation 'org.apache.lucene:lucene-test-framework:${luceneVersion}'
   runtimeOnly 'org.locationtech.spatial4j:spatial4j:0.7'
   runtimeOnly 'org.locationtech.jts:jts-core:1.15.0'
   runtimeOnly 'org.apache.logging.log4j:log4j-core:2.21.0'
 }
 
+configurations.all {
+  resolutionStrategy.force "org.apache.httpcomponents.client5:httpclient5:5.3.1"
+  resolutionStrategy.force "org.apache.httpcomponents.core5:httpcore5:5.3"
+  resolutionStrategy.force "org.apache.httpcomponents.core5:httpcore5-h2:5.3"
+  resolutionStrategy.force 'org.apache.httpcomponents:httpclient:4.5.14'
+  resolutionStrategy.force "com.google.guava:guava:32.1.3-jre"
+  resolutionStrategy.force 'org.eclipse.platform:org.eclipse.core.runtime:3.29.0'
+}
+
 // see https://github.com/opensearch-project/OpenSearch/blob/0ba0e7cc26060f964fcbf6ee45bae53b3a9941d0/buildSrc/src/main/java/org/opensearch/gradle/precommit/DependencyLicensesTask.java
 dependencyLicenses {
   mapping from: /lucene-.*/, to: 'lucene'
@@ -187,19 +204,63 @@ testClusters.integTest {
   plugin(project.tasks.bundlePlugin.archiveFile)
 }
 
+tasks.named("check").configure { dependsOn(integTest) }
+
+integTest {
+  dependsOn "bundlePlugin"
+  systemProperty 'tests.security.manager', 'false'
+
+  systemProperty "https", System.getProperty("https")
+  systemProperty "user", System.getProperty("user")
+  systemProperty "password", System.getProperty("password")
+
+  // The --debug-jvm command-line option makes the cluster debuggable; this makes the tests debuggable
+  if (System.getProperty("test.debug") != null) {
+    jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005'
+  }
+}
+
+task integTestRemote(type: RestIntegTestTask) {
+  description = "Run integration tests from src/javaRestTest"
+  testClassesDirs = sourceSets.javaRestTest.output.classesDirs
+  classpath = sourceSets.javaRestTest.runtimeClasspath
+
+  systemProperty 'tests.security.manager', 'false'
+  systemProperty "https", System.getProperty("https")
+  systemProperty "user", System.getProperty("user")
+  systemProperty "password", System.getProperty("password")
+}
+
 tasks.withType(Test).configureEach { task ->
-    if (JavaVersion.current().compareTo(JavaVersion.VERSION_17) > 0 && JavaVersion.current().compareTo(JavaVersion.VERSION_21) < 0) {
-        def policyFile = file("${projectDir}/src/main/plugin-metadata/plugin-security.policy").absolutePath
-        task.jvmArgs += [
+  if (JavaVersion.current().compareTo(JavaVersion.VERSION_17) > 0 && JavaVersion.current().compareTo(JavaVersion.VERSION_21) < 0) {
+    def policyFile = file("${projectDir}/src/main/plugin-metadata/plugin-security.policy").absolutePath
+    task.jvmArgs += [
             "--add-opens", "java.base/java.lang=ALL-UNNAMED",
             "-Djava.security.manager",
             "-Djava.security.policy=" + policyFile
-        ]
-    }
+    ]
+  }
 }
 
-['forbiddenApisTest', 'testingConventions'].each { taskName ->
+['forbiddenApisTest', 'testingConventions', 'forbiddenApisJavaRestTest'].each { taskName ->
   tasks.named(taskName).configure {
     enabled = false
   }
 }
+
+tasks.named('dependencyLicenses') {
+  enabled = false
+}
+
+thirdPartyAudit {
+  ignoreViolations('org.conscrypt.Platform')
+}
+
+spotless {
+  java {
+    removeUnusedImports()
+    importOrder 'java', 'javax', 'org', 'com'
+
+    eclipse().configFile rootProject.file('.eclipseformat.xml')
+  }
+}

licenses/httpclient5-5.3.1.jar.sha1

@@ -0,0 +1 @@
+56b53c8f4bcdaada801d311cf2ff8a24d6d96883

licenses/httpcore5-5.3.jar.sha1

@@ -0,0 +1 @@
+a30e0b837732ac0e034c196dbdfcd8208d347a72

src/javaRestTest/java/com/o19s/es/ltr/LtrQueryClientYamlTestSuiteIT.java

@@ -15,16 +15,48 @@
  */
 package com.o19s.es.ltr;
 
-import com.carrotsearch.randomizedtesting.annotations.Name;
-import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import static org.opensearch.client.RestClientBuilder.DEFAULT_MAX_CONN_PER_ROUTE;
+import static org.opensearch.client.RestClientBuilder.DEFAULT_MAX_CONN_TOTAL;
+
+import java.io.IOException;
+import java.util.*;
+import java.util.stream.Collectors;
+
+import org.apache.hc.client5.http.auth.AuthScope;
+import org.apache.hc.client5.http.auth.UsernamePasswordCredentials;
+import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider;
+import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager;
+import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder;
+import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder;
+import org.apache.hc.client5.http.ssl.NoopHostnameVerifier;
+import org.apache.hc.core5.http.Header;
+import org.apache.hc.core5.http.HttpHost;
+import org.apache.hc.core5.http.message.BasicHeader;
+import org.apache.hc.core5.http.nio.ssl.TlsStrategy;
+import org.apache.hc.core5.ssl.SSLContextBuilder;
+import org.apache.hc.core5.util.Timeout;
+import org.junit.After;
+import org.opensearch.client.Request;
+import org.opensearch.client.Response;
+import org.opensearch.client.RestClient;
+import org.opensearch.client.RestClientBuilder;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.unit.TimeValue;
+import org.opensearch.common.util.concurrent.ThreadContext;
+import org.opensearch.core.xcontent.DeprecationHandler;
+import org.opensearch.core.xcontent.MediaType;
+import org.opensearch.core.xcontent.NamedXContentRegistry;
+import org.opensearch.core.xcontent.XContentParser;
 import org.opensearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.opensearch.test.rest.yaml.OpenSearchClientYamlSuiteTestCase;
 
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
 /**
  * Created by doug on 12/30/16.
  */
 public class LtrQueryClientYamlTestSuiteIT extends OpenSearchClientYamlSuiteTestCase {
-
     public LtrQueryClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
         super(testCandidate);
     }
@@ -33,4 +65,114 @@ public LtrQueryClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testC
     public static Iterable<Object[]> parameters() throws Exception {
         return OpenSearchClientYamlSuiteTestCase.createParameters();
     }
+
+    @Override
+    protected boolean preserveIndicesUponCompletion() {
+        return true;
+    }
+
+    @SuppressWarnings("unchecked")
+    @After
+    protected void wipeAllODFEIndices() throws IOException {
+        Response response = adminClient().performRequest(new Request("GET", "/_cat/indices?format=json&expand_wildcards=all"));
+        MediaType xContentType = MediaType.fromMediaType(response.getEntity().getContentType());
+        try (
+            XContentParser parser = xContentType
+                .xContent()
+                .createParser(
+                    NamedXContentRegistry.EMPTY,
+                    DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
+                    response.getEntity().getContent()
+                )
+        ) {
+            XContentParser.Token token = parser.nextToken();
+            List<Map<String, Object>> parserList = null;
+            if (token == XContentParser.Token.START_ARRAY) {
+                parserList = parser.listOrderedMap().stream().map(obj -> (Map<String, Object>) obj).collect(Collectors.toList());
+            } else {
+                parserList = Collections.singletonList(parser.mapOrdered());
+            }
+
+            for (Map<String, Object> index : parserList) {
+                String indexName = (String) index.get("index");
+                if (indexName != null && !".opendistro_security".equals(indexName)) {
+                    adminClient().performRequest(new Request("DELETE", "/" + indexName));
+                }
+            }
+        }
+    }
+
+    protected String getProtocol() {
+        return isHttps() ? "https" : "http";
+    }
+
+    protected boolean isHttps() {
+        boolean isHttps = Optional.ofNullable(System.getProperty("https")).map("true"::equalsIgnoreCase).orElse(false);
+        if (isHttps) {
+            // currently only external cluster is supported for security enabled testing
+            if (!Optional.ofNullable(System.getProperty("tests.rest.cluster")).isPresent()) {
+                throw new RuntimeException("cluster url should be provided for security enabled testing");
+            }
+        }
+
+        return isHttps;
+    }
+
+    @Override
+    protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOException {
+        RestClientBuilder builder = RestClient.builder(hosts);
+        if (isHttps()) {
+            configureHttpsClient(builder, settings);
+        } else {
+            configureClient(builder, settings);
+        }
+
+        builder.setStrictDeprecationMode(false);
+        return builder.build();
+    }
+
+    protected static void configureHttpsClient(RestClientBuilder builder, Settings settings) throws IOException {
+        Map<String, String> headers = ThreadContext.buildDefaultHeaders(settings);
+        Header[] defaultHeaders = new Header[headers.size()];
+        int i = 0;
+        for (Map.Entry<String, String> entry : headers.entrySet()) {
+            defaultHeaders[i++] = new BasicHeader(entry.getKey(), entry.getValue());
+        }
+        builder.setDefaultHeaders(defaultHeaders);
+        builder.setHttpClientConfigCallback(httpClientBuilder -> {
+            String userName = Optional
+                .ofNullable(System.getProperty("user"))
+                .orElseThrow(() -> new RuntimeException("user name is missing"));
+            String password = Optional
+                .ofNullable(System.getProperty("password"))
+                .orElseThrow(() -> new RuntimeException("password is missing"));
+            BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider();
+            credentialsProvider.setCredentials(new AuthScope(null, -1), new UsernamePasswordCredentials(userName, password.toCharArray()));
+            try {
+                final TlsStrategy tlsStrategy = ClientTlsStrategyBuilder
+                    .create()
+                    .setHostnameVerifier(NoopHostnameVerifier.INSTANCE)
+                    .setSslContext(SSLContextBuilder.create().loadTrustMaterial(null, (chains, authType) -> true).build())
+                    .build();
+                final PoolingAsyncClientConnectionManager connectionManager = PoolingAsyncClientConnectionManagerBuilder
+                    .create()
+                    .setMaxConnPerRoute(DEFAULT_MAX_CONN_PER_ROUTE)
+                    .setMaxConnTotal(DEFAULT_MAX_CONN_TOTAL)
+                    .setTlsStrategy(tlsStrategy)
+                    .build();
+                return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider).setConnectionManager(connectionManager);
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        });
+
+        final String socketTimeoutString = settings.get(CLIENT_SOCKET_TIMEOUT);
+        final TimeValue socketTimeout = TimeValue
+            .parseTimeValue(socketTimeoutString == null ? "60s" : socketTimeoutString, CLIENT_SOCKET_TIMEOUT);
+        builder
+            .setRequestConfigCallback(conf -> conf.setResponseTimeout(Timeout.ofMilliseconds(Math.toIntExact(socketTimeout.getMillis()))));
+        if (settings.hasValue(CLIENT_PATH_PREFIX)) {
+            builder.setPathPrefix(settings.get(CLIENT_PATH_PREFIX));
+        }
+    }
 }