Merge remote-tracking branch 'upstream/8.16' into add_eck_migration_8_16

jfreden · Oct 24, 2024 · 2563993 · 2563993
2 parents 5a96adf + 6c3168a
commit 2563993
Show file tree

Hide file tree

Showing 57 changed files with 2,125 additions and 2,584 deletions.
diff --git a/...nal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildCompletePlugin.java b/...nal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildCompletePlugin.java
@@ -15,6 +15,7 @@
 import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
 import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream;
 import org.apache.commons.io.IOUtils;
+import org.elasticsearch.gradle.OS;
 import org.elasticsearch.gradle.util.GradleUtils;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
@@ -61,7 +62,7 @@ public void apply(Project target) {
             ? System.getenv("BUILD_NUMBER")
             : System.getenv("BUILDKITE_BUILD_NUMBER");
         String performanceTest = System.getenv("BUILD_PERFORMANCE_TEST");
-        if (buildNumber != null && performanceTest == null && GradleUtils.isIncludedBuild(target) == false) {
+        if (buildNumber != null && performanceTest == null && GradleUtils.isIncludedBuild(target) == false && OS.current() != OS.WINDOWS) {
             File targetFile = calculateTargetFile(target, buildNumber);
             File projectDir = target.getProjectDir();
             File gradleWorkersDir = new File(target.getGradle().getGradleUserHomeDir(), "workers/");

diff --git a/docs/changelog/114482.yaml b/docs/changelog/114482.yaml
diff --git a/docs/changelog/115308.yaml b/docs/changelog/115308.yaml
@@ -0,0 +1,6 @@
+pr: 115308
+summary: "ESQL: Disable pushdown of WHERE past STATS"
+area: ES|QL
+type: bug
+issues:
+ - 115281
diff --git a/docs/changelog/115312.yaml b/docs/changelog/115312.yaml
@@ -0,0 +1,6 @@
+pr: 115312
+summary: "ESQL: Fix filtered grouping on ords"
+area: ES|QL
+type: bug
+issues:
+ - 114897
diff --git a/docs/changelog/115404.yaml b/docs/changelog/115404.yaml
@@ -0,0 +1,5 @@
+pr: 115404
+summary: Fix NPE in Get Deployment Stats
+area: Machine Learning
+type: bug
+issues: []
diff --git a/docs/reference/esql/functions/kibana/definition/match.json b/docs/reference/esql/functions/kibana/definition/match.json
diff --git a/docs/reference/esql/functions/kibana/definition/qstr.json b/docs/reference/esql/functions/kibana/definition/qstr.json
diff --git a/...javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeCustomSettingsIT.java b/...javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeCustomSettingsIT.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.cluster.local.distribution.DistributionType;
+import org.hamcrest.Matchers;
 import org.junit.Before;
 import org.junit.ClassRule;
 
@@ -438,6 +439,66 @@ public void testIgnoreMalformedSetting() throws IOException {
         }
     }
 
+    public void testIgnoreAboveSetting() throws IOException {
+        // with default template
+        {
+            assertOK(createDataStream(client, "logs-test-1"));
+            String logsIndex1 = getDataStreamBackingIndex(client, "logs-test-1", 0);
+            assertThat(getSetting(client, logsIndex1, "index.mapping.ignore_above"), equalTo("8191"));
+            for (String newValue : List.of("512", "2048", "12000", String.valueOf(Integer.MAX_VALUE))) {
+                closeIndex(logsIndex1);
+                updateIndexSettings(logsIndex1, Settings.builder().put("index.mapping.ignore_above", newValue));
+                assertThat(getSetting(client, logsIndex1, "index.mapping.ignore_above"), equalTo(newValue));
+            }
+            for (String newValue : List.of(String.valueOf((long) Integer.MAX_VALUE + 1), String.valueOf(Long.MAX_VALUE))) {
+                closeIndex(logsIndex1);
+                ResponseException ex = assertThrows(
+                    ResponseException.class,
+                    () -> updateIndexSettings(logsIndex1, Settings.builder().put("index.mapping.ignore_above", newValue))
+                );
+                assertThat(
+                    ex.getMessage(),
+                    Matchers.containsString("Failed to parse value [" + newValue + "] for setting [index.mapping.ignore_above]")
+                );
+            }
+        }
+        // with override template
+        {
+            var template = """
+                {
+                  "template": {
+                    "settings": {
+                      "index": {
+                        "mapping": {
+                          "ignore_above": "128"
+                        }
+                      }
+                    }
+                  }
+                }""";
+            assertOK(putComponentTemplate(client, "logs@custom", template));
+            assertOK(createDataStream(client, "logs-custom-dev"));
+            String index = getDataStreamBackingIndex(client, "logs-custom-dev", 0);
+            assertThat(getSetting(client, index, "index.mapping.ignore_above"), equalTo("128"));
+            for (String newValue : List.of("64", "256", "12000", String.valueOf(Integer.MAX_VALUE))) {
+                closeIndex(index);
+                updateIndexSettings(index, Settings.builder().put("index.mapping.ignore_above", newValue));
+                assertThat(getSetting(client, index, "index.mapping.ignore_above"), equalTo(newValue));
+            }
+        }
+        // standard index
+        {
+            String index = "test-index";
+            createIndex(index);
+            assertThat(getSetting(client, index, "index.mapping.ignore_above"), equalTo(Integer.toString(Integer.MAX_VALUE)));
+            for (String newValue : List.of("256", "512", "12000", String.valueOf(Integer.MAX_VALUE))) {
+                closeIndex(index);
+                updateIndexSettings(index, Settings.builder().put("index.mapping.ignore_above", newValue));
+                assertThat(getSetting(client, index, "index.mapping.ignore_above"), equalTo(newValue));
+            }
+        }
+    }
+
     private static Map<String, Object> getMapping(final RestClient client, final String indexName) throws IOException {
         final Request request = new Request("GET", "/" + indexName + "/_mapping");
 

diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/HttpClientTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/HttpClientTests.java
@@ -47,6 +47,7 @@ public static void startServer() throws Throwable {
         server.createContext("/404/", exchange -> {
             try {
                 exchange.sendResponseHeaders(404, 0);
+                exchange.close();
             } catch (Exception e) {
                 fail(e);
             }
@@ -102,6 +103,7 @@ public boolean checkCredentials(String username, String password) {
                     exchange.getResponseHeaders().add("Location", "/" + destination + "/");
                 }
                 exchange.sendResponseHeaders(302, 0);
+                exchange.close();
             } catch (Exception e) {
                 fail(e);
             }

diff --git a/muted-tests.yml b/muted-tests.yml
@@ -375,9 +375,6 @@ tests:
 - class: org.elasticsearch.xpack.enrich.EnrichIT
   method: testEnrichSpecialTypes
   issue: https://github.com/elastic/elasticsearch/issues/114773
-- class: org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT
-  method: test {string.ReverseGraphemeClusters}
-  issue: https://github.com/elastic/elasticsearch/issues/114560
 - class: org.elasticsearch.license.LicensingTests
   issue: https://github.com/elastic/elasticsearch/issues/114865
 - class: org.elasticsearch.xpack.enrich.EnrichIT
@@ -388,17 +385,23 @@ tests:
   issue: https://github.com/elastic/elasticsearch/issues/114839
 - class: org.elasticsearch.xpack.security.authc.ldap.GroupMappingIT
   issue: https://github.com/elastic/elasticsearch/issues/115221
-- class: org.elasticsearch.ingest.geoip.HttpClientTests
-  issue: https://github.com/elastic/elasticsearch/issues/115169
-- class: org.elasticsearch.reservedstate.service.FileSettingsServiceTests
-  method: testProcessFileChanges
-  issue: https://github.com/elastic/elasticsearch/issues/115280
 - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
   method: test {yaml=reference/rest-api/usage/line_38}
   issue: https://github.com/elastic/elasticsearch/issues/113694
 - class: org.elasticsearch.xpack.enrich.EnrichIT
   method: testDeleteExistingPipeline
   issue: https://github.com/elastic/elasticsearch/issues/114775
+- class: org.elasticsearch.xpack.security.operator.OperatorPrivilegesIT
+  method: testEveryActionIsEitherOperatorOnlyOrNonOperator
+  issue: https://github.com/elastic/elasticsearch/issues/102992
+- class: org.elasticsearch.bootstrap.SpawnerNoBootstrapTests
+  issue: https://github.com/elastic/elasticsearch/issues/114555
+- class: org.elasticsearch.ingest.geoip.DatabaseNodeServiceIT
+  method: testNonGzippedDatabase
+  issue: https://github.com/elastic/elasticsearch/issues/113821
+- class: org.elasticsearch.xpack.inference.DefaultEndPointsIT
+  method: testInferDeploysDefaultE5
+  issue: https://github.com/elastic/elasticsearch/issues/115361
 
 # Examples:
 #

diff --git a/...de/src/javaRestTest/java/org/elasticsearch/upgrades/FileSettingsRoleMappingUpgradeIT.java b/...de/src/javaRestTest/java/org/elasticsearch/upgrades/FileSettingsRoleMappingUpgradeIT.java
@@ -25,12 +25,10 @@
 
 import java.io.IOException;
 import java.util.List;
-import java.util.Map;
 import java.util.function.Supplier;
 
-import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.nullValue;
@@ -104,15 +102,17 @@ public void testRoleMappingsAppliedOnUpgrade() throws IOException {
             // the nodes have all been upgraded. Check they re-processed the role mappings in the settings file on
             // upgrade
             Request clusterStateRequest = new Request("GET", "/_cluster/state/metadata");
-            List<Object> roleMappings = new XContentTestUtils.JsonMapView(entityAsMap(client().performRequest(clusterStateRequest))).get(
-                "metadata.role_mappings.role_mappings"
+            List<Object> clusterStateRoleMappings = new XContentTestUtils.JsonMapView(
+                entityAsMap(client().performRequest(clusterStateRequest))
+            ).get("metadata.role_mappings.role_mappings");
+            assertThat(clusterStateRoleMappings, is(not(nullValue())));
+            assertThat(clusterStateRoleMappings.size(), equalTo(1));
+
+            assertThat(
+                entityAsMap(client().performRequest(new Request("GET", "/_security/role_mapping"))).keySet(),
+                // TODO change this to `contains` once the clean-up migration work is merged
+                hasItem("everyone_kibana-read-only-operator-mapping")
             );
-            assertThat(roleMappings, is(not(nullValue())));
-            assertThat(roleMappings.size(), equalTo(1));
-            assertThat(roleMappings, is(instanceOf(Map.class)));
-            @SuppressWarnings("unchecked")
-            Map<String, Object> roleMapping = (Map<String, Object>) roleMappings;
-            assertThat(roleMapping.keySet(), contains("everyone_kibana-read-only-operator-mapping"));
         }
     }
 }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/discovery/MasterDisruptionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/discovery/MasterDisruptionIT.java
@@ -21,66 +21,22 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.disruption.BlockMasterServiceOnMaster;
-import org.elasticsearch.test.disruption.IntermittentLongGCDisruption;
 import org.elasticsearch.test.disruption.NetworkDisruption;
 import org.elasticsearch.test.disruption.NetworkDisruption.TwoPartitions;
 import org.elasticsearch.test.disruption.ServiceDisruptionScheme;
-import org.elasticsearch.test.disruption.SingleNodeDisruption;
 import org.elasticsearch.xcontent.XContentType;
 
-import java.util.ArrayList;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Set;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.not;
 
 /**
  * Tests relating to the loss of the master.
  */
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0)
 public class MasterDisruptionIT extends AbstractDisruptionTestCase {
 
-    /**
-     * Test that cluster recovers from a long GC on master that causes other nodes to elect a new one
-     */
-    public void testMasterNodeGCs() throws Exception {
-        List<String> nodes = startCluster(3);
-        // NOTE: this assume must happen after starting the cluster, so that cleanup will have something to cleanup.
-        assumeFalse("jdk20 removed thread suspend/resume", Runtime.version().feature() >= 20);
-
-        String oldMasterNode = internalCluster().getMasterName();
-        // a very long GC, but it's OK as we remove the disruption when it has had an effect
-        SingleNodeDisruption masterNodeDisruption = new IntermittentLongGCDisruption(random(), oldMasterNode, 100, 200, 30000, 60000);
-        internalCluster().setDisruptionScheme(masterNodeDisruption);
-        masterNodeDisruption.startDisrupting();
-
-        Set<String> oldNonMasterNodesSet = new HashSet<>(nodes);
-        oldNonMasterNodesSet.remove(oldMasterNode);
-
-        List<String> oldNonMasterNodes = new ArrayList<>(oldNonMasterNodesSet);
-
-        logger.info("waiting for nodes to de-elect master [{}]", oldMasterNode);
-        for (String node : oldNonMasterNodesSet) {
-            assertDifferentMaster(node, oldMasterNode);
-        }
-
-        logger.info("waiting for nodes to elect a new master");
-        ensureStableCluster(2, oldNonMasterNodes.get(0));
-
-        // restore GC
-        masterNodeDisruption.stopDisrupting();
-        final TimeValue waitTime = new TimeValue(DISRUPTION_HEALING_OVERHEAD.millis() + masterNodeDisruption.expectedTimeToHeal().millis());
-        ensureStableCluster(3, waitTime, false, oldNonMasterNodes.get(0));
-
-        // make sure all nodes agree on master
-        String newMaster = internalCluster().getMasterName();
-        assertThat(newMaster, not(equalTo(oldMasterNode)));
-        assertMaster(newMaster, nodes);
-    }
-
     /**
      * This test isolates the master from rest of the cluster, waits for a new master to be elected, restores the partition
      * and verifies that all node agree on the new cluster state