Add CircuitBreakerServicePlugin

Existing_ES_Changes.txt

@@ -0,0 +1,50 @@
+
+1. Parent breaker's real memory breaking excluded for specified child circuit breakers
+
+Files changed:
+- ChildMemoryCircuitBreaker.java
+  Pass along child memory breaker's name as well in call to checkParentLimit()
+
+- ClusterSettings.java
+  Add HierarchyCircuitBreakerService.TOTAL_CIRCUIT_BREAKER_EXCLUDE_REAL_MEMORY_FOR setting
+
+- HierarchyCircuitBreakerService.java
+  Add setting "indices.breaker.total.exclude_real_memory_for" that takes a list of child circuit breakers' names.
+  For these paths, the parent real memory breaker will not break.
+
+2. Add hook for plugins to get hold of HierarchyCircuitBreakerService:
+
+Files changed:
+- CircuitBreakerServicePlugin.java
+  An interface for all plugins that need HierarchyCircuitBreakerService instance. The plugins can use the same passed
+  to setCircuitBreakerService()
+
+- Node.java
+  Use pluginsService to filter all plugins implementing CircuitBreakerServicePlugin and pass the breaker service
+  to them.
+
+3. Circuit Breaking Wrappers over java Collections
+
+Files Added:
+- CircuitBreakingCollection.java
+- CircuitBreakingHashMap.java
+- CircuitBreakingHashSet.java
+- CircuitBreakingList.java
+- CircuitBreakingMap.java
+- CircuitBreakingSet.java
+- CircuitBreakingTreeMap.java
+- CircuitBreakingTreeSet.java
+  Circuit Breaking wrappers over java collections. All these reserve sizes preemptively matching the growth rate of the arrays used in inner containers' implementation. For TreeMap and TreeSet, an imaginary growth rate similar to ArrayList is used to use CB in bulk with less overhead.
+
+- CBUtilsFactory.java
+  Factory class that takes a circuit breaker instance, and provides factory methods with same signatures as inner
+  collections' constructors for the circuit breaking wrappers and finalizing circuit breaking wrappers.
+
+4. Append index-shard & task-id to the search/throttled search pool thread names
+
+Files changed:
+- SearchService.java
+  Add a AutoClosable private class ThreadInfoAppenderOnName whose constructor takes the shardId and task.
+  It appends the thread name if not already appended as required. The close() resets the thread name to original.
+
+  Was used with a try() {} finally {} block wrapping around the thread's actual execution.

server/src/main/java/org/elasticsearch/common/breaker/ChildMemoryCircuitBreaker.java

@@ -95,7 +95,7 @@ public double addEstimateBytesAndMaybeBreak(long bytes, String label) throws Cir
 
         // Additionally, we need to check that we haven't exceeded the parent's limit
         try {
-            parent.checkParentLimit((long) (bytes * overheadConstant), label);
+            parent.checkParentLimit((long) (bytes * overheadConstant), label, this.name);
         } catch (CircuitBreakingException e) {
             // If the parent breaker is tripped, this breaker has to be
             // adjusted back down because the allocation is "blocked" but the

server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java

@@ -301,6 +301,7 @@ public void apply(Settings value, Settings current, Settings previous) {
             HttpTransportSettings.SETTING_HTTP_TRACE_LOG_EXCLUDE,
             HierarchyCircuitBreakerService.USE_REAL_MEMORY_USAGE_SETTING,
             HierarchyCircuitBreakerService.TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING,
+            HierarchyCircuitBreakerService.TOTAL_CIRCUIT_BREAKER_EXCLUDE_REAL_MEMORY_FOR,
             HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_LIMIT_SETTING,
             HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_OVERHEAD_SETTING,
             HierarchyCircuitBreakerService.IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_LIMIT_SETTING,

server/src/main/java/org/elasticsearch/common/util/breakingcollections/CBUtilsFactory.java

@@ -0,0 +1,261 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.common.util.breakingcollections;
+
+import org.elasticsearch.common.breaker.CircuitBreaker;
+
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+
+public class CBUtilsFactory {
+    private final CircuitBreaker circuitBreaker;
+
+    public CBUtilsFactory(CircuitBreaker circuitBreaker) {
+        this.circuitBreaker = circuitBreaker;
+    }
+
+    public <E> List<E> newArrayList() {
+        return new CircuitBreakingList<>(circuitBreaker);
+    }
+
+    public <E> List<E> newArrayList(int initialCapacity) {
+        return new CircuitBreakingList<>(circuitBreaker, initialCapacity);
+    }
+
+    public <E> List<E> newArrayList(Collection<? extends E> collection) {
+        return new CircuitBreakingList<>(circuitBreaker, collection);
+    }
+
+    public <E> List<E> newFinalizingArrayList() {
+        return new CircuitBreakingList<E>(circuitBreaker) {
+            @Override
+            public void finalize() {
+                this.close();
+            }
+        };
+    }
+
+    public <E> List<E> newFinalizingArrayList(int initialCapacity) {
+        return new CircuitBreakingList<E>(circuitBreaker, initialCapacity) {
+            @Override
+            public void finalize() {
+                this.close();
+            }
+        };
+    }
+
+    public <E> List<E> newFinalizingArrayList(Collection<? extends E> collection) {
+        return new CircuitBreakingList<E>(circuitBreaker, collection) {
+            @Override
+            public void finalize() {
+                this.close();
+            }
+        };
+    }
+
+    public <K, V> Map<K, V> newHashMap() {
+        return new CircuitBreakingHashMap<>(circuitBreaker);
+    }
+
+    public <K, V> Map<K, V> newHashMap(int initialCapacity, float loadFactor) {
+        return new CircuitBreakingHashMap<>(circuitBreaker, initialCapacity, loadFactor);
+    }
+
+    public <K, V> Map<K, V> newHashMap(int initialCapacity) {
+        return new CircuitBreakingHashMap<>(circuitBreaker, initialCapacity);
+    }
+
+    public <K, V> Map<K, V> newFinalizingHashMap() {
+        return new CircuitBreakingHashMap<K, V>(circuitBreaker) {
+            @Override
+            public void finalize() {
+                this.close();
+            }
+        };
+    }
+
+    public <K, V> Map<K, V> newFinalizingHashMap(int initialCapacity, float loadFactor) {
+        return new CircuitBreakingHashMap<K, V>(circuitBreaker, initialCapacity, loadFactor) {
+            @Override
+            public void finalize() {
+                this.close();
+            }
+        };
+    }
+
+    public <K, V> Map<K, V> newFinalizingHashMap(int initialCapacity) {
+        return new CircuitBreakingHashMap<K, V>(circuitBreaker, initialCapacity) {
+            @Override
+            public void finalize() {
+                this.close();
+            }
+        };
+    }
+
+    public <K, V> Map<K, V> newTreeMap() {
+        return new CircuitBreakingTreeMap<>(circuitBreaker);
+    }
+
+    public <K, V> Map<K, V> newTreeMap(Comparator<? super K> comparator) {
+        return new CircuitBreakingTreeMap<>(circuitBreaker, comparator);
+    }
+
+    public <K, V> Map<K, V> newTreeMap(Map<? extends K, ? extends V> m) {
+        return new CircuitBreakingTreeMap<>(circuitBreaker, m);
+    }
+
+    public <K, V> Map<K, V> newTreeMap(SortedMap<K, ? extends V> m) {
+        return new CircuitBreakingTreeMap<>(circuitBreaker, m);
+    }
+
+    public <K, V> Map<K, V> newFinalizingTreeMap() {
+        return new CircuitBreakingTreeMap<K, V>(circuitBreaker) {
+            @Override
+            public void finalize() {
+                this.close();
+            }
+        };
+    }
+
+    public <K, V> Map<K, V> newFinalizingTreeMap(Comparator<? super K> comparator) {
+        return new CircuitBreakingTreeMap<K, V>(circuitBreaker, comparator) {
+            @Override
+            public void finalize() {
+                this.close();
+            }
+        };
+    }
+
+    public <K, V> Map<K, V> newFinalizingTreeMap(Map<? extends K, ? extends V> m) {
+        return new CircuitBreakingTreeMap<K, V>(circuitBreaker, m) {
+            @Override
+            public void finalize() {
+                this.close();
+            }
+        };
+    }
+
+    public <K, V> Map<K, V> newFinalizingTreeMap(SortedMap<K, ? extends V> m) {
+        return new CircuitBreakingTreeMap<K, V>(circuitBreaker, m) {
+            @Override
+            public void finalize() {
+                this.close();
+            }
+        };
+    }
+
+    public <E> Set<E> newHashSet() {
+        return new CircuitBreakingHashSet<>(circuitBreaker);
+    }
+
+    public <E> Set<E> newHashSet(Collection<? extends E> c) {
+        return new CircuitBreakingHashSet<>(circuitBreaker, c);
+    }
+
+    public <E> Set<E> newHashSet(int initialCapacity, float loadFactor) {
+        return new CircuitBreakingHashSet<>(circuitBreaker, initialCapacity, loadFactor);
+    }
+
+    public <E> Set<E> newHashSet(int initialCapacity) {
+        return new CircuitBreakingHashSet<>(circuitBreaker, initialCapacity);
+    }
+
+    public <E> Set<E> newFinalizingHashSet() {
+        return new CircuitBreakingHashSet<E>(circuitBreaker) {
+            @Override
+            public void finalize() {
+                this.close();
+            }
+        };
+    }
+
+    public <E> Set<E> newFinalizingHashSet(Collection<? extends E> c) {
+        return new CircuitBreakingHashSet<E>(circuitBreaker, c) {
+            @Override
+            public void finalize() {
+                this.close();
+            }
+        };
+    }
+
+    public <E> Set<E> newFinalizingHashSet(int initialCapacity, float loadFactor) {
+        return new CircuitBreakingHashSet<E>(circuitBreaker, initialCapacity, loadFactor) {
+            @Override
+            public void finalize() {
+                this.close();
+            }
+        };
+    }
+
+    public <E> Set<E> newFinalizingHashSet(int initialCapacity) {
+        return new CircuitBreakingHashSet<E>(circuitBreaker, initialCapacity) {
+            @Override
+            public void finalize() {
+                this.close();
+            }
+        };
+    }
+
+    public <E> Set<E> newTreeSet() {
+        return new CircuitBreakingTreeSet<>(circuitBreaker);
+    }
+
+    public <E> Set<E> newTreeSet(Collection<? extends E> c) {
+        return new CircuitBreakingTreeSet<>(circuitBreaker, c);
+    }
+
+    public <E> Set<E> newTreeSet(Comparator<? super E> comparator) {
+        return new CircuitBreakingTreeSet<>(circuitBreaker, comparator);
+    }
+
+    public <E> Set<E> newTreeSet(SortedSet<E> s) {
+        return new CircuitBreakingTreeSet<>(circuitBreaker, s);
+    }
+
+    public <E> Set<E> newFinalizingTreeSet() {
+        return new CircuitBreakingTreeSet<E>(circuitBreaker) {
+            @Override
+            public void finalize() {
+                this.close();
+            }
+        };
+    }
+
+    public <E> Set<E> newFinalizingTreeSet(Collection<? extends E> c) {
+        return new CircuitBreakingTreeSet<E>(circuitBreaker, c) {
+            @Override
+            public void finalize() {
+                this.close();
+            }
+        };
+    }
+
+    public <E> Set<E> newFinalizingTreeSet(Comparator<? super E> comparator) {
+        return new CircuitBreakingTreeSet<E>(circuitBreaker, comparator) {
+            @Override
+            public void finalize() {
+                this.close();
+            }
+        };
+    }
+
+    public <E> Set<E> newFinalizingTreeSet(SortedSet<E> s) {
+        return new CircuitBreakingTreeSet<E>(circuitBreaker, s) {
+            @Override
+            public void finalize() {
+                this.close();
+            }
+        };
+    }
+}