diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 5ab0112d822ce..fd2d24c73c77f 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -96,5 +96,7 @@ tasks.named("yamlRestCompatTestTransform").configure({ task -> task.skipTest("esql/180_match_operator/match with non text field", "Match operator can now be used on non-text fields") task.skipTest("esql/180_match_operator/match with functions", "Error message changed") task.skipTest("esql/40_unsupported_types/semantic_text declared in mapping", "The semantic text field format changed") + task.replaceValueInMatch("Size", 49, "Test flamegraph from profiling-events") + task.replaceValueInMatch("Size", 49, "Test flamegraph from test-events") }) diff --git a/x-pack/plugin/core/template-resources/src/main/resources/profiling/component-template/profiling-events.json b/x-pack/plugin/core/template-resources/src/main/resources/profiling/component-template/profiling-events.json index c7424571dd678..7a5076b8db047 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/profiling/component-template/profiling-events.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/profiling/component-template/profiling-events.json @@ -13,6 +13,7 @@ "orchestrator.resource.name", "host.name", "container.name", + "process.executable.name", "process.thread.name" ] }, @@ -59,6 +60,9 @@ "process.thread.name": { "type": "keyword" }, + "process.executable.name": { + "type": "keyword" + }, "Stacktrace.count": { "type": "short", "index": false diff --git a/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/action/GetFlameGraphActionIT.java b/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/action/GetFlameGraphActionIT.java index 49a5cfa7ca067..1d2f5fa049ff0 100644 --- a/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/action/GetFlameGraphActionIT.java +++ b/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/action/GetFlameGraphActionIT.java @@ -26,10 +26,10 @@ public void testGetStackTracesUnfiltered() throws Exception { ); GetFlamegraphResponse response = client().execute(GetFlamegraphAction.INSTANCE, request).get(); // only spot-check top level properties - detailed tests are done in unit tests - assertEquals(994, response.getSize()); + assertEquals(1010, response.getSize()); assertEquals(1.0d, response.getSamplingRate(), 0.001d); assertEquals(46, response.getSelfCPU()); - assertEquals(1903, response.getTotalCPU()); + assertEquals(1995, response.getTotalCPU()); assertEquals(46, response.getTotalSamples()); // The root node's values are the same as the top-level values. diff --git a/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/action/GetStackTracesActionIT.java b/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/action/GetStackTracesActionIT.java index 4a9f1146fe95c..7983543eece67 100644 --- a/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/action/GetStackTracesActionIT.java +++ b/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/action/GetStackTracesActionIT.java @@ -12,6 +12,7 @@ import org.elasticsearch.index.query.TermQueryBuilder; import java.util.List; +import java.util.Map; public class GetStackTracesActionIT extends ProfilingTestCase { public void testGetStackTracesUnfiltered() throws Exception { @@ -36,7 +37,11 @@ public void testGetStackTracesUnfiltered() throws Exception { assertEquals(1821, response.getTotalFrames()); assertNotNull(response.getStackTraceEvents()); - assertEquals(3L, response.getStackTraceEvents().get("L7kj7UvlKbT-vN73el4faQ").count); + + Map traceEvents = response.getStackTraceEvents(); + + TraceEventID traceEventID = new TraceEventID("", "497295213074376", "8457605156473051743", "L7kj7UvlKbT-vN73el4faQ"); + assertEquals(3L, response.getStackTraceEvents().get(traceEventID).count); assertNotNull(response.getStackTraces()); // just do a high-level spot check. Decoding is tested in unit-tests @@ -45,8 +50,6 @@ public void testGetStackTracesUnfiltered() throws Exception { assertEquals(18, stackTrace.fileIds.length); assertEquals(18, stackTrace.frameIds.length); assertEquals(18, stackTrace.typeIds.length); - assertEquals(0.0000051026469d, stackTrace.annualCO2Tons, 0.0000000001d); - assertEquals(0.19825d, stackTrace.annualCostsUSD, 0.00001d); // not determined by default assertNull(stackTrace.subGroups); @@ -80,7 +83,10 @@ public void testGetStackTracesGroupedByServiceName() throws Exception { assertEquals(1821, response.getTotalFrames()); assertNotNull(response.getStackTraceEvents()); - assertEquals(3L, response.getStackTraceEvents().get("L7kj7UvlKbT-vN73el4faQ").count); + + TraceEventID traceEventID = new TraceEventID("", "497295213074376", "8457605156473051743", "L7kj7UvlKbT-vN73el4faQ"); + assertEquals(3L, response.getStackTraceEvents().get(traceEventID).count); + assertEquals(Long.valueOf(2L), response.getStackTraceEvents().get(traceEventID).subGroups.getCount("basket")); assertNotNull(response.getStackTraces()); // just do a high-level spot check. Decoding is tested in unit-tests @@ -89,9 +95,6 @@ public void testGetStackTracesGroupedByServiceName() throws Exception { assertEquals(18, stackTrace.fileIds.length); assertEquals(18, stackTrace.frameIds.length); assertEquals(18, stackTrace.typeIds.length); - assertEquals(0.0000051026469d, stackTrace.annualCO2Tons, 0.0000000001d); - assertEquals(0.19825d, stackTrace.annualCostsUSD, 0.00001d); - assertEquals(Long.valueOf(2L), stackTrace.subGroups.getCount("basket")); assertNotNull(response.getStackFrames()); StackFrame stackFrame = response.getStackFrames().get("8NlMClggx8jaziUTJXlmWAAAAAAAAIYI"); @@ -127,8 +130,13 @@ public void testGetStackTracesFromAPMWithMatchNoDownsampling() throws Exception assertEquals(1.0d, response.getSamplingRate(), 0.001d); assertNotNull(response.getStackTraceEvents()); - assertEquals(3L, response.getStackTraceEvents().get("Ce77w10WeIDow3kd1jowlA").count); - assertEquals(2L, response.getStackTraceEvents().get("JvISdnJ47BQ01489cwF9DA").count); + + TraceEventID traceEventID = new TraceEventID("", "", "", "Ce77w10WeIDow3kd1jowlA"); + assertEquals(3L, response.getStackTraceEvents().get(traceEventID).count); + assertEquals(Long.valueOf(3L), response.getStackTraceEvents().get(traceEventID).subGroups.getCount("encodeSha1")); + + traceEventID = new TraceEventID("", "", "", "JvISdnJ47BQ01489cwF9DA"); + assertEquals(2L, response.getStackTraceEvents().get(traceEventID).count); assertNotNull(response.getStackTraces()); // just do a high-level spot check. Decoding is tested in unit-tests @@ -137,9 +145,6 @@ public void testGetStackTracesFromAPMWithMatchNoDownsampling() throws Exception assertEquals(39, stackTrace.fileIds.length); assertEquals(39, stackTrace.frameIds.length); assertEquals(39, stackTrace.typeIds.length); - assertTrue(stackTrace.annualCO2Tons > 0.0d); - assertTrue(stackTrace.annualCostsUSD > 0.0d); - assertEquals(Long.valueOf(3L), stackTrace.subGroups.getCount("encodeSha1")); assertNotNull(response.getStackFrames()); StackFrame stackFrame = response.getStackFrames().get("fhsEKXDuxJ-jIJrZpdRuSAAAAAAAAFtj"); @@ -175,9 +180,13 @@ public void testGetStackTracesFromAPMWithMatchAndDownsampling() throws Exception assertEquals(0.2d, response.getSamplingRate(), 0.001d); assertNotNull(response.getStackTraceEvents()); + // as the sampling rate is 0.2, we see 5 times more samples (random sampler agg automatically adjusts sample count) - assertEquals(5 * 3L, response.getStackTraceEvents().get("Ce77w10WeIDow3kd1jowlA").count); - assertEquals(5 * 2L, response.getStackTraceEvents().get("JvISdnJ47BQ01489cwF9DA").count); + TraceEventID traceEventID = new TraceEventID("", "", "", "Ce77w10WeIDow3kd1jowlA"); + assertEquals(5 * 3L, response.getStackTraceEvents().get(traceEventID).count); + + traceEventID = new TraceEventID("", "", "", "JvISdnJ47BQ01489cwF9DA"); + assertEquals(5 * 2L, response.getStackTraceEvents().get(traceEventID).count); assertNotNull(response.getStackTraces()); // just do a high-level spot check. Decoding is tested in unit-tests @@ -186,8 +195,6 @@ public void testGetStackTracesFromAPMWithMatchAndDownsampling() throws Exception assertEquals(39, stackTrace.fileIds.length); assertEquals(39, stackTrace.frameIds.length); assertEquals(39, stackTrace.typeIds.length); - assertTrue(stackTrace.annualCO2Tons > 0.0d); - assertTrue(stackTrace.annualCostsUSD > 0.0d); // not determined by default assertNull(stackTrace.subGroups); diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/GetStackTracesResponse.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/GetStackTracesResponse.java index c2dd51acfd76b..cac5221e42d62 100644 --- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/GetStackTracesResponse.java +++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/GetStackTracesResponse.java @@ -32,7 +32,7 @@ public class GetStackTracesResponse extends ActionResponse implements ChunkedToX private final Map executables; @UpdateForV9(owner = UpdateForV9.Owner.PROFILING) // remove this field - it is unused in Kibana @Nullable - private final Map stackTraceEvents; + private final Map stackTraceEvents; @UpdateForV9(owner = UpdateForV9.Owner.PROFILING) // remove this field - it is unused in Kibana private final int totalFrames; private final double samplingRate; @@ -42,7 +42,7 @@ public GetStackTracesResponse( Map stackTraces, Map stackFrames, Map executables, - Map stackTraceEvents, + Map stackTraceEvents, int totalFrames, double samplingRate, long totalSamples @@ -73,7 +73,7 @@ public Map getExecutables() { return executables; } - public Map getStackTraceEvents() { + public Map getStackTraceEvents() { return stackTraceEvents; } @@ -100,23 +100,24 @@ public Iterator toXContentChunked(ToXContent.Params params optional( "stack_trace_events", stackTraceEvents, - (n, v) -> ChunkedToXContentHelper.object(n, v, entry -> (b, p) -> b.field(entry.getKey(), entry.getValue().count)) + (n, v) -> ChunkedToXContentHelper.object( + n, + Iterators.map(v.entrySet().iterator(), e -> (b, p) -> b.field(e.getKey().stacktraceID(), e.getValue().count)) + ) ), - Iterators.single((b, p) -> b.field("total_frames", totalFrames)), - Iterators.single((b, p) -> b.field("sampling_rate", samplingRate)), + Iterators.single((b, p) -> b.field("total_frames", totalFrames).field("sampling_rate", samplingRate).endObject()) // the following fields are intentionally not written to the XContent representation (only needed on the transport layer): // // * start // * end // * totalSamples - ChunkedToXContentHelper.endObject() ); } - private static Iterator optional( + private static Iterator optional( String name, - Map values, - BiFunction, Iterator> supplier + Map values, + BiFunction, Iterator> supplier ) { return (values != null) ? supplier.apply(name, values) : Collections.emptyIterator(); } diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/GetStackTracesResponseBuilder.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/GetStackTracesResponseBuilder.java index 8bb207c0f990f..bf1111e2ff1d5 100644 --- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/GetStackTracesResponseBuilder.java +++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/GetStackTracesResponseBuilder.java @@ -8,7 +8,6 @@ package org.elasticsearch.xpack.profiling.action; import java.time.Instant; -import java.util.List; import java.util.Map; class GetStackTracesResponseBuilder { @@ -18,8 +17,7 @@ class GetStackTracesResponseBuilder { private int totalFrames; private Map stackFrames; private Map executables; - private Map stackTraceEvents; - private List hostEventCounts; + private Map stackTraceEvents; private double samplingRate; private long totalSamples; private Double requestedDuration; @@ -67,19 +65,11 @@ public void setExecutables(Map executables) { this.executables = executables; } - public void setStackTraceEvents(Map stackTraceEvents) { + public void setStackTraceEvents(Map stackTraceEvents) { this.stackTraceEvents = stackTraceEvents; } - public void setHostEventCounts(List hostEventCounts) { - this.hostEventCounts = hostEventCounts; - } - - public List getHostEventCounts() { - return hostEventCounts; - } - - public Map getStackTraceEvents() { + public Map getStackTraceEvents() { return stackTraceEvents; } @@ -149,17 +139,17 @@ public void setTotalSamples(long totalSamples) { public GetStackTracesResponse build() { // Merge the TraceEvent data into the StackTraces. if (stackTraces != null) { - for (Map.Entry entry : stackTraces.entrySet()) { - String stacktraceID = entry.getKey(); - TraceEvent event = stackTraceEvents.get(stacktraceID); - if (event != null) { - StackTrace stackTrace = entry.getValue(); - stackTrace.count = event.count; + for (Map.Entry entry : stackTraceEvents.entrySet()) { + TraceEventID traceEventID = entry.getKey(); + StackTrace stackTrace = stackTraces.get(traceEventID.stacktraceID()); + if (stackTrace != null) { + TraceEvent event = entry.getValue(); if (event.subGroups != null) { stackTrace.subGroups = event.subGroups; } - stackTrace.annualCO2Tons = event.annualCO2Tons; - stackTrace.annualCostsUSD = event.annualCostsUSD; + stackTrace.count += event.count; + stackTrace.annualCO2Tons += event.annualCO2Tons; + stackTrace.annualCostsUSD += event.annualCostsUSD; } } } diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/StackTrace.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/StackTrace.java index 0be6d91450eda..48c66d91abfb4 100644 --- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/StackTrace.java +++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/StackTrace.java @@ -31,22 +31,14 @@ final class StackTrace implements ToXContentObject { double annualCostsUSD; long count; - StackTrace( - int[] addressOrLines, - String[] fileIds, - String[] frameIds, - int[] typeIds, - double annualCO2Tons, - double annualCostsUSD, - long count - ) { + StackTrace(int[] addressOrLines, String[] fileIds, String[] frameIds, int[] typeIds) { this.addressOrLines = addressOrLines; this.fileIds = fileIds; this.frameIds = frameIds; this.typeIds = typeIds; - this.annualCO2Tons = annualCO2Tons; - this.annualCostsUSD = annualCostsUSD; - this.count = count; + annualCO2Tons = 0.0d; + annualCostsUSD = 0.0d; + count = 0; } private static final int BASE64_FRAME_ID_LENGTH = 32; @@ -210,7 +202,7 @@ public static StackTrace fromSource(Map source) { // Step 2: Convert the run-length byte encoding into a list of uint8s. int[] typeIDs = runLengthDecodeBase64Url(inputFrameTypes, inputFrameTypes.length(), countsFrameIDs); - return new StackTrace(addressOrLines, fileIDs, frameIDs, typeIDs, 0, 0, 0); + return new StackTrace(addressOrLines, fileIDs, frameIDs, typeIDs); } public void forNativeAndKernelFrames(Consumer consumer) { @@ -224,15 +216,15 @@ public void forNativeAndKernelFrames(Consumer consumer) { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - builder.field("address_or_lines", this.addressOrLines); - builder.field("file_ids", this.fileIds); - builder.field("frame_ids", this.frameIds); - builder.field("type_ids", this.typeIds); - builder.field("annual_co2_tons", this.annualCO2Tons); - builder.field("annual_costs_usd", this.annualCostsUSD); - builder.field("count", this.count); - builder.endObject(); + builder.startObject() + .field("address_or_lines", this.addressOrLines) + .field("file_ids", this.fileIds) + .field("frame_ids", this.frameIds) + .field("type_ids", this.typeIds) + .field("annual_co2_tons", this.annualCO2Tons) + .field("annual_costs_usd", this.annualCostsUSD) + .field("count", this.count) + .endObject(); return builder; } diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TraceEvent.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TraceEvent.java index b2c50512a5b9c..88baefe14ae53 100644 --- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TraceEvent.java +++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TraceEvent.java @@ -7,53 +7,29 @@ package org.elasticsearch.xpack.profiling.action; -import java.util.Objects; - final class TraceEvent { - final String stacktraceID; + long count; double annualCO2Tons; double annualCostsUSD; - long count; SubGroup subGroups; - TraceEvent(String stacktraceID) { - this(stacktraceID, 0); + TraceEvent() { + this(0); } - TraceEvent(String stacktraceID, long count) { - this.stacktraceID = stacktraceID; + TraceEvent(long count) { this.count = count; } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - TraceEvent event = (TraceEvent) o; - return count == event.count && Objects.equals(stacktraceID, event.stacktraceID); - } - - @Override - public int hashCode() { - return Objects.hash(stacktraceID, count); - } - @Override public String toString() { return "TraceEvent{" - + "stacktraceID='" - + stacktraceID - + '\'' + + "count=" + + count + ", annualCO2Tons=" + annualCO2Tons + ", annualCostsUSD=" + annualCostsUSD - + ", count=" - + count + ", subGroups=" + subGroups + '}'; diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TraceEventID.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TraceEventID.java new file mode 100644 index 0000000000000..73e4e0e083a62 --- /dev/null +++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TraceEventID.java @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.profiling.action; + +record TraceEventID(String processName, String threadName, String hostID, String stacktraceID) {} diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetFlamegraphAction.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetFlamegraphAction.java index 66c9eaea3ec41..48b9670221eca 100644 --- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetFlamegraphAction.java +++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetFlamegraphAction.java @@ -24,10 +24,13 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.SortedMap; -import java.util.TreeMap; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicInteger; public class TransportGetFlamegraphAction extends TransportAction { + public static final int FRAMETYPE_ROOT = 0x100; + public static final int FRAMETYPE_PROCESS = 0x101; + public static final int FRAMETYPE_THREAD = 0x102; private static final Logger log = LogManager.getLogger(TransportGetFlamegraphAction.class); private final NodeClient nodeClient; private final TransportService transportService; @@ -73,22 +76,30 @@ static GetFlamegraphResponse buildFlamegraph(GetStackTracesResponse response) { return builder.build(); } - SortedMap sortedStacktraces = new TreeMap<>(response.getStackTraces()); - for (Map.Entry st : sortedStacktraces.entrySet()) { - StackTrace stackTrace = st.getValue(); - builder.setCurrentNode(0); + Map stackTraces = response.getStackTraces(); + AtomicInteger lostStackTraces = new AtomicInteger(); + + response.getStackTraceEvents().forEach((eventId, event) -> { + StackTrace stackTrace = stackTraces.get(eventId.stacktraceID()); + if (stackTrace == null) { + lostStackTraces.getAndIncrement(); + return; + } - long samples = stackTrace.count; - builder.addSamplesInclusive(0, samples); - builder.addSamplesExclusive(0, 0L); + long samples = event.count; + double annualCO2Tons = event.annualCO2Tons; + double annualCostsUSD = event.annualCostsUSD; - double annualCO2Tons = stackTrace.annualCO2Tons; - builder.addAnnualCO2TonsInclusive(0, annualCO2Tons); - builder.addAnnualCO2TonsExclusive(0, 0.0d); + String processName = eventId.processName(); + if (processName.isEmpty() && stackTrace.typeIds.length > 0 && stackTrace.typeIds[0] == StackTrace.KERNEL_FRAME_TYPE) { + // kernel threads are not associated with a process + processName = "kernel"; + } - double annualCostsUSD = stackTrace.annualCostsUSD; - builder.addAnnualCostsUSDInclusive(0, annualCostsUSD); - builder.addAnnualCostsUSDExclusive(0, 0.0d); + builder.setCurrentNode(0); + builder.addToRootNode(samples, annualCO2Tons, annualCostsUSD); + builder.addOrUpdateAggregationNode(processName, samples, annualCO2Tons, annualCostsUSD, FRAMETYPE_PROCESS); + builder.addOrUpdateAggregationNode(eventId.threadName(), samples, annualCO2Tons, annualCostsUSD, FRAMETYPE_THREAD); int frameCount = stackTrace.frameIds.length; for (int i = 0; i < frameCount; i++) { @@ -103,9 +114,8 @@ static GetFlamegraphResponse buildFlamegraph(GetStackTracesResponse response) { stackFrame.forEach(frame -> { String frameGroupId = FrameGroupID.create(fileId, addressOrLine, executable, frame.fileName(), frame.functionName()); - int nodeId; - if (builder.isExists(frameGroupId)) { - nodeId = builder.getNodeId(frameGroupId); + int nodeId = builder.getNodeId(frameGroupId); + if (nodeId != -1) { builder.addSamplesInclusive(nodeId, samples); builder.addAnnualCO2TonsInclusive(nodeId, annualCO2Tons); builder.addAnnualCostsUSDInclusive(nodeId, annualCostsUSD); @@ -135,7 +145,7 @@ static GetFlamegraphResponse buildFlamegraph(GetStackTracesResponse response) { builder.setCurrentNode(nodeId); }); } - } + }); return builder.build(); } @@ -186,10 +196,11 @@ private static class FlamegraphBuilder { this.annualCostsUSDInclusive = new ArrayList<>(capacity); this.annualCostsUSDExclusive = new ArrayList<>(capacity); this.totalSamples = totalSamples; + this.samplingRate = samplingRate; + // always insert root node - int nodeId = this.addNode("", 0, false, "", 0, "", 0, "", 0, 0, 0.0, 0.0, null); + int nodeId = this.addNode("", FRAMETYPE_ROOT, false, "", 0, "", 0, "", 0, 0, 0.0, 0.0, null); this.setCurrentNode(nodeId); - this.samplingRate = samplingRate; } // returns the new node's id @@ -233,6 +244,26 @@ public int addNode( return node; } + public void addToRootNode(long samples, double annualCO2Tons, double annualCostsUSD) { + addSamplesInclusive(0, samples); + addAnnualCO2TonsInclusive(0, annualCO2Tons); + addAnnualCostsUSDInclusive(0, annualCostsUSD); + } + + public void addOrUpdateAggregationNode(String name, long samples, double annualCO2Tons, double annualCostsUSD, int frameType) { + String frameGroupId = Integer.toString(Objects.hash(name, frameType)); + int nodeId = getNodeId(frameGroupId); + + if (nodeId != -1) { + addSamplesInclusive(nodeId, samples); + addAnnualCO2TonsInclusive(nodeId, annualCO2Tons); + addAnnualCostsUSDInclusive(nodeId, annualCostsUSD); + setCurrentNode(nodeId); + } else { + setCurrentNode(addNode("", frameType, false, name, 0, "", 0, "", 0, samples, annualCO2Tons, annualCostsUSD, frameGroupId)); + } + } + public void setCurrentNode(int nodeId) { this.currentNode = nodeId; } @@ -242,7 +273,7 @@ public boolean isExists(String frameGroupId) { } public int getNodeId(String frameGroupId) { - return this.edges.get(currentNode).get(frameGroupId); + return this.edges.get(currentNode).getOrDefault(frameGroupId, -1); } public void addSamplesInclusive(int nodeId, long sampleCount) { diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetStackTracesAction.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetStackTracesAction.java index 982410e8a5345..fecd5de13ad20 100644 --- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetStackTracesAction.java +++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetStackTracesAction.java @@ -21,7 +21,6 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsExecutors; @@ -57,11 +56,10 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.TreeMap; +import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicInteger; @@ -278,28 +276,22 @@ private void searchGenericEventGroupedByStackTrace( SingleBucketAggregation sample = searchResponse.getAggregations().get("sample"); Terms stacktraces = sample.getAggregations().get("group_by"); - // When we retrieve host data for generic events, we need to adapt the handler similar to searchEventGroupedByStackTrace(). - List hostEventCounts = new ArrayList<>(stacktraces.getBuckets().size()); + // When we retrieve data for generic events, we need to adapt the handler similar to searchEventGroupedByStackTrace(). // aggregation - Map stackTraceEvents = new TreeMap<>(); + Map stackTraceEvents = new HashMap<>(); for (Terms.Bucket stacktraceBucket : stacktraces.getBuckets()) { long count = stacktraceBucket.getDocCount(); totalSamples += count; String stackTraceID = stacktraceBucket.getKeyAsString(); - // For now, add a dummy-entry so CO2 and cost calculation can operate. In the future we will have one value per host. - hostEventCounts.add(new HostEventCount("unknown", stackTraceID, (int) count)); - TraceEvent event = stackTraceEvents.get(stackTraceID); - if (event == null) { - event = new TraceEvent(stackTraceID); - stackTraceEvents.put(stackTraceID, event); - } + + TraceEventID eventID = new TraceEventID("", "", "", stackTraceID); + TraceEvent event = stackTraceEvents.computeIfAbsent(eventID, k -> new TraceEvent()); event.count += count; subGroups.collectResults(stacktraceBucket, event); } responseBuilder.setTotalSamples(totalSamples); - responseBuilder.setHostEventCounts(hostEventCounts); log.debug("Found [{}] stacktrace events.", stackTraceEvents.size()); return stackTraceEvents; })); @@ -313,6 +305,9 @@ private void searchEventGroupedByStackTrace( GetStackTracesResponseBuilder responseBuilder, EventsIndex eventsIndex ) { + // We have nested aggregations, which in theory might blow up to MAX_TRACE_EVENTS_RESULT_SIZE^2 items + // reported. But we know that the total number of items is limited by our down-sampling to + // a maximum of ~100k (MAX_TRACE_EVENTS_RESULT_SIZE is higher to be on the safe side). responseBuilder.setSamplingRate(eventsIndex.getSampleRate()); TermsAggregationBuilder groupByStackTraceId = new TermsAggregationBuilder("group_by") // 'size' should be max 100k, but might be slightly more. Better be on the safe side. @@ -322,6 +317,22 @@ private void searchEventGroupedByStackTrace( // Especially with high cardinality fields, this makes aggregations really slow. .executionHint("map") .subAggregation(new SumAggregationBuilder("count").field("Stacktrace.count")); + TermsAggregationBuilder groupByHostId = new TermsAggregationBuilder("group_by") + // 'size' specifies the max number of host ID we support per request. + .size(MAX_TRACE_EVENTS_RESULT_SIZE) + .field("host.id") + // 'execution_hint: map' skips the slow building of ordinals that we don't need. + // Especially with high cardinality fields, this makes aggregations really slow. + .executionHint("map") + .subAggregation(groupByStackTraceId); + TermsAggregationBuilder groupByThreadName = new TermsAggregationBuilder("group_by") + // 'size' specifies the max number of host ID we support per request. + .size(MAX_TRACE_EVENTS_RESULT_SIZE) + .field("process.thread.name") + // 'execution_hint: map' skips the slow building of ordinals that we don't need. + // Especially with high cardinality fields, this makes aggregations really slow. + .executionHint("map") + .subAggregation(groupByHostId); SubGroupCollector subGroups = SubGroupCollector.attach(groupByStackTraceId, request.getAggregationFields()); client.prepareSearch(eventsIndex.getName()) .setTrackTotalHits(false) @@ -333,62 +344,62 @@ private void searchEventGroupedByStackTrace( .addAggregation(new MinAggregationBuilder("min_time").field("@timestamp")) .addAggregation(new MaxAggregationBuilder("max_time").field("@timestamp")) .addAggregation( - // We have nested aggregations, which in theory might blow up to MAX_TRACE_EVENTS_RESULT_SIZE^2 items - // reported. But we know that the total number of items is limited by our down-sampling to - // a maximum of ~100k (MAX_TRACE_EVENTS_RESULT_SIZE is higher to be on the safe side). new TermsAggregationBuilder("group_by") // 'size' specifies the max number of host ID we support per request. .size(MAX_TRACE_EVENTS_RESULT_SIZE) - .field("host.id") + // 'process.executable.name' is the process name (OTEL semconv), not the executable name. + .field("process.executable.name") + // missing("") is used to include documents where the field is missing. + .missing("") // 'execution_hint: map' skips the slow building of ordinals that we don't need. // Especially with high cardinality fields, this makes aggregations really slow. .executionHint("map") - .subAggregation(groupByStackTraceId) + .subAggregation(groupByThreadName) ) .addAggregation(new SumAggregationBuilder("total_count").field("Stacktrace.count")) .execute(handleEventsGroupedByStackTrace(submitTask, client, responseBuilder, submitListener, searchResponse -> { long totalCount = getAggValueAsLong(searchResponse, "total_count"); Resampler resampler = new Resampler(request, responseBuilder.getSamplingRate(), totalCount); - Terms hosts = searchResponse.getAggregations().get("group_by"); // Sort items lexicographically to access Lucene's term dictionary more efficiently when issuing an mget request. // The term dictionary is lexicographically sorted and using the same order reduces the number of page faults // needed to load it. long totalFinalCount = 0; - List hostEventCounts = new ArrayList<>(MAX_TRACE_EVENTS_RESULT_SIZE); - Map stackTraceEvents = new TreeMap<>(); - for (Terms.Bucket hostBucket : hosts.getBuckets()) { - String hostid = hostBucket.getKeyAsString(); - - Terms stacktraces = hostBucket.getAggregations().get("group_by"); - for (Terms.Bucket stacktraceBucket : stacktraces.getBuckets()) { - Sum count = stacktraceBucket.getAggregations().get("count"); - int finalCount = resampler.adjustSampleCount((int) count.value()); - if (finalCount <= 0) { - continue; - } - totalFinalCount += finalCount; - - /* - The same stacktraces may come from different hosts (eventually from different datacenters). - We make a list of the triples here. As soon as we have the host metadata, we can calculate - the CO2 emission and the costs for each TraceEvent. - */ - String stackTraceID = stacktraceBucket.getKeyAsString(); - hostEventCounts.add(new HostEventCount(hostid, stackTraceID, finalCount)); - - TraceEvent event = stackTraceEvents.get(stackTraceID); - if (event == null) { - event = new TraceEvent(stackTraceID); - stackTraceEvents.put(stackTraceID, event); + Map stackTraceEvents = new HashMap<>(MAX_TRACE_EVENTS_RESULT_SIZE); + + Terms processNames = searchResponse.getAggregations().get("group_by"); + for (Terms.Bucket processBucket : processNames.getBuckets()) { + String processName = processBucket.getKeyAsString(); + + Terms threads = processBucket.getAggregations().get("group_by"); + for (Terms.Bucket threadBucket : threads.getBuckets()) { + String threadName = threadBucket.getKeyAsString(); + + Terms hosts = threadBucket.getAggregations().get("group_by"); + for (Terms.Bucket hostBucket : hosts.getBuckets()) { + String hostID = hostBucket.getKeyAsString(); + + Terms stacktraces = hostBucket.getAggregations().get("group_by"); + for (Terms.Bucket stacktraceBucket : stacktraces.getBuckets()) { + Sum count = stacktraceBucket.getAggregations().get("count"); + int finalCount = resampler.adjustSampleCount((int) count.value()); + if (finalCount <= 0) { + continue; + } + totalFinalCount += finalCount; + + String stackTraceID = stacktraceBucket.getKeyAsString(); + + TraceEventID eventID = new TraceEventID(processName, threadName, hostID, stackTraceID); + TraceEvent event = stackTraceEvents.computeIfAbsent(eventID, k -> new TraceEvent()); + event.count += finalCount; + subGroups.collectResults(stacktraceBucket, event); + } } - event.count += finalCount; - subGroups.collectResults(stacktraceBucket, event); } } responseBuilder.setTotalSamples(totalFinalCount); - responseBuilder.setHostEventCounts(hostEventCounts); log.debug( "Found [{}] stacktrace events, resampled with sample rate [{}] to [{}] events ([{}] unique stack traces).", totalCount, @@ -405,14 +416,14 @@ private ActionListener handleEventsGroupedByStackTrace( Client client, GetStackTracesResponseBuilder responseBuilder, ActionListener submitListener, - Function> stacktraceCollector + Function> stacktraceCollector ) { StopWatch watch = new StopWatch("eventsGroupedByStackTrace"); return ActionListener.wrap(searchResponse -> { long minTime = getAggValueAsLong(searchResponse, "min_time"); long maxTime = getAggValueAsLong(searchResponse, "max_time"); - Map stackTraceEvents = stacktraceCollector.apply(searchResponse); + Map stackTraceEvents = stacktraceCollector.apply(searchResponse); log.debug(watch::report); if (stackTraceEvents.isEmpty() == false) { @@ -448,19 +459,20 @@ private void retrieveStackTraces( if (submitTask.notifyIfCancelled(submitListener)) { return; } - List eventIds = new ArrayList<>(responseBuilder.getStackTraceEvents().keySet()); + Set stacktraceIds = new TreeSet<>(); + Set hostIds = new TreeSet<>(); + for (TraceEventID id : responseBuilder.getStackTraceEvents().keySet()) { + stacktraceIds.add(id.stacktraceID()); + hostIds.add(id.hostID()); + } + log.info("Using [{}] hostIds and [{}] stacktraceIds.", hostIds.size(), stacktraceIds.size()); + ClusterState clusterState = clusterService.state(); List indices = resolver.resolve(clusterState, "profiling-stacktraces", responseBuilder.getStart(), responseBuilder.getEnd()); // Avoid parallelism if there is potential we are on spinning disks (frozen tier uses searchable snapshots) int sliceCount = IndexAllocation.isAnyOnWarmOrColdTier(clusterState, indices) ? 1 : desiredSlices; log.trace("Using [{}] slice(s) to lookup stacktraces.", sliceCount); - List> slicedEventIds = sliced(eventIds, sliceCount); - - // Build a set of unique host IDs. - Set uniqueHostIDs = new HashSet<>(responseBuilder.getHostEventCounts().size()); - for (HostEventCount hec : responseBuilder.getHostEventCounts()) { - uniqueHostIDs.add(hec.hostID); - } + List> slicedEventIds = sliced(new ArrayList<>(stacktraceIds), sliceCount); StackTraceHandler handler = new StackTraceHandler( submitTask, @@ -468,16 +480,16 @@ private void retrieveStackTraces( client, responseBuilder, submitListener, - eventIds.size(), + stacktraceIds.size(), // We need to expect a set of slices for each resolved index, plus one for the host metadata. - slicedEventIds.size() * indices.size() + (uniqueHostIDs.isEmpty() ? 0 : 1), - uniqueHostIDs.size() + slicedEventIds.size() * indices.size() + (hostIds.isEmpty() ? 0 : 1), + hostIds.size() ); for (List slice : slicedEventIds) { mget(client, indices, slice, ActionListener.wrap(handler::onStackTraceResponse, submitListener::onFailure)); } - if (uniqueHostIDs.isEmpty()) { + if (hostIds.isEmpty()) { return; } @@ -494,7 +506,7 @@ private void retrieveStackTraces( .lt(responseBuilder.getEnd().toEpochMilli()) .format("epoch_millis") ) - .filter(QueryBuilders.termsQuery("host.id", uniqueHostIDs)) + .filter(QueryBuilders.termsQuery("host.id", hostIds)) ) .setCollapse( // Collapse on host.id to get a single host metadata for each host. @@ -574,7 +586,7 @@ public void onStackTraceResponse(MultiGetResponse multiGetItemResponses) { if (stackTracePerId.putIfAbsent(id, stacktrace) == null) { totalFrames.addAndGet(stacktrace.frameIds.length); stackFrameIds.addAll(List.of(stacktrace.frameIds)); - stacktrace.forNativeAndKernelFrames(e -> executableIds.add(e)); + stacktrace.forNativeAndKernelFrames(executableIds::add); } } } @@ -612,25 +624,13 @@ public void calculateCO2AndCosts() { responseBuilder.getAzureCostFactor(), responseBuilder.getCustomCostPerCoreHour() ); - Map events = responseBuilder.getStackTraceEvents(); - List missingStackTraces = new ArrayList<>(); - for (HostEventCount hec : responseBuilder.getHostEventCounts()) { - TraceEvent event = events.get(hec.stacktraceID); - if (event == null) { - // If this happens, hostEventsCounts and events are out of sync, which indicates a bug. - missingStackTraces.add(hec.stacktraceID); - continue; - } - event.annualCO2Tons += co2Calculator.getAnnualCO2Tons(hec.hostID, hec.count); - event.annualCostsUSD += costCalculator.annualCostsUSD(hec.hostID, hec.count); - } - log.debug(watch::report); - if (missingStackTraces.isEmpty() == false) { - StringBuilder stringBuilder = new StringBuilder(); - Strings.collectionToDelimitedStringWithLimit(missingStackTraces, ",", "", "", 80, stringBuilder); - log.warn("CO2/cost calculator: missing trace events for StackTraceID [" + stringBuilder + "]."); - } + responseBuilder.getStackTraceEvents().forEach((eventId, event) -> { + event.annualCO2Tons += co2Calculator.getAnnualCO2Tons(eventId.hostID(), event.count); + event.annualCostsUSD += costCalculator.annualCostsUSD(eventId.hostID(), event.count); + }); + + log.debug(watch::report); } public void mayFinish() { @@ -824,6 +824,4 @@ private void mget(Client client, List indices, List slice, Action .execute(new RefCountAwareThreadedActionListener<>(responseExecutor, listener)); } } - - record HostEventCount(String hostID, String stacktraceID, int count) {} } diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetTopNFunctionsAction.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetTopNFunctionsAction.java index c9c83560ed03c..3cf63bf533f49 100644 --- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetTopNFunctionsAction.java +++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetTopNFunctionsAction.java @@ -25,7 +25,9 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; public class TransportGetTopNFunctionsAction extends TransportAction { private static final Logger log = LogManager.getLogger(TransportGetTopNFunctionsAction.class); @@ -57,11 +59,20 @@ static GetTopNFunctionsResponse buildTopNFunctions(GetStackTracesResponse respon return builder.build(); } - for (StackTrace stackTrace : response.getStackTraces().values()) { + Map stackTraces = response.getStackTraces(); + AtomicInteger lostStackTraces = new AtomicInteger(); + + response.getStackTraceEvents().forEach((eventId, event) -> { Set frameGroupsPerStackTrace = new HashSet<>(); - long samples = stackTrace.count; - double annualCO2Tons = stackTrace.annualCO2Tons; - double annualCostsUSD = stackTrace.annualCostsUSD; + long samples = event.count; + double annualCO2Tons = event.annualCO2Tons; + double annualCostsUSD = event.annualCostsUSD; + + StackTrace stackTrace = stackTraces.get(eventId.stacktraceID()); + if (stackTrace == null) { + lostStackTraces.getAndIncrement(); + return; + } int frameCount = stackTrace.frameIds.length; for (int i = 0; i < frameCount; i++) { @@ -97,8 +108,8 @@ static GetTopNFunctionsResponse buildTopNFunctions(GetStackTracesResponse respon ); } TopNFunction current = builder.getTopNFunction(frameGroupId); - if (stackTrace.subGroups != null) { - current.addSubGroups(stackTrace.subGroups); + if (event.subGroups != null) { + current.addSubGroups(event.subGroups); } if (frameGroupsPerStackTrace.contains(frameGroupId) == false) { frameGroupsPerStackTrace.add(frameGroupId); @@ -116,8 +127,7 @@ static GetTopNFunctionsResponse buildTopNFunctions(GetStackTracesResponse respon } }); } - } - + }); return builder.build(); } diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/persistence/ProfilingIndexTemplateRegistry.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/persistence/ProfilingIndexTemplateRegistry.java index 71e8dcbff4ee6..15ab1b0227f54 100644 --- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/persistence/ProfilingIndexTemplateRegistry.java +++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/persistence/ProfilingIndexTemplateRegistry.java @@ -54,7 +54,8 @@ public class ProfilingIndexTemplateRegistry extends IndexTemplateRegistry { // version 11: Added 'profiling.agent.protocol' keyword mapping to profiling-hosts // version 12: Added 'profiling.agent.env_https_proxy' keyword mapping to profiling-hosts // version 13: Added 'container.id' keyword mapping to profiling-events - public static final int INDEX_TEMPLATE_VERSION = 14; + // version 15: Added 'profiling.executable.name' keyword mapping to profiling-events + public static final int INDEX_TEMPLATE_VERSION = 15; // history for individual indices / index templates. Only bump these for breaking changes that require to create a new index public static final int PROFILING_EVENTS_VERSION = 5; diff --git a/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/GetStackTracesResponseTests.java b/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/GetStackTracesResponseTests.java index 973f9ce3df820..9cfc529ec6814 100644 --- a/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/GetStackTracesResponseTests.java +++ b/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/GetStackTracesResponseTests.java @@ -27,10 +27,7 @@ private GetStackTracesResponse createTestInstance() { new int[] { 1083999 }, new String[] { "QCCDqjSg3bMK1C4YRK6Tiw" }, new String[] { "QCCDqjSg3bMK1C4YRK6TiwAAAAAAEIpf" }, - new int[] { 2 }, - 0.3d, - 2.7d, - 1 + new int[] { 2 } ) ) ); @@ -49,15 +46,17 @@ private GetStackTracesResponse createTestInstance() { Map executables = randomNullable(Map.of("QCCDqjSg3bMK1C4YRK6Tiw", "libc.so.6")); long totalSamples = randomLongBetween(1L, 200L); String stackTraceID = randomAlphaOfLength(12); - Map stackTraceEvents = randomNullable(Map.of(stackTraceID, new TraceEvent(stackTraceID, totalSamples))); + Map stackTraceEvents = randomNullable( + Map.of(new TraceEventID("", "", "", stackTraceID), new TraceEvent(totalSamples)) + ); return new GetStackTracesResponse(stackTraces, stackFrames, executables, stackTraceEvents, totalFrames, 1.0, totalSamples); } public void testChunking() { AbstractChunkedSerializingTestCase.assertChunkCount(createTestInstance(), instance -> { - // start, end, total_frames, samplingrate - int chunks = 4; + // start and {total_frames, sampling_rate; end}; see GetStackTracesResponse.toXContentChunked() + int chunks = 2; chunks += size(instance.getExecutables()); chunks += size(instance.getStackFrames()); chunks += size(instance.getStackTraces()); diff --git a/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/StackTraceTests.java b/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/StackTraceTests.java index ee85c4b9cb01f..7044d7b73daef 100644 --- a/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/StackTraceTests.java +++ b/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/StackTraceTests.java @@ -96,11 +96,11 @@ public void testToXContent() throws IOException { new int[] { 1027822 }, new String[] { "AAAAAAAAAAUAAAAAAAAB3g" }, new String[] { "AAAAAAAAAAUAAAAAAAAB3gAAAAAAD67u" }, - new int[] { 2 }, - 0.3d, - 2.7d, - 1 + new int[] { 2 } ); + stackTrace.annualCO2Tons = 0.3d; + stackTrace.annualCostsUSD = 2.7d; + stackTrace.count = 1; stackTrace.toXContent(actualRequest, ToXContent.EMPTY_PARAMS); assertToXContentEquivalent(BytesReference.bytes(expectedRequest), BytesReference.bytes(actualRequest), contentType); @@ -111,10 +111,7 @@ public void testEquality() { new int[] { 102782 }, new String[] { "AAAAAAAAAAUAAAAAAAAB3g" }, new String[] { "AAAAAAAAAAUAAAAAAAAB3gAAAAAAD67u" }, - new int[] { 2 }, - 0.3d, - 2.7d, - 1 + new int[] { 2 } ); EqualsHashCodeTestUtils.checkEqualsAndHashCode( @@ -123,10 +120,7 @@ public void testEquality() { Arrays.copyOf(o.addressOrLines, o.addressOrLines.length), Arrays.copyOf(o.fileIds, o.fileIds.length), Arrays.copyOf(o.frameIds, o.frameIds.length), - Arrays.copyOf(o.typeIds, o.typeIds.length), - 0.3d, - 2.7d, - 1 + Arrays.copyOf(o.typeIds, o.typeIds.length) )) ); } diff --git a/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/SubGroupCollectorTests.java b/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/SubGroupCollectorTests.java index 73164ed9069d9..c4971441ff402 100644 --- a/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/SubGroupCollectorTests.java +++ b/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/SubGroupCollectorTests.java @@ -19,7 +19,7 @@ public class SubGroupCollectorTests extends ESTestCase { public void testNoAggs() { TermsAggregationBuilder stackTraces = new TermsAggregationBuilder("stacktraces").field("stacktrace.id"); - TraceEvent traceEvent = new TraceEvent("1"); + TraceEvent traceEvent = new TraceEvent(1L); SubGroupCollector collector = SubGroupCollector.attach(stackTraces, new String[0]); assertTrue("Sub aggregations attached", stackTraces.getSubAggregations().isEmpty()); @@ -32,7 +32,7 @@ public void testNoAggs() { public void testMultipleAggsInSingleStackTrace() { TermsAggregationBuilder stackTraces = new TermsAggregationBuilder("stacktraces").field("stacktrace.id"); - TraceEvent traceEvent = new TraceEvent("1"); + TraceEvent traceEvent = new TraceEvent(1L); SubGroupCollector collector = SubGroupCollector.attach(stackTraces, new String[] { "service.name", "transaction.name" }); assertFalse("No sub aggregations attached", stackTraces.getSubAggregations().isEmpty()); @@ -71,7 +71,7 @@ public void testMultipleAggsInSingleStackTrace() { public void testSingleAggInMultipleStackTraces() { TermsAggregationBuilder stackTraces = new TermsAggregationBuilder("stacktraces").field("stacktrace.id"); - TraceEvent traceEvent = new TraceEvent("1"); + TraceEvent traceEvent = new TraceEvent(1L); SubGroupCollector collector = SubGroupCollector.attach(stackTraces, new String[] { "service.name" }); assertFalse("No sub aggregations attached", stackTraces.getSubAggregations().isEmpty()); diff --git a/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/TransportGetFlamegraphActionTests.java b/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/TransportGetFlamegraphActionTests.java index 46d8df0a91bbd..797ea54c0a43b 100644 --- a/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/TransportGetFlamegraphActionTests.java +++ b/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/TransportGetFlamegraphActionTests.java @@ -14,6 +14,10 @@ public class TransportGetFlamegraphActionTests extends ESTestCase { public void testCreateFlamegraph() { + TraceEvent traceEvent = new TraceEvent(1L); + traceEvent.annualCO2Tons = 0.3d; + traceEvent.annualCostsUSD = 2.7d; + GetStackTracesResponse stacktraces = new GetStackTracesResponse( Map.of( "2buqP1GpF-TXYmL4USW8gA", @@ -39,42 +43,43 @@ public void testCreateFlamegraph() { "fr28zxcZ2UDasxYuu6dV-wAAAAAAxjUZ", "fr28zxcZ2UDasxYuu6dV-wAAAAAA0Gra", "fr28zxcZ2UDasxYuu6dV-wAAAAAA-VK9" }, - new int[] { 3, 3, 3, 3, 3, 3, 3, 3, 3 }, - 0.3d, - 2.7d, - 1 + new int[] { 3, 3, 3, 3, 3, 3, 3, 3, 3 } ) ), Map.of(), Map.of("fr28zxcZ2UDasxYuu6dV-w", "containerd"), - Map.of("2buqP1GpF-TXYmL4USW8gA", new TraceEvent("2buqP1GpF-TXYmL4USW8gA", 1L)), + Map.of(new TraceEventID("", "", "", "2buqP1GpF-TXYmL4USW8gA"), traceEvent), 9, 1.0d, 1 ); GetFlamegraphResponse response = TransportGetFlamegraphAction.buildFlamegraph(stacktraces); assertNotNull(response); - assertEquals(10, response.getSize()); + assertEquals(12, response.getSize()); assertEquals(1.0d, response.getSamplingRate(), 0.001d); - assertEquals(List.of(1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L), response.getCountInclusive()); - assertEquals(List.of(0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 1L), response.getCountExclusive()); + assertEquals(List.of(1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L), response.getCountInclusive()); + assertEquals(List.of(0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 1L), response.getCountExclusive()); assertEquals( List.of( - Map.of("174640828", 1), - Map.of("181190529", 2), - Map.of("181192637", 3), - Map.of("180652335", 4), - Map.of("180479184", 5), - Map.of("180475689", 6), - Map.of("174846197", 7), - Map.of("175515318", 8), - Map.of("178196121", 9), + Map.of("1218", 1), + Map.of("1219", 2), + Map.of("174640828", 3), + Map.of("181190529", 4), + Map.of("181192637", 5), + Map.of("180652335", 6), + Map.of("180479184", 7), + Map.of("180475689", 8), + Map.of("174846197", 9), + Map.of("175515318", 10), + Map.of("178196121", 11), Map.of() ), response.getEdges() ); assertEquals( List.of( + "", + "", "", "fr28zxcZ2UDasxYuu6dV-w", "fr28zxcZ2UDasxYuu6dV-w", @@ -88,10 +93,31 @@ public void testCreateFlamegraph() { ), response.getFileIds() ); - assertEquals(List.of(0, 3, 3, 3, 3, 3, 3, 3, 3, 3), response.getFrameTypes()); - assertEquals(List.of(false, false, false, false, false, false, false, false, false, false), response.getInlineFrames()); assertEquals( List.of( + TransportGetFlamegraphAction.FRAMETYPE_ROOT, + TransportGetFlamegraphAction.FRAMETYPE_PROCESS, + TransportGetFlamegraphAction.FRAMETYPE_THREAD, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3 + ), + response.getFrameTypes() + ); + assertEquals( + List.of(false, false, false, false, false, false, false, false, false, false, false, false), + response.getInlineFrames() + ); + assertEquals( + List.of( + "", + "", "", "containerd", "containerd", @@ -106,15 +132,15 @@ public void testCreateFlamegraph() { response.getFileNames() ); assertEquals( - List.of(0, 12784352, 19334053, 19336161, 18795859, 18622708, 18619213, 12989721, 13658842, 16339645), + List.of(0, 0, 0, 12784352, 19334053, 19336161, 18795859, 18622708, 18619213, 12989721, 13658842, 16339645), response.getAddressOrLines() ); - assertEquals(List.of("", "", "", "", "", "", "", "", "", ""), response.getFunctionNames()); - assertEquals(List.of(0, 0, 0, 0, 0, 0, 0, 0, 0, 0), response.getFunctionOffsets()); - assertEquals(List.of("", "", "", "", "", "", "", "", "", ""), response.getSourceFileNames()); - assertEquals(List.of(0, 0, 0, 0, 0, 0, 0, 0, 0, 0), response.getSourceLines()); + assertEquals(List.of("", "", "", "", "", "", "", "", "", "", "", ""), response.getFunctionNames()); + assertEquals(List.of(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0), response.getFunctionOffsets()); + assertEquals(List.of("", "", "", "", "", "", "", "", "", "", "", ""), response.getSourceFileNames()); + assertEquals(List.of(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0), response.getSourceLines()); assertEquals(1L, response.getSelfCPU()); - assertEquals(10L, response.getTotalCPU()); + assertEquals(12L, response.getTotalCPU()); assertEquals(1L, response.getTotalSamples()); } @@ -128,7 +154,7 @@ public void testCreateEmptyFlamegraphWithRootNode() { assertEquals(List.of(0L), response.getCountExclusive()); assertEquals(List.of(Map.of()), response.getEdges()); assertEquals(List.of(""), response.getFileIds()); - assertEquals(List.of(0), response.getFrameTypes()); + assertEquals(List.of(TransportGetFlamegraphAction.FRAMETYPE_ROOT), response.getFrameTypes()); assertEquals(List.of(false), response.getInlineFrames()); assertEquals(List.of(""), response.getFileNames()); assertEquals(List.of(0), response.getAddressOrLines()); diff --git a/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/TransportGetTopNFunctionsActionTests.java b/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/TransportGetTopNFunctionsActionTests.java index 2fcf961f9b9a5..49311c0719222 100644 --- a/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/TransportGetTopNFunctionsActionTests.java +++ b/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/TransportGetTopNFunctionsActionTests.java @@ -14,6 +14,10 @@ public class TransportGetTopNFunctionsActionTests extends ESTestCase { public void testCreateAllTopNFunctions() { + TraceEvent traceEvent = new TraceEvent(1L); + traceEvent.annualCO2Tons = 0.3d; + traceEvent.annualCostsUSD = 2.7d; + GetStackTracesResponse stacktraces = new GetStackTracesResponse( Map.of( "2buqP1GpF-TXYmL4USW8gA", @@ -39,15 +43,12 @@ public void testCreateAllTopNFunctions() { "fr28zxcZ2UDasxYuu6dV-wAAAAAAxjUZ", "fr28zxcZ2UDasxYuu6dV-wAAAAAA0Gra", "fr28zxcZ2UDasxYuu6dV-wAAAAAA-VK9" }, - new int[] { 3, 3, 3, 3, 3, 3, 3, 3, 3 }, - 0.3d, - 2.7d, - 1 + new int[] { 3, 3, 3, 3, 3, 3, 3, 3, 3 } ) ), Map.of(), Map.of("fr28zxcZ2UDasxYuu6dV-w", "containerd"), - Map.of("2buqP1GpF-TXYmL4USW8gA", new TraceEvent("2buqP1GpF-TXYmL4USW8gA", 1L)), + Map.of(new TraceEventID("", "", "", "2buqP1GpF-TXYmL4USW8gA"), traceEvent), 9, 1.0d, 1 @@ -79,6 +80,10 @@ public void testCreateAllTopNFunctions() { } public void testCreateTopNFunctionsWithLimit() { + TraceEvent traceEvent = new TraceEvent(1L); + traceEvent.annualCO2Tons = 0.3d; + traceEvent.annualCostsUSD = 2.7d; + GetStackTracesResponse stacktraces = new GetStackTracesResponse( Map.of( "2buqP1GpF-TXYmL4USW8gA", @@ -104,15 +109,12 @@ public void testCreateTopNFunctionsWithLimit() { "fr28zxcZ2UDasxYuu6dV-wAAAAAAxjUZ", "fr28zxcZ2UDasxYuu6dV-wAAAAAA0Gra", "fr28zxcZ2UDasxYuu6dV-wAAAAAA-VK9" }, - new int[] { 3, 3, 3, 3, 3, 3, 3, 3, 3 }, - 0.3d, - 2.7d, - 1 + new int[] { 3, 3, 3, 3, 3, 3, 3, 3, 3 } ) ), Map.of(), Map.of("fr28zxcZ2UDasxYuu6dV-w", "containerd"), - Map.of("2buqP1GpF-TXYmL4USW8gA", new TraceEvent("2buqP1GpF-TXYmL4USW8gA", 1L)), + Map.of(new TraceEventID("", "", "", "2buqP1GpF-TXYmL4USW8gA"), traceEvent), 9, 1.0d, 1 diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/profiling/10_basic.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/profiling/10_basic.yml index 4a9212b8a7158..7023e915dcf23 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/profiling/10_basic.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/profiling/10_basic.yml @@ -244,7 +244,7 @@ teardown: } } } - - match: { Size: 47} + - match: { Size: 49} --- "Test flamegraph from test-events": @@ -272,7 +272,7 @@ teardown: } } } - - match: { Size: 47} + - match: { Size: 49} --- "Test topN functions from profiling-events":