From d2b1059d09c77af2c1ed5734fdc0da58d24f16ef Mon Sep 17 00:00:00 2001
From: Travis Downs <travis.downs@redpanda.com>
Date: Thu, 23 Mar 2023 11:46:29 -0300
Subject: [PATCH] Grow the histogram buffer if it is too small
When we serialize a histogram to a byte array, the intermediate
ByteBuffer that we pass may be too small which may result in silent
truncation of the encoded histogram.
This will manifest on the driver side as a decoding failure.
This change detects this case and grows the buffer by a
factor of 2 until it fits.
Fixes openmessaging/benchmark#369.
---
.../worker/jackson/HistogramSerializer.java | 38 ++++++++++++++++---
1 file changed, 32 insertions(+), 6 deletions(-)
diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/jackson/HistogramSerializer.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/jackson/HistogramSerializer.java
index a1b61afa3..02287393c 100644
--- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/jackson/HistogramSerializer.java
+++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/jackson/HistogramSerializer.java
@@ -17,6 +17,8 @@
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.ser.std.StdSerializer;
+import com.google.common.base.Preconditions;
+
import java.io.IOException;
import java.nio.ByteBuffer;
import org.HdrHistogram.Histogram;
@@ -30,16 +32,40 @@ public HistogramSerializer() {
super(Histogram.class);
}
+ static byte[] toByteArray(ByteBuffer buffer) {
+ byte[] encodedBuffer = new byte[buffer.remaining()];
+ buffer.get(encodedBuffer);
+ return encodedBuffer;
+ }
+
+ static ByteBuffer serializeHistogram(Histogram histo, ByteBuffer buffer) {
+ buffer.clear();
+ while (true) {
+ final int outBytes = histo.encodeIntoCompressedByteBuffer(buffer);
+ Preconditions.checkState(outBytes == buffer.position());
+ final int capacity = buffer.capacity();
+ if (outBytes < capacity) {
+ // encoding succesful
+ break;
+ }
+ // We filled the entire buffer, an indication that the buffer was not
+ // large enough, so we double the buffer and try again.
+ // See: https://github.com/HdrHistogram/HdrHistogram/issues/201
+ buffer = ByteBuffer.allocate(capacity * 2);
+ }
+ buffer.flip();
+ return buffer;
+ }
+
@Override
public void serialize(
Histogram histogram, JsonGenerator jsonGenerator, SerializerProvider serializerProvider)
throws IOException {
ByteBuffer buffer = threadBuffer.get();
- buffer.clear();
- histogram.encodeIntoCompressedByteBuffer(buffer);
- byte[] arr = new byte[buffer.position()];
- buffer.flip();
- buffer.get(arr);
- jsonGenerator.writeBinary(arr);
+ ByteBuffer newBuffer = serializeHistogram(histogram, buffer);
+ if (newBuffer != buffer) {
+ threadBuffer.set(newBuffer);
+ }
+ jsonGenerator.writeBinary(toByteArray(newBuffer));
}
}