VarInt reading optimizations

src/main/java/ua/nanit/limbo/connection/pipeline/VarIntFrameDecoder.java

@@ -20,7 +20,9 @@
 import io.netty.buffer.ByteBuf;
 import io.netty.channel.ChannelHandlerContext;
 import io.netty.handler.codec.ByteToMessageDecoder;
-import ua.nanit.limbo.server.Log;
+import io.netty.handler.codec.DecoderException;
+import io.netty.util.ByteProcessor;
+import org.jetbrains.annotations.NotNull;
 
 import java.util.List;
 
@@ -33,28 +35,81 @@ protected void decode(ChannelHandlerContext ctx, ByteBuf in, List<Object> out) {
             return;
         }
 
-        VarIntByteDecoder reader = new VarIntByteDecoder();
-        int varIntEnd = in.forEachByte(reader);
+        // Skip any runs of 0x00 we might find
+        final int packetStart = in.forEachByte(ByteProcessor.FIND_NON_NUL);
+        if (packetStart == -1) {
+            return;
+        }
+        in.readerIndex(packetStart);
 
-        if (varIntEnd == -1) return;
+        // Try to read the length of the packet
+        in.markReaderIndex();
+        final int preIndex = in.readerIndex();
+        final int length = readRawVarInt21(in);
+        if (preIndex == in.readerIndex()) {
+            return;
+        }
+        if (length <= 0) {
+            throw new DecoderException("Bad VarInt length: " + length);
+        }
 
-        if (reader.getResult() == VarIntByteDecoder.DecodeResult.SUCCESS) {
-            int readVarInt = reader.getReadVarInt();
-            int bytesRead = reader.getBytesRead();
-            if (readVarInt < 0) {
-                Log.error("[VarIntFrameDecoder] Bad data length");
-            } else if (readVarInt == 0) {
-                in.readerIndex(varIntEnd + 1);
-            } else {
-                int minimumRead = bytesRead + readVarInt;
+        if (in.readableBytes() < length) {
+            in.resetReaderIndex();
+        } else {
+            out.add(in.readRetainedSlice(length));
+        }
+    }
+
+    private static int readRawVarInt21(final @NotNull ByteBuf byteBuf) {
+        if (byteBuf.readableBytes() < 4) {
+            return readRawVarIntSmallBuffer(byteBuf);
+        }
+
+        // take the last three bytes and check if any of them have the high bit set
+        final int wholeOrMore = byteBuf.getIntLE(byteBuf.readerIndex());
+        final int atStop = ~wholeOrMore & 0x808080;
+        if (atStop == 0) {
+            throw new DecoderException("VarInt too big");
+        }
 
-                if (in.isReadable(minimumRead)) {
-                    out.add(in.retainedSlice(varIntEnd + 1, readVarInt));
-                    in.skipBytes(minimumRead);
-                }
-            }
-        } else if (reader.getResult() == VarIntByteDecoder.DecodeResult.TOO_BIG) {
-            Log.error("[VarIntFrameDecoder] Too big data");
+        final int bitsToKeep = Integer.numberOfTrailingZeros(atStop) + 1;
+        byteBuf.skipBytes(bitsToKeep >> 3);
+
+        // https://github.com/netty/netty/pull/14050#issuecomment-2107750734
+        int preservedBytes = wholeOrMore & (atStop ^ (atStop - 1));
+
+        // https://github.com/netty/netty/pull/14050#discussion_r1597896639
+        preservedBytes = (preservedBytes & 0x007F007F) | ((preservedBytes & 0x00007F00) >> 1);
+        preservedBytes = (preservedBytes & 0x00003FFF) | ((preservedBytes & 0x3FFF0000) >> 2);
+        return preservedBytes;
+    }
+
+    private static int readRawVarIntSmallBuffer(final @NotNull ByteBuf byteBuf) {
+        if (!byteBuf.isReadable()) {
+            return 0;
+        }
+        byteBuf.markReaderIndex();
+
+        byte tmp = byteBuf.readByte();
+        if (tmp >= 0) {
+            return tmp;
+        }
+        int result = tmp & 0x7F;
+        if (!byteBuf.isReadable()) {
+            byteBuf.resetReaderIndex();
+            return 0;
+        }
+        if ((tmp = byteBuf.readByte()) >= 0) {
+            return result | tmp << 7;
+        }
+        result |= (tmp & 0x7F) << 7;
+        if (!byteBuf.isReadable()) {
+            byteBuf.resetReaderIndex();
+            return 0;
+        }
+        if ((tmp = byteBuf.readByte()) >= 0) {
+            return result | tmp << 14;
         }
+        return result | (tmp & 0x7F) << 14;
     }
 }

src/main/java/ua/nanit/limbo/protocol/ByteMessage.java

@@ -53,23 +53,29 @@ public byte[] toByteArray() {
         return bytes;
     }
 
-    /* Minecraft's protocol methods */
-
     public int readVarInt() {
-        int i = 0;
-        int maxRead = Math.min(5, buf.readableBytes());
+        final int readable = buf.readableBytes();
+        if (readable == 0) {
+            throw new DecoderException("Empty buffer");
+        }
+
+        // We can read at least one byte, and this should be a common case
+        int k = buf.readByte();
+        if ((k & 0x80) != 128) {
+            return k;
+        }
 
-        for (int j = 0; j < maxRead; j++) {
-            int k = buf.readByte();
+        // In case decoding one byte was not enough, use a loop to decode up to the next 4 bytes
+        final int maxRead = Math.min(5, readable);
+        int i = k & 0x7F;
+        for (int j = 1; j < maxRead; j++) {
+            k = buf.readByte();
             i |= (k & 0x7F) << j * 7;
             if ((k & 0x80) != 128) {
                 return i;
             }
         }
-
-        buf.readBytes(maxRead);
-
-        throw new IllegalArgumentException("Cannot read VarInt");
+        throw new DecoderException("Bad VarInt");
     }
 
     public void writeVarInt(int value) {