Add support to 32 bytes chain work

core/src/main/java/org/bitcoinj/core/CheckpointManager.java

@@ -60,14 +60,14 @@
  * different concept of checkpoints that are used to hard-code the validity of blocks that violate BIP30 (duplicate
  * coinbase transactions). Those "checkpoints" can be found in NetworkParameters.</p>
  *
- * <p>The file format consists of the string "CHECKPOINTS 1", followed by a uint32 containing the number of signatures
- * to read. The value may not be larger than 256 (so it could have been a byte but isn't for historical reasons).
+ * <p>Checkpoints are read from a text file, one value per line.
+ * It consists of the magic string "TXT CHECKPOINTS 1", followed by the number of signatures
+ * to read. The value may not be larger than 256.
  * If the number of signatures is larger than zero, each 65 byte ECDSA secp256k1 signature then follows. The signatures
  * sign the hash of all bytes that follow the last signature.</p>
  *
- * <p>After the signatures come an int32 containing the number of checkpoints in the file. Then each checkpoint follows
- * one after the other. A checkpoint is 12 bytes for the total work done field, 4 bytes for the height, 80 bytes
- * for the block header and then 1 zero byte at the end (i.e. number of transactions in the block: always zero).</p>
+ * <p>After the signatures come the number of checkpoints in the file. Then each checkpoint follows one per line in
+ * compact format (as written by {@link StoredBlock#serializeCompactV2(ByteBuffer)}) as a base64-encoded blob.</p>
  */
 public class CheckpointManager {
     private static final Logger log = LoggerFactory.getLogger(CheckpointManager.class);
@@ -112,6 +112,11 @@ public static InputStream openStream(NetworkParameters params) {
         return CheckpointManager.class.getResourceAsStream("/" + params.getId() + ".checkpoints.txt");
     }
 
+    /** @deprecated Use {@link #readTextual(InputStream)}
+     * The binary format does not support mixed stored block sizes.
+     * After implementing support to 32-byte chain work to StoredBlock class,
+    this method cannot read blocks which chain work surpassed 12 byte. */
+    @Deprecated
     private Sha256Hash readBinary(InputStream inputStream) throws IOException {
         DataInputStream dis = null;
         try {
@@ -132,15 +137,26 @@ private Sha256Hash readBinary(InputStream inputStream) throws IOException {
             digestInputStream.on(true);
             int numCheckpoints = dis.readInt();
             checkState(numCheckpoints > 0);
-            final int size = StoredBlock.COMPACT_SERIALIZED_SIZE;
+            final int size = StoredBlock.COMPACT_SERIALIZED_SIZE_LEGACY;
             ByteBuffer buffer = ByteBuffer.allocate(size);
             for (int i = 0; i < numCheckpoints; i++) {
                 if (dis.read(buffer.array(), 0, size) < size)
                     throw new IOException("Incomplete read whilst loading checkpoints.");
-                StoredBlock block = StoredBlock.deserializeCompact(params, buffer);
+                StoredBlock block = StoredBlock.deserializeCompactLegacy(params, buffer);
                 buffer.position(0);
                 checkpoints.put(block.getHeader().getTimeSeconds(), block);
             }
+
+            int actualCheckpointsSize = dis.available();
+            int expectedCheckpointsSize = numCheckpoints * size;
+            // Check if there are any bytes left in the stream. If it does, it means that checkpoints are malformed
+            if (actualCheckpointsSize > 0) {
+                String message = String.format(
+                    "Checkpoints size did not match size for version 1 format. Expected checkpoints %d with size of %d bytes, but actual size was %d.",
+                    numCheckpoints, expectedCheckpointsSize, actualCheckpointsSize);
+                throw new IOException(message);
+            }
+
             Sha256Hash dataHash = Sha256Hash.wrap(digest.digest());
             log.info("Read {} checkpoints up to time {}, hash is {}", checkpoints.size(),
                     Utils.dateTimeFormat(checkpoints.lastEntry().getKey() * 1000), dataHash);
@@ -168,15 +184,18 @@ private Sha256Hash readTextual(InputStream inputStream) throws IOException {
             checkState(numCheckpoints > 0);
             // Hash numCheckpoints in a way compatible to the binary format.
             hasher.putBytes(ByteBuffer.allocate(4).order(ByteOrder.BIG_ENDIAN).putInt(numCheckpoints).array());
-            final int size = StoredBlock.COMPACT_SERIALIZED_SIZE;
-            ByteBuffer buffer = ByteBuffer.allocate(size);
             for (int i = 0; i < numCheckpoints; i++) {
                 byte[] bytes = BASE64.decode(reader.readLine());
                 hasher.putBytes(bytes);
-                buffer.position(0);
-                buffer.put(bytes);
-                buffer.position(0);
-                StoredBlock block = StoredBlock.deserializeCompact(params, buffer);
+                ByteBuffer buffer = ByteBuffer.wrap(bytes);
+                StoredBlock block;
+                if (bytes.length == StoredBlock.COMPACT_SERIALIZED_SIZE_LEGACY) {
+                    block = StoredBlock.deserializeCompactLegacy(params, buffer);
+                } else if (bytes.length == StoredBlock.COMPACT_SERIALIZED_SIZE_V2) {
+                    block = StoredBlock.deserializeCompactV2(params, buffer);
+                } else {
+                    throw new IllegalStateException("unexpected length of checkpoint: " + bytes.length);
+                }
                 checkpoints.put(block.getHeader().getTimeSeconds(), block);
             }
             HashCode hash = hasher.hash();

core/src/main/java/org/bitcoinj/core/StoredBlock.java

@@ -35,11 +35,20 @@
  */
 public class StoredBlock {
 
-    // A BigInteger representing the total amount of work done so far on this chain. As of May 2011 it takes 8
-    // bytes to represent this field, so 12 bytes should be plenty for now.
-    public static final int CHAIN_WORK_BYTES = 12;
-    public static final byte[] EMPTY_BYTES = new byte[CHAIN_WORK_BYTES];
-    public static final int COMPACT_SERIALIZED_SIZE = Block.HEADER_SIZE + CHAIN_WORK_BYTES + 4;  // for height
+    /* @deprecated Use {@link #CHAIN_WORK_BYTES_V2} instead.
+        Size in bytes to represent the total amount of work done so far on this chain. As of June 22, 2024, it takes 12
+        unsigned bytes to store this value, so developers should use the V2 format.
+     */
+    private static final int CHAIN_WORK_BYTES_LEGACY = 12;
+    // Size in bytes to represent the total amount of work done so far on this chain.
+    private static final int CHAIN_WORK_BYTES_V2 = 32;
+    // Size in bytes(int) to represent btc block height
+    private static final int HEIGHT_BYTES = 4;
+
+    // Size in bytes of serialized block in legacy format by {@link #serializeCompactLegacy(ByteBuffer)}
+    public static final int COMPACT_SERIALIZED_SIZE_LEGACY = Block.HEADER_SIZE + CHAIN_WORK_BYTES_LEGACY + HEIGHT_BYTES;
+    // Size in bytes of serialized block in V2 format by {@link #serializeCompactV2(ByteBuffer)}
+    public static final int COMPACT_SERIALIZED_SIZE_V2 = Block.HEADER_SIZE + CHAIN_WORK_BYTES_V2 + HEIGHT_BYTES;
 
     private Block header;
     private BigInteger chainWork;
@@ -113,13 +122,32 @@ public StoredBlock getPrev(BlockStore store) throws BlockStoreException {
         return store.get(getHeader().getPrevBlockHash());
     }
 
-    /** Serializes the stored block to a custom packed format. Used by {@link CheckpointManager}. */
-    public void serializeCompact(ByteBuffer buffer) {
-        byte[] chainWorkBytes = Utils.bigIntegerToBytes(getChainWork(), CHAIN_WORK_BYTES);
-        if (chainWorkBytes.length < CHAIN_WORK_BYTES) {
-            // Pad to the right size.
-            buffer.put(EMPTY_BYTES, 0, CHAIN_WORK_BYTES - chainWorkBytes.length);
-        }
+    /**
+     * @deprecated Use {@link #serializeCompactV2(ByteBuffer)} instead.
+     *
+     * Serializes the stored block to a custom packed format. Used internally.
+     * As of June 22, 2024, it takes 12 unsigned bytes to store the chain work value,
+     * so developers should use the V2 format.
+     *
+     * @param buffer buffer to write to
+     */
+    @Deprecated
+    public void serializeCompactLegacy(ByteBuffer buffer) {
+        serializeCompact(buffer, CHAIN_WORK_BYTES_LEGACY);
+
+    }
+
+    /**
+     * Serializes the stored block to a custom packed format. Used internally.
+     *
+     * @param buffer buffer to write to
+     */
+    public void serializeCompactV2(ByteBuffer buffer) {
+        serializeCompact(buffer, CHAIN_WORK_BYTES_V2);
+    }
+
+    private void serializeCompact(ByteBuffer buffer, int chainWorkSize) {
+        byte[] chainWorkBytes = Utils.bigIntegerToBytes(getChainWork(), chainWorkSize);
         buffer.put(chainWorkBytes);
         buffer.putInt(getHeight());
         // Using unsafeBitcoinSerialize here can give us direct access to the same bytes we read off the wire,
@@ -128,9 +156,34 @@ public void serializeCompact(ByteBuffer buffer) {
         buffer.put(bytes, 0, Block.HEADER_SIZE);  // Trim the trailing 00 byte (zero transactions).
     }
 
-    /** De-serializes the stored block from a custom packed format. Used by {@link CheckpointManager}. */
-    public static StoredBlock deserializeCompact(NetworkParameters params, ByteBuffer buffer) throws ProtocolException {
-        byte[] chainWorkBytes = new byte[StoredBlock.CHAIN_WORK_BYTES];
+    /**
+     * @deprecated Use {@link #deserializeCompactV2(NetworkParameters, ByteBuffer)} instead.
+     *
+     * Deserializes the stored block from a custom packed format. Used internally.
+     * As of June 22, 2024, it takes 12 unsigned bytes to store the chain work value,
+     * so developers should use the V2 format.
+     *
+     * @param buffer data to deserialize
+     * @return deserialized stored block
+     */
+    @Deprecated
+    public static StoredBlock deserializeCompactLegacy(NetworkParameters params, ByteBuffer buffer) throws ProtocolException {
+        return deserializeCompact(params, buffer, StoredBlock.CHAIN_WORK_BYTES_LEGACY);
+    }
+
+    /**
+     * Deserializes the stored block from a custom packed format. Used internally.
+     *
+     * @param buffer data to deserialize
+     * @return deserialized stored block
+     */
+    public static StoredBlock deserializeCompactV2(NetworkParameters params, ByteBuffer buffer) throws ProtocolException {
+        return deserializeCompact(params, buffer, StoredBlock.CHAIN_WORK_BYTES_V2);
+    }
+
+    private static StoredBlock deserializeCompact(NetworkParameters params, ByteBuffer buffer,
+        int chainWorkSize) {
+        byte[] chainWorkBytes = new byte[chainWorkSize];
         buffer.get(chainWorkBytes);
         BigInteger chainWork = new BigInteger(1, chainWorkBytes);
         int height = buffer.getInt();  // +4 bytes

core/src/main/java/org/bitcoinj/store/LevelDBBlockStore.java

@@ -16,6 +16,7 @@
 
 package org.bitcoinj.store;
 
+import java.math.BigInteger;
 import org.bitcoinj.core.*;
 import org.fusesource.leveldbjni.*;
 import org.iq80.leveldb.*;
@@ -35,7 +36,10 @@ public class LevelDBBlockStore implements BlockStore {
 
     private final Context context;
     private DB db;
-    private final ByteBuffer buffer = ByteBuffer.allocate(StoredBlock.COMPACT_SERIALIZED_SIZE);
+
+    private final ByteBuffer legacyBuffer = ByteBuffer.allocate(StoredBlock.COMPACT_SERIALIZED_SIZE_LEGACY);
+    private final ByteBuffer bufferV2 = ByteBuffer.allocate(StoredBlock.COMPACT_SERIALIZED_SIZE_V2);
+
     private final File path;
 
     /** Creates a LevelDB SPV block store using the JNI/C++ version of LevelDB. */
@@ -76,19 +80,33 @@ private synchronized void initStoreIfNeeded() throws BlockStoreException {
         setChainHead(storedGenesis);
     }
 
+    private static final BigInteger MAX_WORK_V1 = new BigInteger(/* 12 bytes */ "ffffffffffffffffffffffff", 16);
+
     @Override
     public synchronized void put(StoredBlock block) throws BlockStoreException {
-        buffer.clear();
-        block.serializeCompact(buffer);
-        db.put(block.getHeader().getHash().getBytes(), buffer.array());
+        legacyBuffer.clear();
+        if (block.getChainWork().compareTo(MAX_WORK_V1) <= 0) {
+            legacyBuffer.rewind();
+            block.serializeCompactLegacy(legacyBuffer);
+            db.put(block.getHeader().getHash().getBytes(), legacyBuffer.array());
+        } else {
+            bufferV2.rewind();
+            block.serializeCompactV2(bufferV2);
+            db.put(block.getHeader().getHash().getBytes(), bufferV2.array());
+        }
     }
 
     @Override @Nullable
     public synchronized StoredBlock get(Sha256Hash hash) throws BlockStoreException {
         byte[] bits = db.get(hash.getBytes());
         if (bits == null)
             return null;
-        return StoredBlock.deserializeCompact(context.getParams(), ByteBuffer.wrap(bits));
+
+        if (bits.length == StoredBlock.COMPACT_SERIALIZED_SIZE_LEGACY){
+            return StoredBlock.deserializeCompactLegacy(context.getParams(), ByteBuffer.wrap(bits));
+        } else {
+            return StoredBlock.deserializeCompactV2(context.getParams(), ByteBuffer.wrap(bits));
+        }
     }
 
     @Override

core/src/main/java/org/bitcoinj/store/LevelDBFullPrunedBlockStore.java

@@ -495,7 +495,7 @@ protected void putUpdateStoredBlock(StoredBlock storedBlock, boolean wasUndoable
             beginMethod("putUpdateStoredBlock");
         Sha256Hash hash = storedBlock.getHeader().getHash();
         ByteBuffer bb = ByteBuffer.allocate(97);
-        storedBlock.serializeCompact(bb);
+        storedBlock.serializeCompactLegacy(bb);
         bb.put((byte) (wasUndoable ? 1 : 0));
         batchPut(getKey(KeyType.HEADERS_ALL, hash), bb.array());
         if (instrument)
@@ -640,7 +640,7 @@ public StoredBlock get(Sha256Hash hash, boolean wasUndoableOnly) throws BlockSto
         }
         // TODO Should I chop the last byte off? Seems to work with it left
         // there...
-        StoredBlock stored = StoredBlock.deserializeCompact(params, ByteBuffer.wrap(result));
+        StoredBlock stored = StoredBlock.deserializeCompactLegacy(params, ByteBuffer.wrap(result));
         stored.getHeader().verifyHeader();
 
         if (instrument)

core/src/main/java/org/bitcoinj/store/SPVBlockStore.java

@@ -196,7 +196,7 @@ public void put(StoredBlock block) throws BlockStoreException {
             Sha256Hash hash = block.getHeader().getHash();
             notFoundCache.remove(hash);
             buffer.put(hash.getBytes());
-            block.serializeCompact(buffer);
+            block.serializeCompactLegacy(buffer);
             setRingCursor(buffer, buffer.position());
             blockCache.put(hash, block);
         } finally { lock.unlock(); }
@@ -233,7 +233,7 @@ public StoredBlock get(Sha256Hash hash) throws BlockStoreException {
                 buffer.get(scratch);
                 if (Arrays.equals(scratch, targetHashBytes)) {
                     // Found the target.
-                    StoredBlock storedBlock = StoredBlock.deserializeCompact(params, buffer);
+                    StoredBlock storedBlock = StoredBlock.deserializeCompactLegacy(params, buffer);
                     blockCache.put(hash, storedBlock);
                     return storedBlock;
                 }
@@ -300,7 +300,7 @@ public NetworkParameters getParams() {
         return params;
     }
 
-    protected static final int RECORD_SIZE = 32 /* hash */ + StoredBlock.COMPACT_SERIALIZED_SIZE;
+    protected static final int RECORD_SIZE = 32 /* hash */ + StoredBlock.COMPACT_SERIALIZED_SIZE_LEGACY;
 
     // File format:
     //   4 header bytes = "SPVB"