Debug traces and Fix Sentinel Bug

.gitignore

@@ -1,7 +1,7 @@
 .gradle
 /build/
 !gradle/wrapper/gradle-wrapper.jar
-
+.idea
 ### vi ###
 *.swp
 

src/main/java/co/nyzo/verifier/BalanceListManager.java

@@ -4,6 +4,7 @@
 import java.nio.ByteBuffer;
 import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
+import co.nyzo.verifier.util.*;  // egg
 
 public class BalanceListManager {
 
@@ -62,41 +63,70 @@ public static BalanceList balanceListForBlock(Block block) {
 
                 // If the balance list was not in the map, try to derive it.
                 if (balanceList == null) {
+                    //System.out.println("X:BLM:Rewind from "+block.getBlockHeight()+ " " + block);
+                    /*
+                    for (ByteBuffer key: balanceListMap.keySet()) {
+                        System.out.println("X:BLM:hash="+PrintUtil.compactPrintByteArray(key.array())+" h="+balanceListMap.get(key).getBlockHeight() );
+                    }*/
 
                     // Step back to previous blocks until we are able to find a balance list that we have.
                     Block startBlock = block;
                     List<Block> blocks = new ArrayList<>(Collections.singletonList(startBlock));
                     BalanceList startBalanceList = null;
-                    while (startBlock != null && startBalanceList == null) {
-                        startBlock = startBlock.getPreviousBlock();
+                    if (balanceListMap.size() > 0) {
+                        long rewinded = 0L;
+                        while (startBlock != null && startBalanceList == null) {
+                            startBlock = startBlock.getPreviousBlock();
+                            rewinded++;
+                            if (startBlock != null) {
+                                blocks.add(0, startBlock);
+                                // System.out.println("X:BLM:Previous is "+startBlock.getBlockHeight());
+                                startBalanceList = balanceListMap.get(ByteBuffer.wrap(startBlock.getBalanceListHash()));
+                            } else {
+                                //System.out.println("X:BLM:Previous was null");
+                            }
+                        }
                         if (startBlock != null) {
-                            blocks.add(0, startBlock);
-                            startBalanceList = balanceListMap.get(ByteBuffer.wrap(startBlock.getBalanceListHash()));
+                            //System.out.println("X:BLM:Rewinded to "+startBlock);
+                        } else {
+                            System.out.println("X:BLM:Rewinded to null after " + rewinded);
                         }
+                    } else {
+                        System.out.println("X:BLM:!Can't rewind, No BL at all");
                     }
-
                     // If a suitable start balance list was found, derive the desired list.
                     if (startBalanceList != null) {
                         balanceList = startBalanceList;
                         for (int i = 0; i < blocks.size() - 1; i++) {
+                            //System.out.println("X:BLM:Adding block "+blocks.get(i) );
+                            //System.out.println("X:BLM:and block "+blocks.get(i+1) );
                             balanceList = Block.balanceListForNextBlock(blocks.get(i), balanceList,
                                     blocks.get(i + 1).getTransactions(), blocks.get(i + 1).getVerifierIdentifier(),
                                     blocks.get(i + 1).getBlockchainVersion());
                         }
-
+                        //System.out.println("X:Resulting BL Hash="+PrintUtil.compactPrintByteArray(balanceList.getHash()) );
                         registerBalanceList(balanceList);
+                    } else {
+                        System.out.println("X:BLM:Rewinded BL is null");
                     }
+                } else {
+                    //System.out.println("X:BLM:BL From map");
                 }
             }
+        } else {
+            System.out.println("X:BLM:Block is null");
         }
 
         return balanceList;
     }
 
     public static void registerBalanceList(BalanceList balanceList) {
-
-        if (balanceList != null && balanceListMap.size() < maximumMapSize) {
+        if (balanceList == null) {
+            System.out.println("X:BLM:Not registering null BL");
+        } else if (balanceListMap.size() < maximumMapSize) {
             balanceListMap.put(ByteBuffer.wrap(balanceList.getHash()), balanceList);
+        } else {
+            System.out.println("X:BLM:Not registering BL, mapsize exceeded");
         }
     }
 
@@ -116,21 +146,28 @@ public static boolean accountIsInSystem(byte[] identifier) {
     public static void updateFrozenEdge(BalanceList frozenEdgeList) {
 
         if (frozenEdgeList != null) {
-            for (int i = numberOfRecentLists - 1; i > 0; i--) {
-                recentLists[i] = recentLists[i - 1];
-            }
-            recentLists[0] = frozenEdgeList;
-            frozenEdgeBalanceMap = BalanceManager.makeBalanceMap(frozenEdgeList);
+            try {
+                for (int i = numberOfRecentLists - 1; i > 0; i--) {
+                    recentLists[i] = recentLists[i - 1];
+                }
+                recentLists[0] = frozenEdgeList;
+                frozenEdgeBalanceMap = BalanceManager.makeBalanceMap(frozenEdgeList);
 
-            Set<ByteBuffer> accountsInSystem = ConcurrentHashMap.newKeySet();
-            for (BalanceListItem item : frozenEdgeList.getItems()) {
-                accountsInSystem.add(ByteBuffer.wrap(item.getIdentifier()));
-            }
+                Set<ByteBuffer> accountsInSystem = ConcurrentHashMap.newKeySet();
+                for (BalanceListItem item : frozenEdgeList.getItems()) {
+                    accountsInSystem.add(ByteBuffer.wrap(item.getIdentifier()));
+                }
 
-            BalanceListManager.accountsInSystem = accountsInSystem;
+                BalanceListManager.accountsInSystem = accountsInSystem;
 
-            balanceListMap.clear();
-            balanceListMap.put(ByteBuffer.wrap(frozenEdgeList.getHash()), frozenEdgeList);
+                balanceListMap.clear();
+                //System.out.println("X:BLM:balanceListMap.cleared, add "+ PrintUtil.compactPrintByteArray(frozenEdgeList.getHash())+" for h="+frozenEdgeList.getBlockHeight());
+                balanceListMap.put(ByteBuffer.wrap(frozenEdgeList.getHash()), frozenEdgeList);
+            } catch (Exception e) {
+                System.out.println("X:Exception BLM.updateFrozenEdge " + PrintUtil.printException(e));
+            }
+        } else {
+            System.out.println("X:BLM:updateFrozenEdge BL is null");
         }
     }
 }

src/main/java/co/nyzo/verifier/Block.java

@@ -172,10 +172,19 @@ public Block getPreviousBlock() {
                 Block frozenBlock = BlockManager.frozenBlockForHeight(height - 1);
                 if (frozenBlock != null && ByteUtil.arraysAreEqual(frozenBlock.getHash(), previousBlockHash)) {
                     previousBlock = frozenBlock;
+                } else {
+                    if (frozenBlock == null) {
+                        System.out.println("X:GPB:Frozenblock is null");
+                    }
+                    else if (!ByteUtil.arraysAreEqual(frozenBlock.getHash(), previousBlockHash)) {
+                        System.out.println("X:GPB:arrays Are not Equal, frozen "+frozenBlock.getHash()+" vs "+previousBlockHash );
+                    }
                 }
             } else {
                 previousBlock = UnfrozenBlockManager.unfrozenBlockAtHeight(height - 1, previousBlockHash);
             }
+        } else {
+            System.out.println("X:GPB:getBlockHeight<=0");
         }
 
         return previousBlock;
@@ -620,7 +629,7 @@ public static BalanceList balanceListForNextBlock(Block previousBlock, BalanceLi
                 long organicTransactionFees = 0L;
                 long transactionSumFromLockedAccounts = 0L;
                 for (Transaction transaction : transactions) {
-
+                    //System.out.println("X:AddTx "+transaction);
                     // In blockchain version 1, cycle transactions are processed here. In later versions, cycle
                     // transactions are incorporated in the blockchain before being approved for transfer, so they are
                     // handled separately.
@@ -1025,8 +1034,12 @@ public static void setBlockDelayHeight() {
 
     @Override
     public String toString() {
-        return "[Block: v=" + getBlockchainVersion() + ", height=" + getBlockHeight() + ", hash=" +
+        return "[Block: v=" + getBlockchainVersion() + ", height=" + getBlockHeight() +
+                " txcount=" + transactions.size() +
+                ", hash=" +
                 PrintUtil.compactPrintByteArray(getHash()) + ", id=" +
-                PrintUtil.compactPrintByteArray(getVerifierIdentifier()) + "]";
+                PrintUtil.compactPrintByteArray(getVerifierIdentifier()) +  ", blh=" +
+                PrintUtil.compactPrintByteArray(getBalanceListHash())+
+                "]";
     }
 }

src/main/java/co/nyzo/verifier/BlockManager.java

@@ -231,6 +231,8 @@ public static void freezeBlock(Block block) {
         if (previousBlock != null) {
             BalanceList balanceList = BalanceListManager.balanceListForBlock(block);
             freezeBlock(block, previousBlock.getHash(), balanceList, null);
+        } else {
+            System.out.println("X:Previous block is null "+ block.getBlockHeight());
         }
     }
 
@@ -239,23 +241,34 @@ public static synchronized void freezeBlock(Block block, byte[] previousBlockHas
 
         // Only continue if the block's previous hash is correct and the balance list is available.
         if (ByteUtil.arraysAreEqual(previousBlockHash, block.getPreviousBlockHash()) && balanceList != null) {
+            if (!ByteUtil.arraysAreEqual(balanceList.getHash(), block.getBalanceListHash())) {
+                System.out.println("X:!!BLHash mismatch "+ PrintUtil.compactPrintByteArray(balanceList.getHash()) + " vs block " + block) ;
+            } else {
+                try {
+                    setFrozenEdge(block, cycleVerifiers);
+                    BalanceListManager.updateFrozenEdge(balanceList);
 
-            try {
-                setFrozenEdge(block, cycleVerifiers);
-                BalanceListManager.updateFrozenEdge(balanceList);
+                    writeBlocksToFile(Collections.singletonList(block), Collections.singletonList(balanceList),
+                            individualFileForBlockHeight(block.getBlockHeight()));
 
-                writeBlocksToFile(Collections.singletonList(block), Collections.singletonList(balanceList),
-                        individualFileForBlockHeight(block.getBlockHeight()));
+                    if (block.getBlockHeight() == 0L) {
 
-                if (block.getBlockHeight() == 0L) {
+                        genesisBlockStartTimestamp = block.getStartTimestamp();
+                        completedInitialization.set(true);
+                    }
 
-                    genesisBlockStartTimestamp = block.getStartTimestamp();
-                    completedInitialization.set(true);
+                } catch (Exception reportOnly) {
+                    reportOnly.printStackTrace();
+                    System.err.println("exception writing block to file " + reportOnly.getMessage());
+                    System.out.println("X:exception writing block to file " + reportOnly.getMessage());
                 }
-
-            } catch (Exception reportOnly) {
-                reportOnly.printStackTrace();
-                System.err.println("exception writing block to file " + reportOnly.getMessage());
+            }
+        } else {
+            if (balanceList == null) {
+                 System.out.println("X:BalanceList is null");
+            }
+            if (!ByteUtil.arraysAreEqual(previousBlockHash, block.getPreviousBlockHash())) {
+                System.out.println("X:Hashes mismatch "+ PrintUtil.compactPrintByteArray(previousBlockHash) + " vs block " + block.getPreviousBlock());
             }
         }
     }

src/main/java/co/nyzo/verifier/Message.java

@@ -235,7 +235,8 @@ public void run() {
                             response = readFromStream(socket.getInputStream(), socket.getInetAddress().getAddress(),
                                     message.getType());
                         } catch (Exception reportOnly) {
-                            System.err.println("Exception sending message " + message.getType() + " to " +
+                            // Egg: was err
+                            System.out.println("Exception sending message " + message.getType() + " to " +
                                     hostNameOrIp + ":" + port + ": " + PrintUtil.printException(reportOnly));
                         }
 

src/main/java/co/nyzo/verifier/Version.java

@@ -2,7 +2,7 @@
 
 public class Version {
 
-    private static final int version = 607;
+    private static final int version = 607111;
 
     public static int getVersion() {