Aristo db update merkle hasher (#1925)

* Register paths for added leafs because of trie re-balancing

why:
  While the payload would not change, the prefix in the leaf vertex
  would. So it needs to be flagged for hash recompilation for the
  `hashify()` module.

also:
  Make sure that `Hike` paths which might have vertex links into the
  backend filter are replaced by vertex copies before manipulating.
  Otherwise the vertices on the immutable filter might be involuntarily
  changed.

* Also check for paths where the leaf vertex is on the backend, already

why:
  A a path can have dome vertices on the top layer cache with the
  `Leaf` vertex on  the backend.

* Re-define a void `HashLabel` type.

why:
  A `HashLabel` type is a pair `(root-vertex-ID, Keccak-hash)`. Previously,
  a valid `HashLabel` consisted of a non-empty hash and a non-zero vertex
  ID. This definition leads to a non-unique representation of a void
  `HashLabel` with either root-ID or has void. This has been changed to
  the unique void `HashLabel` exactly if the hash entry is void.

* Update consistency checkers

* Re-org `hashify()` procedure

why:
  Syncing against block chain showed serious deficiencies which produced
  wrong hashes or simply bailed out with error.

  So all fringe cases (mainly due to deleted entries) could be integrated
  into the labelling schedule rather than handling separate fringe cases.

nimbus/db/aristo/aristo_check/check_be.nim

@@ -117,14 +117,12 @@ proc checkBE*[T: RdbBackendRef|MemBackendRef|VoidBackendRef](
   for (_,vid,key) in T.walkKeyBE db:
     if not key.isvalid:
       return err((vid,CheckBeKeyInvalid))
-    let rc = db.getVtxBE vid
-    if rc.isErr or not rc.value.isValid:
+    let vtx = db.getVtxBE(vid).valueOr:
       return err((vid,CheckBeVtxMissing))
-    let rx = rc.value.toNodeBE db # backend only
-    if rx.isErr:
+    let node = vtx.toNodeBE(db).valueOr: # backend links only
       return err((vid,CheckBeKeyCantCompile))
     if not relax:
-      let expected = rx.value.digestTo(HashKey)
+      let expected = node.digestTo(HashKey)
       if expected != key:
         return err((vid,CheckBeKeyMismatch))
     discard vids.reduce Interval[VertexID,uint64].new(vid,vid)
@@ -186,19 +184,21 @@ proc checkBE*[T: RdbBackendRef|MemBackendRef|VoidBackendRef](
         lastTrg = filter.trg
 
     # Check key table
+    var list: seq[VertexID]
     for (vid,lbl) in db.top.kMap.pairs:
+      list.add vid
       let vtx = db.getVtx vid
       if not db.top.sTab.hasKey(vid) and not vtx.isValid:
         return err((vid,CheckBeCacheKeyDangling))
-      if lbl.isValid and not relax:
-        if not vtx.isValid:
-          return err((vid,CheckBeCacheVtxDangling))
-        let rc = vtx.toNode db # compile cache first
-        if rc.isErr:
-          return err((vid,CheckBeCacheKeyCantCompile))
-        let expected = rc.value.digestTo(HashKey)
-        if expected != lbl.key:
-          return err((vid,CheckBeCacheKeyMismatch))
+      if not lbl.isValid or relax:
+        continue
+      if not vtx.isValid:
+        return err((vid,CheckBeCacheVtxDangling))
+      let node = vtx.toNode(db).valueOr: # compile cache first
+        return err((vid,CheckBeCacheKeyCantCompile))
+      let expected = node.digestTo(HashKey)
+      if expected != lbl.key:
+        return err((vid,CheckBeCacheKeyMismatch))
 
     # Check vGen
     let

nimbus/db/aristo/aristo_check/check_top.nim

@@ -40,9 +40,11 @@ proc checkTopStrict*(
       if vid notin revVids:
         return err((vid,CheckStkRevKeyMismatch))
 
-  let pAmkVtxCount = db.top.pAmk.values.toSeq.foldl(a + b.len, 0)
-  if 0 < pAmkVtxCount and pAmkVtxCount < db.top.sTab.len:
-    # Cannot have less changes than cached entries
+  let
+    pAmkVtxCount = db.top.pAmk.values.toSeq.foldl(a + b.len, 0)
+    sTabVtxCount = db.top.sTab.values.toSeq.filterIt(it.isValid).len
+  # Non-zero values mist sum up the same
+  if pAmkVtxCount < sTabVtxCount:
     return err((VertexID(0),CheckStkVtxCountMismatch))
 
   ok()
@@ -95,13 +97,17 @@ proc checkTopCommon*(
     kMapCount = db.top.kMap.values.toSeq.filterIt(it.isValid).len
     kMapNilCount = db.top.kMap.len - kMapCount
 
-  # Check deleted entries
-  var nNilVtx = 0
+  # Collect leafs and check deleted entries
+  var
+    nNilVtx = 0
+    leafs = db.top.lTab.values.toSeq.filterIt(it.isValid).toHashSet
   for (vid,vtx) in db.top.sTab.pairs:
     if vtx.isValid:
       case vtx.vType:
       of Leaf:
-        discard
+        if vid notin leafs:
+          return err((vid,CheckAnyLeafUnregistered))
+        leafs.excl vid
       of Branch:
         block check42Links:
           var seen = false
@@ -123,6 +129,10 @@ proc checkTopCommon*(
       if db.top.kMap.getOrVoid(vid).isValid:
         return err((vid,CheckAnyVtxEmptyKeyExpected))
 
+  # Check for dangling leaf records
+  if 0 < leafs.len:
+    return err((leafs.toSeq[0],CheckAnyLeafVidDangling))
+
   # If present, there are at least as many deleted hashes as there are deleted
   # vertices.
   if kMapNilCount != 0 and kMapNilCount < nNilVtx:

nimbus/db/aristo/aristo_constants.nim

@@ -34,7 +34,7 @@ const
   VOID_HASH_KEY* = HashKey()
     ## Void equivalent for Merkle hash value
 
-  VOID_HASH_LABEL* = HashLabel()
+  VOID_HASH_LABEL* = HashLabel(key: VOID_HASH_KEY)
     ## Void equivalent for Merkle hash value
 
   EmptyQidPairSeq* = seq[(QueueID,QueueID)].default

nimbus/db/aristo/aristo_debug.nim

@@ -15,7 +15,7 @@ import
   eth/[common, trie/nibbles],
   results,
   stew/byteutils,
-  "."/[aristo_constants, aristo_desc, aristo_hike],
+  "."/[aristo_constants, aristo_desc, aristo_get, aristo_hike],
   ./aristo_desc/desc_backend,
   ./aristo_init/[memory_db, memory_only, rocks_db],
   ./aristo_filter/filter_scheduler
@@ -85,8 +85,36 @@ proc squeeze(s: string; hex = false; ignLen = false): string =
       result &= "..(" & $s.len & ")"
     result &= ".." & s[s.len-16 .. ^1]
 
-proc stripZeros(a: string): string =
-  a.strip(leading=true, trailing=false, chars={'0'})
+proc stripZeros(a: string; toExp = false): string =
+  if 0 < a.len:
+    result = a.strip(leading=true, trailing=false, chars={'0'})
+    if result.len == 0:
+      result = "0"
+    elif result[^1] == '0' and toExp:
+      var n = 0
+      while result[^1] == '0':
+        let w = result.len
+        result.setLen(w-1)
+        n.inc
+      if n == 1:
+        result &= "0"
+      elif n == 2:
+        result &= "00"
+      elif 2 < n:
+        result &= "↑" & $n
+
+proc vidCode(lbl: HashLabel, db: AristoDbRef): uint64 =
+  if lbl.isValid:
+    if not db.top.isNil:
+      let vids = db.top.pAmk.getOrVoid lbl
+      if vids.isValid:
+        return vids.sortedKeys[0].uint64
+    block:
+      let vids = db.xMap.getOrVoid lbl
+      if vids.isValid:
+        return vids.sortedKeys[0].uint64
+
+# ---------------------
 
 proc ppVid(vid: VertexID; pfx = true): string =
   if pfx:
@@ -96,6 +124,15 @@ proc ppVid(vid: VertexID; pfx = true): string =
   else:
     result &= "ø"
 
+func ppCodeHash(h: Hash256): string =
+  result = "¢"
+  if h == Hash256():
+    result &= "©"
+  elif h == EMPTY_CODE_HASH:
+    result &= "ø"
+  else:
+    result &= h.data.toHex.squeeze(hex=true,ignLen=true)
+
 proc ppFid(fid: FilterID): string =
   if not fid.isValid:
     return "ø"
@@ -130,17 +167,6 @@ proc ppVidList(vGen: openArray[VertexID]): string =
 #proc ppVidList(vGen: HashSet[VertexID]): string =
 #  "{" & vGen.sortedKeys.mapIt(it.ppVid).join(",") & "}"
 
-proc vidCode(lbl: HashLabel, db: AristoDbRef): uint64 =
-  if lbl.isValid:
-    if not db.top.isNil:
-      let vids = db.top.pAmk.getOrVoid lbl
-      if vids.isValid:
-        return vids.sortedKeys[0].uint64
-    block:
-      let vids = db.xMap.getOrVoid lbl
-      if vids.isValid:
-        return vids.sortedKeys[0].uint64
-
 proc ppKey(key: HashKey; db: AristoDbRef; root: VertexID; pfx = true): string =
   proc getVids: HashSet[VertexID] =
     if not db.top.isNil:
@@ -153,10 +179,10 @@ proc ppKey(key: HashKey; db: AristoDbRef; root: VertexID; pfx = true): string =
         return vids
   if pfx:
     result = "£"
-  if key == VOID_HASH_KEY:
-    result &= "ø"
+  if key.len == 0 or key.to(Hash256) == Hash256():
+    result &= "©"
   elif not key.isValid:
-    result &= "r"
+    result &= "ø"
   else:
     let
       tag = if key.len < 32: "[#" & $key.len & "]" else: ""
@@ -179,11 +205,9 @@ proc ppLabel(lbl: HashLabel; db: AristoDbRef): string =
     "%ø"
 
 proc ppLeafTie(lty: LeafTie, db: AristoDbRef): string =
-  if not db.top.isNil:
-    let vid =  db.top.lTab.getOrVoid lty
-    if vid.isValid:
-      return "@" & vid.ppVid
-  "@" & $lty
+  let pfx = lty.path.to(NibblesSeq)
+  "@" & lty.root.ppVid(pfx=false) & ":" &
+    ($pfx).squeeze(hex=true,ignLen=(pfx.len==64))
 
 proc ppPathPfx(pfx: NibblesSeq): string =
   let s = $pfx
@@ -203,10 +227,10 @@ proc ppPayload(p: PayloadRef, db: AristoDbRef): string =
       result &= "[#" & p.rlpBlob.toHex.squeeze(hex=true) & "]"
     of AccountData:
       result = "("
-      result &= $p.account.nonce & ","
-      result &= $p.account.balance & ","
+      result &= ($p.account.nonce).stripZeros(toExp=true) & ","
+      result &= ($p.account.balance).stripZeros(toExp=true) & ","
       result &= p.account.storageID.ppVid & ","
-      result &= $p.account.codeHash & ")"
+      result &= p.account.codeHash.ppCodeHash & ")"
 
 proc ppVtx(nd: VertexRef, db: AristoDbRef, vid: VertexID): string =
   if not nd.isValid:
@@ -581,7 +605,7 @@ proc pp*(leg: Leg; db = AristoDbRef()): string =
       result &= lbl.ppLabel(db)
   result &= ","
   if leg.backend:
-    result &= "*"
+    result &= "¶"
   result &= ","
   if 0 <= leg.nibble:
     result &= $leg.nibble.ppNibble
@@ -628,6 +652,9 @@ proc pp*(tx: AristoTxRef): string =
     result &= ", par=" & $tx.parent.txUid
   result &= ")"
 
+proc pp*(wp: VidVtxPair; db: AristoDbRef): string =
+  "(" & wp.vid.pp & "," & wp.vtx.pp(db) & ")"
+
 # ---------------------
 
 proc pp*(
@@ -700,14 +727,15 @@ proc pp*(
 
 proc pp*(
     db: AristoDbRef;
-    backendOk = false;
     root = VertexID(1);
     indent = 4;
+    backendOk = false;
+    filterOk = true;
       ): string =
   result = db.top.pp(db, indent=indent) & indent.toPfx
   if backendOk:
     result &= db.backend.pp(db)
-  else:
+  elif filterOk:
     result &= db.roFilter.ppFilter(db, root, indent+1)
 
 proc pp*(sdb: MerkleSignRef; indent = 4): string =

nimbus/db/aristo/aristo_delete.nim

@@ -214,13 +214,20 @@ proc collapseLeaf(
 
   if 2 < hike.legs.len:                                  # (1), (2), or (3)
     # Merge `br` into the leaf `vtx` and unlink `br`.
-    let par = hike.legs[^3].wp
+    let par = hike.legs[^3].wp.dup                       # Writable vertex
     case par.vtx.vType:
     of Branch:                                           # (1)
       # Replace `vtx` by `^2 & vtx` (use `lf` as-is)
       par.vtx.bVid[hike.legs[^3].nibble] = lf.vid
       db.top.sTab[par.vid] = par.vtx
       db.top.sTab[lf.vid] = lf.vtx
+      # Make sure that there is a cache enty in case the leaf was pulled from
+      # the backend.!
+      let
+        lfPath = hike.legsTo(hike.legs.len - 2, NibblesSeq) & lf.vtx.lPfx
+        tag = lfPath.pathToTag.valueOr:
+          return err((lf.vid,error))
+      db.top.lTab[LeafTie(root: hike.root, path: tag)] = lf.vid
       return ok()
 
     of Extension:                                        # (2) or (3)
@@ -230,13 +237,20 @@ proc collapseLeaf(
 
       if 3 < hike.legs.len:                              # (2)
         # Grandparent exists
-        let gpr = hike.legs[^4].wp
+        let gpr = hike.legs[^4].wp.dup                   # Writable vertex
         if gpr.vtx.vType != Branch:
           return err((gpr.vid,DelBranchExpexted))
         db.doneWith par.vid                              # `par` is obsolete now
         gpr.vtx.bVid[hike.legs[^4].nibble] = lf.vid
         db.top.sTab[gpr.vid] = gpr.vtx
         db.top.sTab[lf.vid] = lf.vtx
+        # Make sure that there is a cache enty in case the leaf was pulled from
+        # the backend.!
+        let
+          lfPath = hike.legsTo(hike.legs.len - 3, NibblesSeq) & lf.vtx.lPfx
+          tag = lfPath.pathToTag.valueOr:
+            return err((lf.vid,error))
+        db.top.lTab[LeafTie(root: hike.root, path: tag)] = lf.vid
         return ok()
 
       # No grandparent, so ^3 is root vertex             # (3)
@@ -264,6 +278,17 @@ proc collapseLeaf(
 
   # Clean up stale leaf vertex which has moved to root position
   db.doneWith lf.vid
+
+  # If some `Leaf` vertex was installed as root, there must be a an extra
+  # `LeafTie` lookup entry.
+  let rootVtx = db.getVtx hike.root
+  if rootVtx.isValid and
+     rootVtx != hike.legs[0].wp.vtx and
+     rootVtx.vType == Leaf:
+    let tag = rootVtx.lPfx.pathToTag.valueOr:
+      return err((hike.root,error))
+    db.top.lTab[LeafTie(root: hike.root, path: tag)] = hike.root
+
   ok()
 
 # -------------------------
@@ -380,7 +405,7 @@ proc delete*(
     root: VertexID;
     path: openArray[byte];
      ): Result[void,(VertexID,AristoError)] =
-  ## Variant of `fetchPayload()`
+  ## Variant of `delete()`
   ##
   db.delete(? path.initNibbleRange.hikeUp(root, db).mapErr toVae)
 

nimbus/db/aristo/aristo_desc.nim

@@ -128,7 +128,7 @@ func isValid*(vid: VertexID): bool =
   vid != VertexID(0)
 
 func isValid*(lbl: HashLabel): bool =
-  lbl.root.isValid and lbl.key.isValid
+  lbl.key.isValid
 
 func isValid*(sqv: HashSet[VertexID]): bool =
   sqv != EmptyVidSet

nimbus/db/aristo/aristo_desc/desc_error.nim

@@ -80,8 +80,8 @@ type
     PathExpectedLeaf
 
     # Merge leaf `merge()`
-    MergeBrLinkLeafGarbled
-    MergeBrLinkVtxPfxTooShort
+    MergeBranchLinkLeafGarbled
+    MergeBranchLinkVtxPfxTooShort
     MergeBranchGarbledNibble
     MergeBranchGarbledTail
     MergeBranchLinkLockedKey
@@ -90,6 +90,7 @@ type
     MergeBranchRootExpected
     MergeLeafGarbledHike
     MergeLeafPathCachedAlready
+    MergeLeafPathOnBackendAlready
     MergeNonBranchProofModeLock
     MergeRootBranchLinkBusy
     MergeAssemblyFailed # Ooops, internal error
@@ -106,15 +107,11 @@ type
     MergeNodeVtxDuplicates
 
     # Update `Merkle` hashes `hashify()`
-    HashifyCannotComplete
-    HashifyCannotHashRoot
+    HashifyEmptyHike
     HashifyExistingHashMismatch
-    HashifyDownVtxlevelExceeded
-    HashifyDownVtxLeafUnexpected
+    HashifyNodeUnresolved
     HashifyRootHashMismatch
-    HashifyRootVidMismatch
-    HashifyVidCircularDependence
-    HashifyVtxMissing
+    HashifyRootNodeUnresolved
 
     # Cache checker `checkCache()`
     CheckStkVtxIncomplete
@@ -131,6 +128,8 @@ type
     CheckRlxRevKeyMissing
     CheckRlxRevKeyMismatch
 
+    CheckAnyLeafUnregistered
+    CheckAnyLeafVidDangling
     CheckAnyVidVtxMissing
     CheckAnyVtxEmptyKeyMissing
     CheckAnyVtxEmptyKeyExpected