Fix outpoint serialization issues

packages/client/tests/data/utreexo_169.json

@@ -79,7 +79,7 @@
                     "cached": false
                   },
                   "block_height": 9,
-                  "median_time_past": 1231471428,
+                  "median_time_past": 1231470988,
                   "is_coinbase": true
                 },
                 "witness": []
@@ -128,37 +128,37 @@
       "roots": [
         {
           "variant_id": 0,
-          "value": 2243729579132454921815355740493841076889575709816526684999662912394274975584
+          "value": 1075667171590848154982556435125102517390553068689710620455153660349876510186
         },
         null,
         null,
         {
           "variant_id": 0,
-          "value": 2261147245204776368334862529741319780868768593345793617211117537122266238577
+          "value": 1545231468516220002373273797559461807914970524111940849750328803582855936171
         },
         null,
         {
           "variant_id": 0,
-          "value": 1478944086323777945936459676941601959769102822055825304629072181687547875531
+          "value": 1377893971144855926456275647843230957136416614121223440964512004900657355746
         },
         null,
         {
           "variant_id": 0,
-          "value": 619721724166206394862378867028287088626965757449429663146230416403718619093
+          "value": 2789527937826812553983355805324006356113540169680511206602120580468925994144
         },
         null
       ]
     },
     "proofs": [
       {
         "proof": [
-          274817076080541397502696906861536371115695596063526833550058573072108749115,
-          1259176147345751145191111662219840526356501226618672639406065326898135902264,
-          1848169735107995159136156318494866770346290350559572574934907975254450481704,
-          2706681913355105973250866982294192611208835262570818334870601840808848384459,
-          2214630919606246947408368640763197246751208900324012684012261556337212626757,
-          1101985322418033002091130704802558308187225122297992534396475623983004812420,
-          2729277592763167786852222424289031376235136104897643593004420518718352940231
+          1341831566043387106325844845909158799402866001662974927108069625603583971480,
+          1443675700228503369543683218943466499245617322985505154945738943716743694265,
+          519646392377825945466547611274092870030802470736981675895864465165188803020,
+          3219090166904938598688939525052593187314786152670445479780817128254606574061,
+          3429608730564493038725327726731886582242605618763607208860625722555112702870,
+          1464346753344216951108436013919475968846135803696833681088250234646995926893,
+          2059532798546650597701371553400476891060690283065963625811202716166665983015
         ],
         "leaf_index": 8
       }
@@ -167,26 +167,26 @@
       "roots": [
         {
           "variant_id": 0,
-          "value": 2843980787531725946792369479988202437986876451050335501296002525043057800822
+          "value": 2798110709967837858515372760817606172577565229842771835911030873647893889774
         },
         {
           "variant_id": 0,
-          "value": 3259667504329439104864108759538360061214302036282299607930636468007761714215
+          "value": 143588888139305297948079116321466646636133238970252601366055285460885727445
         },
         null,
         {
           "variant_id": 0,
-          "value": 2261147245204776368334862529741319780868768593345793617211117537122266238577
+          "value": 1545231468516220002373273797559461807914970524111940849750328803582855936171
         },
         null,
         {
           "variant_id": 0,
-          "value": 1478944086323777945936459676941601959769102822055825304629072181687547875531
+          "value": 1377893971144855926456275647843230957136416614121223440964512004900657355746
         },
         null,
         {
           "variant_id": 0,
-          "value": 1597678775304253472610727607736636075835427473969519327752899884098801800240
+          "value": 72427602467111712670190708647558062003041829148690045408562661193902370953
         },
         null
       ]

packages/consensus/src/types/transaction.cairo

@@ -221,8 +221,8 @@ impl TxOutDisplay of Display<TxOut> {
 
 #[cfg(test)]
 mod tests {
-    use super::{OutPoint, OutPointTrait, TxOut, HashStateTrait, HashStateExTrait};
-    use utils::hash::{DigestTrait};
+    use super::{OutPoint, TxOut, HashStateTrait, HashStateExTrait, OutPointTrait};
+    use utils::hex::{hex_to_hash_rev, from_hex};
     use core::poseidon::PoseidonTrait;
 
     fn hash(tx: @TxOut) -> felt252 {
@@ -256,24 +256,64 @@ mod tests {
         assert_ne!(hash(@tx1), hash(@tx_with_value_changed));
     }
 
+    #[derive(Debug, Drop, Default)]
+    pub struct HashState {
+        pub value: Array<felt252>
+    }
+
+    impl HashStateImpl of HashStateTrait<HashState> {
+        fn update(self: HashState, value: felt252) -> HashState {
+            let mut new_value = self.value;
+            new_value.append(value);
+            HashState { value: new_value }
+        }
+
+        fn finalize(self: HashState) -> felt252 {
+            0
+        }
+    }
+
     #[test]
     pub fn test_outpoint_poseidon_hash() {
-        let mut test_outpoint = OutPoint {
-            txid: DigestTrait::new([1, 2, 3, 4, 5, 6, 7, 8]),
-            vout: 2,
-            // https://learnmeabitcoin.com/explorer/tx/0437cd7f8525ceed2324359c2d0ba26006d92d856a9c20fa0241106ee5a597c9#output-0
+        let mut coinbase_9_utxo = OutPoint {
+            txid: hex_to_hash_rev(
+                "0437cd7f8525ceed2324359c2d0ba26006d92d856a9c20fa0241106ee5a597c9"
+            ),
+            vout: 0,
             data: TxOut {
-                value: 50_u64,
-                pk_script: @"410411db93e1dcdb8a016b49840f8c53bc1eb68a382e97b1482ecad7b148a6909a5cb2e0eaddfb84ccf9744464f82e160bfa9b8b64f9d4c03f999b8643f656b412a3ac",
+                value: 5000000000,
+                pk_script: @from_hex(
+                    "410411db93e1dcdb8a016b49840f8c53bc1eb68a382e97b1482ecad7b148a6909a5cb2e0eaddfb84ccf9744464f82e160bfa9b8b64f9d4c03f999b8643f656b412a3ac"
+                ),
                 cached: false,
             },
             block_height: 9,
-            median_time_past: 1650000000,
-            is_coinbase: false,
+            median_time_past: 1231470988,
+            is_coinbase: true,
         };
+
+        let mut state: HashState = Default::default();
+        state = state.update_with(coinbase_9_utxo);
+
+        let expected: Array<felt252> = array![
+            5606656307511680658662848977137541728,
+            9103019671783490751638296454939121609,
+            0,
+            5000000000,
+            2,
+            114873147639302600539941532864842037771792291166958548649371950632810924198,
+            255491345418700057264349667014908841246825595399329019948869966327385048054,
+            372388307884,
+            5,
+            9,
+            1231470988,
+            1
+        ];
+        assert_eq!(expected, state.value);
+
+        let hash = coinbase_9_utxo.hash();
         assert_eq!(
-            test_outpoint.hash(),
-            3426256427357770988835517595549266311441229240020614569376880225204214993534
+            761592244424273723796345514960638980240531938129162865626185984897576522513, hash
         );
     }
 }

packages/consensus/src/types/utxo_set.cairo

@@ -76,13 +76,7 @@ pub impl UtxoSetImpl of UtxoSetTrait {
     fn finalize(ref self: UtxoSet) -> Result<(), ByteArray> {
         if self.num_cached != 0 {
             Result::Err("There are unprocessed cached outputs")
-        } // TODO: uncomment when utreexo is enabled
-        // else if self.leaves_to_add.len() != 0 {
-        //     Result::Err("There are unprocessed leaves to add")
-        // } else if self.leaves_to_delete.len() != 0 {
-        //     Result::Err("There are unprocessed leaves to delete")
-        // }
-        else {
+        } else {
             Result::Ok(())
         }
     }

scripts/data/generate_data.py

@@ -208,12 +208,17 @@ def resolve_outpoint(input: dict):
     """
     tx = request_rpc("getrawtransaction", [input["txid"], True])
     block = request_rpc("getblockheader", [tx["blockhash"]])
+    # Time-based relative lock-times are measured from the
+    # smallest allowed timestamp of the block containing the
+    # txout being spent, which is the median time past of the
+    # block prior.
+    prev_block = request_rpc("getblockheader", [block["previousblockhash"]])
     return {
         "txid": input["txid"],
         "vout": input["vout"],
         "data": format_output(tx["vout"][input["vout"]]),
         "block_height": block["height"],
-        "median_time_past": block["mediantime"],
+        "median_time_past": prev_block["mediantime"],
         "is_coinbase": tx["vin"][0].get("coinbase") is not None,
     }
 
@@ -392,7 +397,9 @@ def generate_data(
 
     if mode == "utreexo":
         utreexo_data = UtreexoData()
-        result["utreexo"] = utreexo_data.apply_blocks(blocks)
+        result["utreexo"] = utreexo_data.apply_blocks(
+            blocks, int(chain_state["mediantime"])
+        )
         result["blocks"] = [result["blocks"][-1]]
         if num_blocks > 1:
             result["chain_state"] = format_chain_state(prev_chain_state)

scripts/data/generate_utreexo_data.py

@@ -47,10 +47,7 @@ def _calculate_sub_data(self) -> t.List[int]:
             pending_word = 0
             pending_word_len = 0
 
-        # Check if there's still a word pending
-        if pending_word_len > 0:
-            sub_data.append(pending_word)
-
+        sub_data.append(pending_word)
         sub_data.append(pending_word_len)
 
         return sub_data
@@ -61,7 +58,8 @@ def serialize(self) -> OrderedDict:
         sub_data = self._calculate_sub_data()
 
         res["value"] = self.value
-        res["sub_data_len"] = len(sub_data)
+        # length of the array containing full words
+        res["sub_data_len"] = len(sub_data) - 2
 
         for idx, word in enumerate(sub_data):
             res["sub_data_{}".format(idx)] = word
@@ -98,7 +96,7 @@ def hash(self):
         tab.append(self.vout)
 
         # prev output
-        for e in self.data.serialize():
+        for _, e in self.data.serialize().items():
             tab.append(e)
 
         tab.append(self.block_height)
@@ -129,7 +127,7 @@ def snapshot_state(self) -> dict:
             "roots": list(map(format_root_node, self.utreexo.root_nodes)),
         }
 
-    def apply_blocks(self, blocks: list) -> dict:
+    def apply_blocks(self, blocks: list, prev_mtp: int) -> dict:
         state = {}
         proofs = []
         for block_idx, block in enumerate(blocks):
@@ -150,10 +148,12 @@ def apply_blocks(self, blocks: list) -> dict:
                 self.handle_txout(
                     tx["outputs"],
                     block["height"],
-                    block["mediantime"],
+                    prev_mtp,
                     txid,
                     i == 0,
                 )
+            prev_mtp = block["mediantime"]
+
         return {"state": state, "proofs": proofs, "expected": self.snapshot_state()}
 
     def handle_txin(self, inputs: list) -> list:

scripts/data/integration_tests.sh

@@ -47,8 +47,7 @@ ignored_files=(
     "tests/data/full_478557.json", # Run panicked with [108217864776563 ('blocks'), ]. 
     "tests/data/full_57042.json",  # Run panicked with [108217864776563 ('blocks'), ]. 
     "tests/data/full_72575.json",  # Run panicked with [108217864776563 ('blocks'), ]. 
-    "tests/data/full_757752.json", # Run panicked with [108217864776563 ('blocks'), ]. 
-    "tests/data/utreexo_169.json", # Unexpected root (TODO: create issue)
+    "tests/data/full_757752.json", # Run panicked with [108217864776563 ('blocks'), ].
     # "tests/data/full_478557.json", #runs on server
 )