From a6403bc4da8e2fb38fec83cbb18878576806f99b Mon Sep 17 00:00:00 2001 From: jackzhhuang Date: Mon, 4 Nov 2024 16:55:41 +0800 Subject: [PATCH 01/16] no store the sync dag block --- sync/src/tasks/block_sync_task.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sync/src/tasks/block_sync_task.rs b/sync/src/tasks/block_sync_task.rs index 1f83b1b344..e68ae73db8 100644 --- a/sync/src/tasks/block_sync_task.rs +++ b/sync/src/tasks/block_sync_task.rs @@ -454,6 +454,7 @@ where if block_header.number() % ASYNC_BLOCK_COUNT == 0 || block_header.number() >= self.target.target_id.number() { + self.sync_dag_store.delete_all_dag_sync_block()?; self.find_absent_ancestor(vec![block_header.clone()]) .await?; @@ -474,7 +475,6 @@ where block: block.clone(), children: vec![], })?; - self.sync_dag_store.save_block(block)?; anyhow::Ok(ParallelSign::NeedMoreBlocks) } }; From c1735590c9b10aa4bc020c148a2ea3a691a6250c Mon Sep 17 00:00:00 2001 From: jackzhhuang Date: Mon, 4 Nov 2024 17:22:27 +0800 Subject: [PATCH 02/16] add rm sync in kube file --- kube/manifest/starcoin-vega.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kube/manifest/starcoin-vega.yaml b/kube/manifest/starcoin-vega.yaml index 9e23dc8a2c..f2f5a7f1c5 100644 --- a/kube/manifest/starcoin-vega.yaml +++ b/kube/manifest/starcoin-vega.yaml @@ -28,7 +28,7 @@ spec: - -c args: - - rm -rf /sc-data/vega/starcoin.ipc /sc-data/vega/starcoindb/db/starcoindb/LOCK /sc-data/vega/genesis_config.json; + rm -rf /sc-data/vega/sync /sc-data/vega/starcoin.ipc /sc-data/vega/starcoindb/db/starcoindb/LOCK /sc-data/vega/genesis_config.json; id=$(echo -e $POD_NAME|awk -F'-' '{print $2}') && IFS='; ' read -r -a node_keys <<< $NODE_KEYS && node_key=${node_keys[$id]}; if [ ! -z $node_key ]; then From 48c9bac65a8f7dd3fb130411aeb0b2413111c12f Mon Sep 17 00:00:00 2001 From: jackzhhuang Date: Wed, 6 Nov 2024 10:39:58 +0800 Subject: [PATCH 03/16] use ghostdata to verify the dag block --- flexidag/src/blockdag.rs | 7 ++++--- flexidag/src/ghostdag/protocol.rs | 2 +- flexidag/src/prune/pruning_point_manager.rs | 5 +++++ 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/flexidag/src/blockdag.rs b/flexidag/src/blockdag.rs index 8b5e2def2f..f162d7ea90 100644 --- a/flexidag/src/blockdag.rs +++ b/flexidag/src/blockdag.rs @@ -561,11 +561,12 @@ impl BlockDAG { pub fn verify_and_ghostdata( &self, - blue_blocks: &[BlockHeader], + _blue_blocks: &[BlockHeader], header: &BlockHeader, ) -> Result { - self.ghost_dag_manager() - .verify_and_ghostdata(blue_blocks, header) + self.ghost_dag_manager().ghostdag(&header.parents()) + // self.ghost_dag_manager() + // .verify_and_ghostdata(blue_blocks, header) } pub fn check_upgrade(&self, main: &BlockHeader, genesis_id: HashValue) -> anyhow::Result<()> { // set the state with key 0 diff --git a/flexidag/src/ghostdag/protocol.rs b/flexidag/src/ghostdag/protocol.rs index 30567d473a..04501c997c 100644 --- a/flexidag/src/ghostdag/protocol.rs +++ b/flexidag/src/ghostdag/protocol.rs @@ -171,7 +171,7 @@ impl< Ok(new_block_data) } - pub(crate) fn verify_and_ghostdata( + pub(crate) fn _verify_and_ghostdata( &self, blue_blocks: &[BlockHeader], header: &BlockHeader, diff --git a/flexidag/src/prune/pruning_point_manager.rs b/flexidag/src/prune/pruning_point_manager.rs index 4e2ee9cf3e..a81597cf69 100644 --- a/flexidag/src/prune/pruning_point_manager.rs +++ b/flexidag/src/prune/pruning_point_manager.rs @@ -75,6 +75,11 @@ impl PruningPointManagerT { min_required_blue_score_for_next_pruning_point ); + debug!("previous_pruning_point: {:?}, previous_ghostdata: {:?}, next_ghostdata: {:?}, pruning_depth: {:?}, pruning_finality: {:?}", + previous_pruning_point, previous_ghostdata, next_ghostdata, + pruning_depth, pruning_finality, + ); + let mut latest_pruning_ghost_data = previous_ghostdata.to_compact(); if min_required_blue_score_for_next_pruning_point + pruning_depth <= next_ghostdata.blue_score From c8b259af405abcb5d439b46e94b730dab8e3b090 Mon Sep 17 00:00:00 2001 From: jackzhhuang Date: Wed, 6 Nov 2024 12:06:17 +0800 Subject: [PATCH 04/16] use ghodata if the verify is not correct --- flexidag/src/blockdag.rs | 8 ++++---- flexidag/src/ghostdag/protocol.rs | 12 ++++-------- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/flexidag/src/blockdag.rs b/flexidag/src/blockdag.rs index f162d7ea90..4a4ea666da 100644 --- a/flexidag/src/blockdag.rs +++ b/flexidag/src/blockdag.rs @@ -561,12 +561,12 @@ impl BlockDAG { pub fn verify_and_ghostdata( &self, - _blue_blocks: &[BlockHeader], + blue_blocks: &[BlockHeader], header: &BlockHeader, ) -> Result { - self.ghost_dag_manager().ghostdag(&header.parents()) - // self.ghost_dag_manager() - // .verify_and_ghostdata(blue_blocks, header) + // self.ghost_dag_manager().ghostdag(&header.parents()) + self.ghost_dag_manager() + .verify_and_ghostdata(blue_blocks, header) } pub fn check_upgrade(&self, main: &BlockHeader, genesis_id: HashValue) -> anyhow::Result<()> { // set the state with key 0 diff --git a/flexidag/src/ghostdag/protocol.rs b/flexidag/src/ghostdag/protocol.rs index 04501c997c..e18a4cf632 100644 --- a/flexidag/src/ghostdag/protocol.rs +++ b/flexidag/src/ghostdag/protocol.rs @@ -2,7 +2,7 @@ use super::util::Refs; use crate::consensusdb::schemadb::{GhostdagStoreReader, HeaderStoreReader, RelationsStoreReader}; use crate::reachability::reachability_service::ReachabilityService; use crate::types::{ghostdata::GhostdagData, ordering::*}; -use anyhow::{bail, ensure, Context, Result}; +use anyhow::{ensure, Context, Result}; use parking_lot::RwLock; use starcoin_crypto::HashValue as Hash; use starcoin_logger::prelude::*; @@ -171,7 +171,7 @@ impl< Ok(new_block_data) } - pub(crate) fn _verify_and_ghostdata( + pub(crate) fn verify_and_ghostdata( &self, blue_blocks: &[BlockHeader], header: &BlockHeader, @@ -221,12 +221,8 @@ impl< .map(|header| header.id()) .collect::>() { - if header.number() < 10000000 { - // no bail before 10000000 - warn!("The data of blue set is not equal when executing the block: {:?}, for {:?}, checking data: {:?}", header.id(), blue_blocks.iter().map(|header| header.id()).collect::>(), new_block_data.mergeset_blues); - } else { - bail!("The data of blue set is not equal when executing the block: {:?}, for {:?}, checking data: {:?}", header.id(), blue_blocks.iter().map(|header| header.id()).collect::>(), new_block_data.mergeset_blues); - } + warn!("The data of blue set is not equal when executing the block: {:?}, for {:?}, checking data: {:?}", header.id(), blue_blocks.iter().map(|header| header.id()).collect::>(), new_block_data.mergeset_blues); + return self.ghostdag(&header.parents_hash()); } let blue_score = self From 1a6c851e24c27357b7d44b9f992b4fa53cce9d8b Mon Sep 17 00:00:00 2001 From: jackzhhuang Date: Wed, 6 Nov 2024 14:58:12 +0800 Subject: [PATCH 05/16] use ghostdata instead of verify --- flexidag/src/blockdag.rs | 4 +--- flexidag/src/ghostdag/protocol.rs | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/flexidag/src/blockdag.rs b/flexidag/src/blockdag.rs index 4a4ea666da..febd5b588f 100644 --- a/flexidag/src/blockdag.rs +++ b/flexidag/src/blockdag.rs @@ -564,9 +564,7 @@ impl BlockDAG { blue_blocks: &[BlockHeader], header: &BlockHeader, ) -> Result { - // self.ghost_dag_manager().ghostdag(&header.parents()) - self.ghost_dag_manager() - .verify_and_ghostdata(blue_blocks, header) + self.ghost_dag_manager().ghostdag(&header.parents()) } pub fn check_upgrade(&self, main: &BlockHeader, genesis_id: HashValue) -> anyhow::Result<()> { // set the state with key 0 diff --git a/flexidag/src/ghostdag/protocol.rs b/flexidag/src/ghostdag/protocol.rs index e18a4cf632..35e1b07dcf 100644 --- a/flexidag/src/ghostdag/protocol.rs +++ b/flexidag/src/ghostdag/protocol.rs @@ -171,7 +171,7 @@ impl< Ok(new_block_data) } - pub(crate) fn verify_and_ghostdata( + pub(crate) fn _verify_and_ghostdata( &self, blue_blocks: &[BlockHeader], header: &BlockHeader, From 264a5d69f03708b4895a66bde4773e6621269017 Mon Sep 17 00:00:00 2001 From: jackzhhuang Date: Wed, 6 Nov 2024 16:09:50 +0800 Subject: [PATCH 06/16] add verification code for discovering the blue blocks error --- flexidag/src/blockdag.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/flexidag/src/blockdag.rs b/flexidag/src/blockdag.rs index febd5b588f..5acca97dd2 100644 --- a/flexidag/src/blockdag.rs +++ b/flexidag/src/blockdag.rs @@ -564,7 +564,19 @@ impl BlockDAG { blue_blocks: &[BlockHeader], header: &BlockHeader, ) -> Result { - self.ghost_dag_manager().ghostdag(&header.parents()) + let ghostdata = self.ghost_dag_manager().ghostdag(&header.parents())?; + if ghostdata + .mergeset_blues + .iter() + .skip(1) + .cloned() + .collect::>() + == blue_blocks.iter().map(|b| b.id()).collect::>() + { + Ok(ghostdata) + } else { + bail!("blue blocks are not correct for remote blue block: {:?}, local blue blocks: {:?}, header: {:?}", blue_blocks, ghostdata.mergeset_blues, header); + } } pub fn check_upgrade(&self, main: &BlockHeader, genesis_id: HashValue) -> anyhow::Result<()> { // set the state with key 0 From 3b2a145675112b4813d0697acc54b3c21ef1d4ba Mon Sep 17 00:00:00 2001 From: jackzhhuang Date: Thu, 7 Nov 2024 18:10:37 +0800 Subject: [PATCH 07/16] verify and if it fails it use ghostdata --- flexidag/src/blockdag.rs | 15 ++------------- flexidag/src/ghostdag/protocol.rs | 2 +- 2 files changed, 3 insertions(+), 14 deletions(-) diff --git a/flexidag/src/blockdag.rs b/flexidag/src/blockdag.rs index 5acca97dd2..8b5e2def2f 100644 --- a/flexidag/src/blockdag.rs +++ b/flexidag/src/blockdag.rs @@ -564,19 +564,8 @@ impl BlockDAG { blue_blocks: &[BlockHeader], header: &BlockHeader, ) -> Result { - let ghostdata = self.ghost_dag_manager().ghostdag(&header.parents())?; - if ghostdata - .mergeset_blues - .iter() - .skip(1) - .cloned() - .collect::>() - == blue_blocks.iter().map(|b| b.id()).collect::>() - { - Ok(ghostdata) - } else { - bail!("blue blocks are not correct for remote blue block: {:?}, local blue blocks: {:?}, header: {:?}", blue_blocks, ghostdata.mergeset_blues, header); - } + self.ghost_dag_manager() + .verify_and_ghostdata(blue_blocks, header) } pub fn check_upgrade(&self, main: &BlockHeader, genesis_id: HashValue) -> anyhow::Result<()> { // set the state with key 0 diff --git a/flexidag/src/ghostdag/protocol.rs b/flexidag/src/ghostdag/protocol.rs index 35e1b07dcf..e18a4cf632 100644 --- a/flexidag/src/ghostdag/protocol.rs +++ b/flexidag/src/ghostdag/protocol.rs @@ -171,7 +171,7 @@ impl< Ok(new_block_data) } - pub(crate) fn _verify_and_ghostdata( + pub(crate) fn verify_and_ghostdata( &self, blue_blocks: &[BlockHeader], header: &BlockHeader, From 197bc4fbdae4ead5b846543ac7bd42ca40e65ccb Mon Sep 17 00:00:00 2001 From: jackzhhuang Date: Thu, 7 Nov 2024 21:31:06 +0800 Subject: [PATCH 08/16] bail out if the ghost data checking is wrong --- flexidag/src/ghostdag/protocol.rs | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/flexidag/src/ghostdag/protocol.rs b/flexidag/src/ghostdag/protocol.rs index e18a4cf632..08c8b6e9df 100644 --- a/flexidag/src/ghostdag/protocol.rs +++ b/flexidag/src/ghostdag/protocol.rs @@ -2,7 +2,7 @@ use super::util::Refs; use crate::consensusdb::schemadb::{GhostdagStoreReader, HeaderStoreReader, RelationsStoreReader}; use crate::reachability::reachability_service::ReachabilityService; use crate::types::{ghostdata::GhostdagData, ordering::*}; -use anyhow::{ensure, Context, Result}; +use anyhow::{bail, ensure, Context, Result}; use parking_lot::RwLock; use starcoin_crypto::HashValue as Hash; use starcoin_logger::prelude::*; @@ -210,19 +210,30 @@ impl< } } + let remote_blue_set = blue_blocks + .iter() + .map(|header| header.id()) + .collect::>(); if new_block_data .mergeset_blues .iter() .skip(1) .cloned() .collect::>() - != blue_blocks - .iter() - .map(|header| header.id()) - .collect::>() + != remote_blue_set { warn!("The data of blue set is not equal when executing the block: {:?}, for {:?}, checking data: {:?}", header.id(), blue_blocks.iter().map(|header| header.id()).collect::>(), new_block_data.mergeset_blues); - return self.ghostdag(&header.parents_hash()); + let ghostdata = self.ghostdag(&header.parents_hash())?; + if ghostdata + .mergeset_blues + .iter() + .skip(1) + .cloned() + .collect::>() + != remote_blue_set + { + bail!("The ghost data of blue set is not equal when executing the block: {:?}, for {:?}, checking data: {:?}", header.id(), blue_blocks.iter().map(|header| header.id()).collect::>(), ghostdata.mergeset_blues); + } } let blue_score = self From adfcb23f95dd0a0b7d1fa0a39d4408dd9a2ce842 Mon Sep 17 00:00:00 2001 From: jackzhhuang Date: Fri, 8 Nov 2024 10:31:28 +0800 Subject: [PATCH 09/16] remove to print red blocks avoiding lots log data --- flexidag/src/blockdag.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flexidag/src/blockdag.rs b/flexidag/src/blockdag.rs index 8b5e2def2f..56b37f96f7 100644 --- a/flexidag/src/blockdag.rs +++ b/flexidag/src/blockdag.rs @@ -482,8 +482,8 @@ impl BlockDAG { let dag_state = self.get_dag_state(previous_pruning_point)?; let next_ghostdata = self.ghostdata(&dag_state.tips)?; info!( - "start to calculate the mergeset and tips for tips: {:?}, and last pruning point: {:?} and next ghostdata: {:?}", - dag_state.tips, previous_pruning_point, next_ghostdata, + "start to calculate the mergeset and tips for tips: {:?}, and last pruning point: {:?} and next ghostdata's selected parents: {:?} and blues set are {:?}", + dag_state.tips, previous_pruning_point, next_ghostdata.selected_parent, next_ghostdata.mergeset_blues, ); let next_pruning_point = self.pruning_point_manager().next_pruning_point( previous_pruning_point, From e06797dfaf33e3053afb34eb15eae1c398ec1cee Mon Sep 17 00:00:00 2001 From: jackzhhuang Date: Mon, 11 Nov 2024 22:04:49 +0800 Subject: [PATCH 10/16] test dag verification --- flexidag/src/ghostdag/protocol.rs | 3 +- flexidag/tests/tests.rs | 190 +++++++++++++++++++++++++++++- 2 files changed, 186 insertions(+), 7 deletions(-) diff --git a/flexidag/src/ghostdag/protocol.rs b/flexidag/src/ghostdag/protocol.rs index 08c8b6e9df..2519e06cf7 100644 --- a/flexidag/src/ghostdag/protocol.rs +++ b/flexidag/src/ghostdag/protocol.rs @@ -181,7 +181,8 @@ impl< !parents.is_empty(), "genesis must be added via a call to init" ); - let selected_parent = self.find_selected_parent(header.parents_hash().into_iter())?; + // let selected_parent = self.find_selected_parent(header.parents_hash().into_iter())?; + let selected_parent = header.parent_hash(); // Initialize new GHOSTDAG block data with the selected parent let mut new_block_data = GhostdagData::new_with_selected_parent(selected_parent, self.k); let ordered_mergeset = self.sort_blocks( diff --git a/flexidag/tests/tests.rs b/flexidag/tests/tests.rs index d3002c213c..8a2efb1d7a 100644 --- a/flexidag/tests/tests.rs +++ b/flexidag/tests/tests.rs @@ -17,12 +17,11 @@ use starcoin_dag::{ }; use starcoin_logger::prelude::debug; use starcoin_types::{ - block::{BlockHeader, BlockHeaderBuilder, BlockNumber}, - blockhash::{BlockHashMap, HashKTypeMap, KType}, + block::{BlockHeader, BlockHeaderBuilder, BlockNumber}, blockhash::{BlockHashMap, HashKTypeMap, KType}, consensus_header::ConsensusHeader, U256 }; use std::{ - collections::HashSet, + collections::{HashMap, HashSet}, ops::{Deref, DerefMut}, sync::Arc, time::Instant, @@ -737,6 +736,39 @@ fn add_and_print_with_ghostdata( Ok(header) } +fn add_and_print_with_difficulty( + number: BlockNumber, + parent: Hash, + parents: Vec, + origin: Hash, + difficulty: U256, + dag: &mut BlockDAG, +) -> anyhow::Result { + let header_builder = BlockHeaderBuilder::random(); + let header = header_builder + .with_parent_hash(parent) + .with_parents_hash(parents) + .with_number(number) + .with_difficulty(difficulty) + .build(); + let start = Instant::now(); + // dag.commit(header.to_owned(), origin)?; + let duration = start.elapsed(); + println!( + "commit header: {:?}, number: {:?}, duration: {:?}", + header.id(), + header.number(), + duration + ); + // let ghostdata = dag.ghostdata(&[header.id()])?; + // let ghostdata = dag.ghostdata_by_hash(header.id())?.unwrap(); + // println!( + // "add a header: {:?}, selected_parent: {:?}, blue set: {:?}, red set: {:?}, blue anticone size: {:?}", + // header, ghostdata.selected_parent, ghostdata.mergeset_blues, ghostdata.mergeset_reds, ghostdata.blues_anticone_sizes + // ); + Ok(header) +} + fn add_and_print_with_pruning_point( number: BlockNumber, parent: Hash, @@ -751,6 +783,7 @@ fn add_and_print_with_pruning_point( .with_parents_hash(parents) .with_number(number) .with_pruning_point(pruning_point) + .with_difficulty(U256::from(10)) .build(); let start = Instant::now(); dag.commit(header.to_owned(), origin)?; @@ -761,10 +794,11 @@ fn add_and_print_with_pruning_point( header.number(), duration ); - let _ghostdata = dag.ghostdata(&[header.id()])?; + // let ghostdata = dag.ghostdata(&[header.id()])?; + // let ghostdata = dag.ghostdata_by_hash(header.id())?.unwrap(); // println!( - // "add a header: {:?}, blue set: {:?}, red set: {:?}, blue anticone size: {:?}", - // header, ghostdata.mergeset_blues, ghostdata.mergeset_reds, ghostdata.blues_anticone_sizes + // "add a header: {:?}, selected_parent: {:?}, blue set: {:?}, red set: {:?}, blue anticone size: {:?}", + // header, ghostdata.selected_parent, ghostdata.mergeset_blues, ghostdata.mergeset_reds, ghostdata.blues_anticone_sizes // ); Ok(header) } @@ -1069,6 +1103,150 @@ fn test_prune() -> anyhow::Result<()> { anyhow::Result::Ok(()) } +#[test] +fn test_verification_blue_block_inconsistent() -> anyhow::Result<()> { + loop_to_blue()?; + anyhow::Result::Ok(()) +} + +fn loop_to_blue() -> anyhow::Result<()> { + // initialzie the dag firstly + let k = 2; + + let mut dag = BlockDAG::create_for_testing_with_parameters(k).unwrap(); + + let origin = BlockHeaderBuilder::random().with_number(0).build(); + let genesis = BlockHeader::dag_genesis_random_with_parent(origin)?; + + dag.init_with_genesis(genesis.clone()).unwrap(); + + let mut storage = HashMap::new(); + + let block1 = add_and_print_with_difficulty( + 1, + genesis.id(), + vec![genesis.id()], + genesis.parent_hash(), + U256::from(10), + &mut dag, + )?; + storage.insert(block1.id(), block1.clone()); + let ghost = dag.ghostdata(&block1.parents())?; + let verified_ghost = dag.verify_and_ghostdata(&ghost.mergeset_blues.iter().skip(1).cloned().map(|x| storage.get(&x).unwrap().clone()).collect::>(), &block1)?; + dag.commit_trusted_block(block1.clone(), genesis.parent_hash(), Arc::new(verified_ghost))?; + + let mut bottom = vec![]; + let mut last = block1.clone(); + for i in 0..500 { + let block2 = add_and_print_with_difficulty( + 1 + i, + last.id(), + vec![last.id()], + genesis.parent_hash(), + U256::from(10), + &mut dag, + )?; + last = block2.clone(); + storage.insert(block2.id(), block2.clone()); + let ghost = dag.ghostdata(&block2.parents())?; + let verified_ghost = dag.verify_and_ghostdata(&ghost.mergeset_blues.iter().skip(1).cloned().map(|x| storage.get(&x).unwrap().clone()).collect::>(), &block2)?; + dag.commit_trusted_block(block2.clone(), genesis.parent_hash(), Arc::new(verified_ghost))?; + bottom.push(block2); + } + + let mut top = vec![]; + let mut iter = bottom.iter().peekable(); + while let Some(first) = iter.next() { + if let Some(second) = iter.next() { + let block = add_and_print_with_difficulty( + 3, + first.id(), + vec![first.id(), second.id()], + genesis.parent_hash(), + U256::from(10), + &mut dag, + )?; + storage.insert(block.id(), block.clone()); + let ghost = dag.ghostdata(&block.parents())?; + let verified_ghost = dag.verify_and_ghostdata(&ghost.mergeset_blues.iter().skip(1).cloned().map(|x| storage.get(&x).unwrap().clone()).collect::>(), &block)?; + dag.commit_trusted_block(block.clone(), genesis.parent_hash(), Arc::new(verified_ghost))?; + + last = block.clone(); + top.push(block); + } else { + let block = add_and_print_with_difficulty( + 3, + first.id(), + vec![first.id(), last.id()], + genesis.parent_hash(), + U256::from(10), + &mut dag, + )?; + storage.insert(block.id(), block.clone()); + let ghost = dag.ghostdata(&block.parents())?; + let verified_ghost = dag.verify_and_ghostdata(&ghost.mergeset_blues.iter().skip(1).cloned().map(|x| storage.get(&x).unwrap().clone()).collect::>(), &block)?; + dag.commit_trusted_block(block.clone(), genesis.parent_hash(), Arc::new(verified_ghost))?; + + + + top.push(block); + if top.len() == 1 { + last = top[0].clone(); + break; + } else { + bottom = top.clone(); + iter = bottom.iter().peekable(); + top.clear(); + } + } + } + + let block1_1 = add_and_print_with_difficulty( + 1, + genesis.id(), + vec![last.id(), block1.id()], + genesis.parent_hash(), + U256::from(99999999), + &mut dag, + )?; + storage.insert(block1_1.id(), block1_1.clone()); + let ghost = dag.ghostdata(&block1_1.parents())?; + let verified_ghost = dag.verify_and_ghostdata(&ghost.mergeset_blues.iter().skip(1).cloned().map(|x| storage.get(&x).unwrap().clone()).collect::>(), &block1_1)?; + dag.commit_trusted_block(block1_1.clone(), genesis.parent_hash(), Arc::new(verified_ghost))?; + + let block3 = add_and_print_with_difficulty( + 3, + block1_1.id(), + vec![block1_1.id(), last.id()], + genesis.parent_hash(), + U256::from(10), + &mut dag, + )?; + + let ghostdata = dag.ghostdata(&block3.parents())?; + println!( + "add a header: {:?}, selected_parent: {:?}, blue set: {:?}, red set: {:?}, blue anticone size: {:?}", + block3, ghostdata.selected_parent, ghostdata.mergeset_blues, ghostdata.mergeset_reds, ghostdata.blues_anticone_sizes + ); + let verified_ghostdata = dag.verify_and_ghostdata( + &ghostdata + .mergeset_blues + .iter() + .skip(1) + .map(|x| dag.storage.header_store.get_header(*x).unwrap()) + .collect::>(), + &block3, + )?; + println!( + "after verification: selected_parent: {:?}, blue set: {:?}, red set: {:?}, blue anticone size: {:?}", + verified_ghostdata.selected_parent, verified_ghostdata.mergeset_blues, verified_ghostdata.mergeset_reds, verified_ghostdata.blues_anticone_sizes + ); + + assert_eq!(ghostdata.mergeset_blues, verified_ghostdata.mergeset_blues); + + anyhow::Ok(()) +} + #[test] fn test_verification_blue_block() -> anyhow::Result<()> { // initialzie the dag firstly From 86ee296f141b0eff09280720c6970ad1537d1665 Mon Sep 17 00:00:00 2001 From: jackzhhuang Date: Mon, 11 Nov 2024 23:48:00 +0800 Subject: [PATCH 11/16] add log to trace --- flexidag/src/ghostdag/protocol.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/flexidag/src/ghostdag/protocol.rs b/flexidag/src/ghostdag/protocol.rs index 2519e06cf7..831de5cb84 100644 --- a/flexidag/src/ghostdag/protocol.rs +++ b/flexidag/src/ghostdag/protocol.rs @@ -181,8 +181,7 @@ impl< !parents.is_empty(), "genesis must be added via a call to init" ); - // let selected_parent = self.find_selected_parent(header.parents_hash().into_iter())?; - let selected_parent = header.parent_hash(); + let selected_parent = self.find_selected_parent(header.parents_hash().into_iter())?; // Initialize new GHOSTDAG block data with the selected parent let mut new_block_data = GhostdagData::new_with_selected_parent(selected_parent, self.k); let ordered_mergeset = self.sort_blocks( @@ -391,6 +390,10 @@ impl< if *candidate_blues_anticone_sizes.get(&block).unwrap() == self.k { // k-cluster violation: A block in candidate's blue anticone already // has k blue blocks in its own anticone + info!( + "Checking blue candidate: {} failed, block {} has k blue blocks in its anticone", + blue_candidate, block + ); return Ok(ColoringState::Red); } @@ -439,6 +442,7 @@ impl< // The maximum length of new_block_data.mergeset_blues can be K+1 because // it contains the selected parent. if new_block_data.mergeset_blues.len() as KType == self.k.checked_add(1).unwrap() { + info!("Checking blue candidate: {} failed, mergeset blues size is K+1", blue_candidate); return Ok(ColoringOutput::Red); } From 825015928bae47c4ba11a9d8bb3912697eed2c86 Mon Sep 17 00:00:00 2001 From: jackzhhuang Date: Tue, 12 Nov 2024 00:25:19 +0800 Subject: [PATCH 12/16] add reachability data with parents --- flexidag/src/blockdag.rs | 3 ++- flexidag/src/ghostdag/protocol.rs | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/flexidag/src/blockdag.rs b/flexidag/src/blockdag.rs index 56b37f96f7..32755b9c88 100644 --- a/flexidag/src/blockdag.rs +++ b/flexidag/src/blockdag.rs @@ -226,7 +226,8 @@ impl BlockDAG { let mut merge_set = ghostdata .unordered_mergeset_without_selected_parent() .filter(|hash| self.storage.reachability_store.read().has(*hash).unwrap()) - .collect::>() + .chain(header.parents_hash().into_iter().filter(|parent_id| *parent_id != ghostdata.selected_parent)) + .collect::>() .into_iter(); let add_block_result = { let mut reachability_writer = reachability_store.write(); diff --git a/flexidag/src/ghostdag/protocol.rs b/flexidag/src/ghostdag/protocol.rs index 831de5cb84..507cde05de 100644 --- a/flexidag/src/ghostdag/protocol.rs +++ b/flexidag/src/ghostdag/protocol.rs @@ -384,6 +384,10 @@ impl< *candidate_blue_anticone_size = (*candidate_blue_anticone_size).checked_add(1).unwrap(); if *candidate_blue_anticone_size > self.k { // k-cluster violation: The candidate's blue anticone exceeded k + info!( + "Checking blue candidate: {} failed, blue anticone exceeded k", + blue_candidate + ); return Ok(ColoringState::Red); } From 573bc8906dc2da5ba6fb5035d52edc1887db543a Mon Sep 17 00:00:00 2001 From: jackzhhuang Date: Wed, 13 Nov 2024 10:44:33 +0800 Subject: [PATCH 13/16] fix fmt --- flexidag/src/blockdag.rs | 7 +- flexidag/src/ghostdag/protocol.rs | 7 +- flexidag/tests/tests.rs | 134 +++++++++++++++++++----------- 3 files changed, 97 insertions(+), 51 deletions(-) diff --git a/flexidag/src/blockdag.rs b/flexidag/src/blockdag.rs index 32755b9c88..f49c93113b 100644 --- a/flexidag/src/blockdag.rs +++ b/flexidag/src/blockdag.rs @@ -226,7 +226,12 @@ impl BlockDAG { let mut merge_set = ghostdata .unordered_mergeset_without_selected_parent() .filter(|hash| self.storage.reachability_store.read().has(*hash).unwrap()) - .chain(header.parents_hash().into_iter().filter(|parent_id| *parent_id != ghostdata.selected_parent)) + .chain( + header + .parents_hash() + .into_iter() + .filter(|parent_id| *parent_id != ghostdata.selected_parent), + ) .collect::>() .into_iter(); let add_block_result = { diff --git a/flexidag/src/ghostdag/protocol.rs b/flexidag/src/ghostdag/protocol.rs index 507cde05de..c6de9aab52 100644 --- a/flexidag/src/ghostdag/protocol.rs +++ b/flexidag/src/ghostdag/protocol.rs @@ -181,7 +181,7 @@ impl< !parents.is_empty(), "genesis must be added via a call to init" ); - let selected_parent = self.find_selected_parent(header.parents_hash().into_iter())?; + let selected_parent = self.find_selected_parent(header.parents_hash().into_iter())?; // Initialize new GHOSTDAG block data with the selected parent let mut new_block_data = GhostdagData::new_with_selected_parent(selected_parent, self.k); let ordered_mergeset = self.sort_blocks( @@ -446,7 +446,10 @@ impl< // The maximum length of new_block_data.mergeset_blues can be K+1 because // it contains the selected parent. if new_block_data.mergeset_blues.len() as KType == self.k.checked_add(1).unwrap() { - info!("Checking blue candidate: {} failed, mergeset blues size is K+1", blue_candidate); + info!( + "Checking blue candidate: {} failed, mergeset blues size is K+1", + blue_candidate + ); return Ok(ColoringOutput::Red); } diff --git a/flexidag/tests/tests.rs b/flexidag/tests/tests.rs index 8a2efb1d7a..8ef4fdbc4f 100644 --- a/flexidag/tests/tests.rs +++ b/flexidag/tests/tests.rs @@ -17,7 +17,10 @@ use starcoin_dag::{ }; use starcoin_logger::prelude::debug; use starcoin_types::{ - block::{BlockHeader, BlockHeaderBuilder, BlockNumber}, blockhash::{BlockHashMap, HashKTypeMap, KType}, consensus_header::ConsensusHeader, U256 + block::{BlockHeader, BlockHeaderBuilder, BlockNumber}, + blockhash::{BlockHashMap, HashKTypeMap, KType}, + consensus_header::ConsensusHeader, + U256, }; use std::{ @@ -740,9 +743,7 @@ fn add_and_print_with_difficulty( number: BlockNumber, parent: Hash, parents: Vec, - origin: Hash, difficulty: U256, - dag: &mut BlockDAG, ) -> anyhow::Result { let header_builder = BlockHeaderBuilder::random(); let header = header_builder @@ -752,7 +753,6 @@ fn add_and_print_with_difficulty( .with_difficulty(difficulty) .build(); let start = Instant::now(); - // dag.commit(header.to_owned(), origin)?; let duration = start.elapsed(); println!( "commit header: {:?}, number: {:?}, duration: {:?}", @@ -760,12 +760,7 @@ fn add_and_print_with_difficulty( header.number(), duration ); - // let ghostdata = dag.ghostdata(&[header.id()])?; - // let ghostdata = dag.ghostdata_by_hash(header.id())?.unwrap(); - // println!( - // "add a header: {:?}, selected_parent: {:?}, blue set: {:?}, red set: {:?}, blue anticone size: {:?}", - // header, ghostdata.selected_parent, ghostdata.mergeset_blues, ghostdata.mergeset_reds, ghostdata.blues_anticone_sizes - // ); + Ok(header) } @@ -1122,35 +1117,49 @@ fn loop_to_blue() -> anyhow::Result<()> { let mut storage = HashMap::new(); - let block1 = add_and_print_with_difficulty( - 1, - genesis.id(), - vec![genesis.id()], - genesis.parent_hash(), - U256::from(10), - &mut dag, - )?; + let block1 = + add_and_print_with_difficulty(1, genesis.id(), vec![genesis.id()], U256::from(10))?; storage.insert(block1.id(), block1.clone()); let ghost = dag.ghostdata(&block1.parents())?; - let verified_ghost = dag.verify_and_ghostdata(&ghost.mergeset_blues.iter().skip(1).cloned().map(|x| storage.get(&x).unwrap().clone()).collect::>(), &block1)?; - dag.commit_trusted_block(block1.clone(), genesis.parent_hash(), Arc::new(verified_ghost))?; + let verified_ghost = dag.verify_and_ghostdata( + &ghost + .mergeset_blues + .iter() + .skip(1) + .cloned() + .map(|x| storage.get(&x).unwrap().clone()) + .collect::>(), + &block1, + )?; + dag.commit_trusted_block( + block1.clone(), + genesis.parent_hash(), + Arc::new(verified_ghost), + )?; let mut bottom = vec![]; let mut last = block1.clone(); for i in 0..500 { - let block2 = add_and_print_with_difficulty( - 1 + i, - last.id(), - vec![last.id()], - genesis.parent_hash(), - U256::from(10), - &mut dag, - )?; + let block2 = + add_and_print_with_difficulty(1 + i, last.id(), vec![last.id()], U256::from(10))?; last = block2.clone(); storage.insert(block2.id(), block2.clone()); let ghost = dag.ghostdata(&block2.parents())?; - let verified_ghost = dag.verify_and_ghostdata(&ghost.mergeset_blues.iter().skip(1).cloned().map(|x| storage.get(&x).unwrap().clone()).collect::>(), &block2)?; - dag.commit_trusted_block(block2.clone(), genesis.parent_hash(), Arc::new(verified_ghost))?; + let verified_ghost = dag.verify_and_ghostdata( + &ghost + .mergeset_blues + .iter() + .skip(1) + .cloned() + .map(|x| storage.get(&x).unwrap().clone()) + .collect::>(), + &block2, + )?; + dag.commit_trusted_block( + block2.clone(), + genesis.parent_hash(), + Arc::new(verified_ghost), + )?; bottom.push(block2); } @@ -1162,14 +1171,25 @@ fn loop_to_blue() -> anyhow::Result<()> { 3, first.id(), vec![first.id(), second.id()], - genesis.parent_hash(), U256::from(10), - &mut dag, )?; storage.insert(block.id(), block.clone()); let ghost = dag.ghostdata(&block.parents())?; - let verified_ghost = dag.verify_and_ghostdata(&ghost.mergeset_blues.iter().skip(1).cloned().map(|x| storage.get(&x).unwrap().clone()).collect::>(), &block)?; - dag.commit_trusted_block(block.clone(), genesis.parent_hash(), Arc::new(verified_ghost))?; + let verified_ghost = dag.verify_and_ghostdata( + &ghost + .mergeset_blues + .iter() + .skip(1) + .cloned() + .map(|x| storage.get(&x).unwrap().clone()) + .collect::>(), + &block, + )?; + dag.commit_trusted_block( + block.clone(), + genesis.parent_hash(), + Arc::new(verified_ghost), + )?; last = block.clone(); top.push(block); @@ -1178,49 +1198,67 @@ fn loop_to_blue() -> anyhow::Result<()> { 3, first.id(), vec![first.id(), last.id()], - genesis.parent_hash(), U256::from(10), - &mut dag, )?; storage.insert(block.id(), block.clone()); let ghost = dag.ghostdata(&block.parents())?; - let verified_ghost = dag.verify_and_ghostdata(&ghost.mergeset_blues.iter().skip(1).cloned().map(|x| storage.get(&x).unwrap().clone()).collect::>(), &block)?; - dag.commit_trusted_block(block.clone(), genesis.parent_hash(), Arc::new(verified_ghost))?; - - + let verified_ghost = dag.verify_and_ghostdata( + &ghost + .mergeset_blues + .iter() + .skip(1) + .cloned() + .map(|x| storage.get(&x).unwrap().clone()) + .collect::>(), + &block, + )?; + dag.commit_trusted_block( + block.clone(), + genesis.parent_hash(), + Arc::new(verified_ghost), + )?; top.push(block); if top.len() == 1 { last = top[0].clone(); break; } else { - bottom = top.clone(); + bottom.clone_from(&top); iter = bottom.iter().peekable(); top.clear(); } } - } + } let block1_1 = add_and_print_with_difficulty( 1, genesis.id(), vec![last.id(), block1.id()], - genesis.parent_hash(), U256::from(99999999), - &mut dag, )?; storage.insert(block1_1.id(), block1_1.clone()); let ghost = dag.ghostdata(&block1_1.parents())?; - let verified_ghost = dag.verify_and_ghostdata(&ghost.mergeset_blues.iter().skip(1).cloned().map(|x| storage.get(&x).unwrap().clone()).collect::>(), &block1_1)?; - dag.commit_trusted_block(block1_1.clone(), genesis.parent_hash(), Arc::new(verified_ghost))?; + let verified_ghost = dag.verify_and_ghostdata( + &ghost + .mergeset_blues + .iter() + .skip(1) + .cloned() + .map(|x| storage.get(&x).unwrap().clone()) + .collect::>(), + &block1_1, + )?; + dag.commit_trusted_block( + block1_1.clone(), + genesis.parent_hash(), + Arc::new(verified_ghost), + )?; let block3 = add_and_print_with_difficulty( 3, block1_1.id(), vec![block1_1.id(), last.id()], - genesis.parent_hash(), U256::from(10), - &mut dag, )?; let ghostdata = dag.ghostdata(&block3.parents())?; From e3dca7089b79a29ca2f261b7e0de718ba5cc2c5e Mon Sep 17 00:00:00 2001 From: jackzhhuang Date: Thu, 14 Nov 2024 00:28:04 +0800 Subject: [PATCH 14/16] use unordered_mergeset_without_selected_parent when storing into the reachability --- flexidag/src/blockdag.rs | 25 +++++++++++++-------- flexidag/tests/tests.rs | 48 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 61 insertions(+), 12 deletions(-) diff --git a/flexidag/src/blockdag.rs b/flexidag/src/blockdag.rs index f49c93113b..1c3a857920 100644 --- a/flexidag/src/blockdag.rs +++ b/flexidag/src/blockdag.rs @@ -223,17 +223,24 @@ impl BlockDAG { ); let reachability_store = self.storage.reachability_store.clone(); - let mut merge_set = ghostdata - .unordered_mergeset_without_selected_parent() - .filter(|hash| self.storage.reachability_store.read().has(*hash).unwrap()) - .chain( - header - .parents_hash() - .into_iter() - .filter(|parent_id| *parent_id != ghostdata.selected_parent), + let mut merge_set = self + .ghost_dag_manager() + .unordered_mergeset_without_selected_parent( + ghostdata.selected_parent, + &header.parents(), ) - .collect::>() .into_iter(); + // let mut merge_set = ghostdata + // .unordered_mergeset_without_selected_parent() + // .filter(|hash| self.storage.reachability_store.read().has(*hash).unwrap()) + // .chain( + // header + // .parents_hash() + // .into_iter() + // .filter(|parent_id| *parent_id != ghostdata.selected_parent), + // ) + // .collect::>() + // .into_iter(); let add_block_result = { let mut reachability_writer = reachability_store.write(); inquirer::add_block( diff --git a/flexidag/tests/tests.rs b/flexidag/tests/tests.rs index 8ef4fdbc4f..a32db0e31c 100644 --- a/flexidag/tests/tests.rs +++ b/flexidag/tests/tests.rs @@ -443,8 +443,18 @@ fn test_reachability_check_ancestor() -> anyhow::Result<()> { reachability_store.write().deref_mut(), child, parent, - &mut vec![parent].into_iter(), + &mut vec![].into_iter(), )?; + // mergetset + let uncle1 = Hash::random(); + let selected_parent_uncle1 = Hash::random(); + inquirer::add_block( + reachability_store.write().deref_mut(), + selected_parent_uncle1, + parent, + &mut vec![].into_iter(), + )?; + let uncle2 = Hash::random(); let mut target = child; let mut target_parent = parent; @@ -457,17 +467,37 @@ fn test_reachability_check_ancestor() -> anyhow::Result<()> { reachability_store.write().deref_mut(), child, parent, - &mut vec![parent].into_iter(), + &mut vec![uncle2, uncle1].into_iter(), )?; target = child; target_parent = parent; + } else if i == 46 { + inquirer::add_block( + reachability_store.write().deref_mut(), + child, + parent, + &mut vec![].into_iter(), + )?; + inquirer::add_block( + reachability_store.write().deref_mut(), + uncle1, + selected_parent_uncle1, + &mut vec![].into_iter(), + )?; + + inquirer::add_block( + reachability_store.write().deref_mut(), + uncle2, + parent, + &mut vec![].into_iter(), + )?; } else { inquirer::add_block( reachability_store.write().deref_mut(), child, parent, - &mut vec![parent].into_iter(), + &mut vec![].into_iter(), )?; } } @@ -475,6 +505,18 @@ fn test_reachability_check_ancestor() -> anyhow::Result<()> { // the relationship // origin.....target_parent-target.....parent-child // ancestor + assert!( + dag.check_ancestor_of(selected_parent_uncle1, vec![parent, child])?, + "failed to check target is the ancestor of its descendant" + ); + assert!( + dag.check_ancestor_of(uncle1, vec![parent, child])?, + "failed to check target is the ancestor of its descendant" + ); + assert!( + dag.check_ancestor_of(uncle2, vec![parent, child])?, + "failed to check target is the ancestor of its descendant" + ); assert!( dag.check_ancestor_of(target, vec![parent, child])?, "failed to check target is the ancestor of its descendant" From 248b7fbfd5bc5307c3885cbec3d8a7966cfa7067 Mon Sep 17 00:00:00 2001 From: jackzhhuang Date: Thu, 14 Nov 2024 14:41:39 +0800 Subject: [PATCH 15/16] rm commented codes --- flexidag/src/blockdag.rs | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/flexidag/src/blockdag.rs b/flexidag/src/blockdag.rs index 1c3a857920..7ea0f3a229 100644 --- a/flexidag/src/blockdag.rs +++ b/flexidag/src/blockdag.rs @@ -230,17 +230,6 @@ impl BlockDAG { &header.parents(), ) .into_iter(); - // let mut merge_set = ghostdata - // .unordered_mergeset_without_selected_parent() - // .filter(|hash| self.storage.reachability_store.read().has(*hash).unwrap()) - // .chain( - // header - // .parents_hash() - // .into_iter() - // .filter(|parent_id| *parent_id != ghostdata.selected_parent), - // ) - // .collect::>() - // .into_iter(); let add_block_result = { let mut reachability_writer = reachability_store.write(); inquirer::add_block( From e4eff6c49516a3306655807ab560343de67e6e8e Mon Sep 17 00:00:00 2001 From: jackzhhuang Date: Thu, 14 Nov 2024 14:54:30 +0800 Subject: [PATCH 16/16] add uncle test in reachability --- flexidag/tests/tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexidag/tests/tests.rs b/flexidag/tests/tests.rs index a32db0e31c..b1c2fbdff4 100644 --- a/flexidag/tests/tests.rs +++ b/flexidag/tests/tests.rs @@ -467,7 +467,7 @@ fn test_reachability_check_ancestor() -> anyhow::Result<()> { reachability_store.write().deref_mut(), child, parent, - &mut vec![uncle2, uncle1].into_iter(), + &mut vec![uncle2, uncle1, selected_parent_uncle1].into_iter(), )?; target = child;