diff --git a/create-hardfork/create-hardfork.cpp b/create-hardfork/create-hardfork.cpp index a9e7403a1..3110b1a85 100644 --- a/create-hardfork/create-hardfork.cpp +++ b/create-hardfork/create-hardfork.cpp @@ -246,6 +246,9 @@ class HardforkCreator : public td::actor::Actor { } void send_shard_block_info(ton::BlockIdExt block_id, ton::CatchainSeqno cc_seqno, td::BufferSlice data) override { } + void send_block_candidate(ton::BlockIdExt block_id, ton::CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data) override { + } void send_broadcast(ton::BlockBroadcast broadcast, bool custom_overlays_only) override { } void download_block(ton::BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, diff --git a/test/test-ton-collator.cpp b/test/test-ton-collator.cpp index d36db5d24..60476cf8f 100644 --- a/test/test-ton-collator.cpp +++ b/test/test-ton-collator.cpp @@ -347,6 +347,9 @@ class TestNode : public td::actor::Actor { } } } + void send_block_candidate(ton::BlockIdExt block_id, ton::CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data) override { + } void send_broadcast(ton::BlockBroadcast broadcast, bool custom_overlays_only) override { } void download_block(ton::BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index d0a063f7b..5d1e5b504 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -396,6 +396,9 @@ tonNode.blockBroadcastCompressed id:tonNode.blockIdExt catchain_seqno:int valida tonNode.ihrMessageBroadcast message:tonNode.ihrMessage = tonNode.Broadcast; tonNode.externalMessageBroadcast message:tonNode.externalMessage = tonNode.Broadcast; tonNode.newShardBlockBroadcast block:tonNode.newShardBlock = tonNode.Broadcast; +// signature may be empty, at least for now +tonNode.newBlockCandidateBroadcast id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int + collator_signature:tonNode.blockSignature data:bytes = tonNode.Broadcast; tonNode.shardPublicOverlayId workchain:int shard:long zero_state_file_hash:int256 = tonNode.ShardPublicOverlayId; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index fb7f21976..89948bb4a 100644 Binary files a/tl/generate/scheme/ton_api.tlo and b/tl/generate/scheme/ton_api.tlo differ diff --git a/ton/ton-types.h b/ton/ton-types.h index 24d542599..3f40fb656 100644 --- a/ton/ton-types.h +++ b/ton/ton-types.h @@ -344,6 +344,10 @@ struct BlockSignature { struct ReceivedBlock { BlockIdExt id; td::BufferSlice data; + + ReceivedBlock clone() const { + return ReceivedBlock{id, data.clone()}; + } }; struct BlockBroadcast { diff --git a/validator/db/files-async.hpp b/validator/db/files-async.hpp index bcb7fa8b7..c85089963 100644 --- a/validator/db/files-async.hpp +++ b/validator/db/files-async.hpp @@ -57,7 +57,7 @@ class WriteFile : public td::actor::Actor { status = file.sync(); } if (status.is_error()) { - td::unlink(old_name); + td::unlink(old_name).ignore(); promise_.set_error(std::move(status)); stop(); return; diff --git a/validator/downloaders/wait-block-data.cpp b/validator/downloaders/wait-block-data.cpp index c40dba487..220a8a2cf 100644 --- a/validator/downloaders/wait-block-data.cpp +++ b/validator/downloaders/wait-block-data.cpp @@ -17,10 +17,14 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "wait-block-data.hpp" + +#include "block-parse.h" +#include "block-auto.h" #include "fabric.h" #include "adnl/utils.hpp" #include "ton/ton-io.hpp" #include "common/delay.h" +#include "vm/cells/MerkleProof.h" namespace ton { @@ -108,7 +112,7 @@ void WaitBlockData::start() { td::actor::send_closure(SelfId, &WaitBlockData::failed_to_get_block_data_from_net, R.move_as_error_prefix("net error: ")); } else { - td::actor::send_closure(SelfId, &WaitBlockData::got_block_data_from_net, R.move_as_ok()); + td::actor::send_closure(SelfId, &WaitBlockData::got_data_from_net, R.move_as_ok()); } }); @@ -133,13 +137,49 @@ void WaitBlockData::failed_to_get_block_data_from_net(td::Status reason) { td::Timestamp::in(0.1)); } -void WaitBlockData::got_block_data_from_net(ReceivedBlock block) { +void WaitBlockData::got_data_from_net(ReceivedBlock block) { auto X = create_block(std::move(block)); if (X.is_error()) { failed_to_get_block_data_from_net(X.move_as_error_prefix("bad block from net: ")); return; } - data_ = X.move_as_ok(); + got_block_data_from_net(X.move_as_ok()); +} + +void WaitBlockData::got_block_data_from_net(td::Ref block) { + if (data_.not_null()) { + return; + } + data_ = std::move(block); + if (handle_->received()) { + finish_query(); + return; + } + if (!handle_->id().is_masterchain() && !handle_->inited_proof_link()) { + // This can happen if we get block from candidates cache. + // Proof link can be derived from the block (but not for masterchain block). + auto r_proof_link = generate_proof_link(handle_->id(), data_->root_cell()); + if (r_proof_link.is_error()) { + abort_query(r_proof_link.move_as_error_prefix("failed to create proof link for block: ")); + return; + } + td::actor::send_closure(manager_, &ValidatorManager::validate_block_proof_link, handle_->id(), + r_proof_link.move_as_ok(), + [id = handle_->id().id, SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &WaitBlockData::abort_query, + R.move_as_error_prefix("validate proof link error: ")); + return; + } + LOG(DEBUG) << "Created and validated proof link for " << id.to_str(); + td::actor::send_closure(SelfId, &WaitBlockData::checked_proof_link); + }); + return; + } + checked_proof_link(); +} + +void WaitBlockData::checked_proof_link() { CHECK(handle_->id().is_masterchain() ? handle_->inited_proof() : handle_->inited_proof_link()); if (!handle_->received()) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { @@ -198,6 +238,41 @@ void WaitBlockData::got_static_file(td::BufferSlice data) { run_hardfork_accept_block_query(handle_->id(), data_, manager_, std::move(P)); } +td::Result WaitBlockData::generate_proof_link(BlockIdExt id, td::Ref block_root) { + // Creating proof link. Similar to accept-block.cpp + if (id.is_masterchain()) { + return td::Status::Error("cannot create proof link for masterchain block"); + } + auto usage_tree = std::make_shared(); + auto usage_cell = vm::UsageCell::create(block_root, usage_tree->root_ptr()); + + block::gen::Block::Record blk; + block::gen::BlockInfo::Record info; + block::gen::BlockExtra::Record extra; + block::gen::ExtBlkRef::Record mcref{}; // _ ExtBlkRef = BlkMasterInfo; + ShardIdFull shard; + if (!(tlb::unpack_cell(usage_cell, blk) && tlb::unpack_cell(blk.info, info) && !info.version && + block::tlb::t_ShardIdent.unpack(info.shard.write(), shard) && + block::gen::BlkPrevInfo{info.after_merge}.validate_ref(info.prev_ref) && + tlb::unpack_cell(std::move(blk.extra), extra) && block::gen::t_ValueFlow.force_validate_ref(blk.value_flow) && + (!info.not_master || tlb::unpack_cell(info.master_ref, mcref)))) { + return td::Status::Error("cannot unpack block header"); + } + vm::CellSlice upd_cs{vm::NoVmSpec(), blk.state_update}; + + auto proof = vm::MerkleProof::generate(block_root, usage_tree.get()); + vm::CellBuilder cb; + td::Ref bs_cell; + if (!(cb.store_long_bool(0xc3, 8) // block_proof#c3 + && block::tlb::t_BlockIdExt.pack(cb, id) // proof_for:BlockIdExt + && cb.store_ref_bool(std::move(proof)) // proof:^Cell + && cb.store_bool_bool(false) // signatures:(Maybe ^BlockSignatures) + && cb.finalize_to(bs_cell))) { + return td::Status::Error("cannot serialize BlockProof"); + } + return std_boc_serialize(bs_cell, 0); +} + } // namespace validator } // namespace ton diff --git a/validator/downloaders/wait-block-data.hpp b/validator/downloaders/wait-block-data.hpp index 9a03b1cb6..229b4bfc8 100644 --- a/validator/downloaders/wait-block-data.hpp +++ b/validator/downloaders/wait-block-data.hpp @@ -57,11 +57,15 @@ class WaitBlockData : public td::actor::Actor { void set_is_hardfork(bool value); void start(); void got_block_data_from_db(td::Ref data); - void got_block_data_from_net(ReceivedBlock data); + void got_data_from_net(ReceivedBlock data); + void got_block_data_from_net(td::Ref block); + void checked_proof_link(); void failed_to_get_block_data_from_net(td::Status reason); void got_static_file(td::BufferSlice data); + static td::Result generate_proof_link(BlockIdExt id, td::Ref block_root); + private: BlockHandle handle_; diff --git a/validator/downloaders/wait-block-state.cpp b/validator/downloaders/wait-block-state.cpp index 42f5c791b..0ae82beaa 100644 --- a/validator/downloaders/wait-block-state.cpp +++ b/validator/downloaders/wait-block-state.cpp @@ -192,7 +192,8 @@ void WaitBlockState::got_proof_link(td::BufferSlice data) { td::actor::send_closure(SelfId, &WaitBlockState::after_get_proof_link); } else { LOG(INFO) << "received bad proof link: " << R.move_as_error(); - td::actor::send_closure(SelfId, &WaitBlockState::after_get_proof_link); + delay_action([SelfId]() { td::actor::send_closure(SelfId, &WaitBlockState::after_get_proof_link); }, + td::Timestamp::in(0.1)); } }); run_check_proof_link_query(handle_->id(), R.move_as_ok(), manager_, timeout_, std::move(P)); diff --git a/validator/full-node-private-overlay.cpp b/validator/full-node-private-overlay.cpp index 74bb75c52..ffe5468d4 100644 --- a/validator/full-node-private-overlay.cpp +++ b/validator/full-node-private-overlay.cpp @@ -17,6 +17,7 @@ #include "full-node-private-overlay.hpp" #include "ton/ton-tl.hpp" #include "common/delay.h" +#include "common/checksum.h" #include "full-node-serializer.hpp" namespace ton::validator::fullnode { @@ -49,6 +50,22 @@ void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, ton_api:: query.block_->cc_seqno_, std::move(query.block_->data_)); } +void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, + ton_api::tonNode_newBlockCandidateBroadcast &query) { + if (query.data_.size() > FullNode::max_block_size()) { + VLOG(FULL_NODE_WARNING) << "received block candidate with too big size from " << src; + return; + } + BlockIdExt block_id = create_block_id(query.id_); + if (td::sha256_bits256(query.data_.as_slice()) != block_id.file_hash) { + VLOG(FULL_NODE_WARNING) << "received block candidate with incorrect file hash from " << src; + return; + } + VLOG(FULL_NODE_DEBUG) << "Received newBlockCandidate in private overlay from " << src << ": " << block_id.to_str(); + td::actor::send_closure(full_node_, &FullNode::process_block_candidate_broadcast, block_id, query.catchain_seqno_, + query.validator_set_hash_, std::move(query.data_)); +} + void FullNodePrivateBlockOverlay::receive_broadcast(PublicKeyHash src, td::BufferSlice broadcast) { if (adnl::AdnlNodeIdShort{src} == local_id_) { return; @@ -77,6 +94,19 @@ void FullNodePrivateBlockOverlay::send_shard_block_info(BlockIdExt block_id, Cat } } +void FullNodePrivateBlockOverlay::send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::BufferSlice data) { + if (!inited_) { + return; + } + VLOG(FULL_NODE_DEBUG) << "Sending newBlockCandidate in private overlay: " << block_id.to_str(); + auto B = create_serialize_tl_object( + create_tl_block_id(block_id), cc_seqno, validator_set_hash, + create_tl_object(Bits256::zero(), td::BufferSlice()), std::move(data)); + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), std::move(B)); +} + void FullNodePrivateBlockOverlay::send_broadcast(BlockBroadcast broadcast) { if (!inited_) { return; @@ -199,6 +229,28 @@ void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNod std::move(query.message_->data_), it->second); } +void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query) { + if (!block_senders_.count(adnl::AdnlNodeIdShort(src))) { + VLOG(FULL_NODE_DEBUG) << "Dropping block candidate broadcast in private overlay \"" << name_ + << "\" from unauthorized sender " << src; + return; + } + if (query.data_.size() > FullNode::max_block_size()) { + VLOG(FULL_NODE_WARNING) << "received block candidate with too big size from " << src; + return; + } + BlockIdExt block_id = create_block_id(query.id_); + if (td::sha256_bits256(query.data_.as_slice()) != block_id.file_hash) { + VLOG(FULL_NODE_WARNING) << "received block candidate with incorrect file hash from " << src; + return; + } + // ignore cc_seqno and validator_hash for now + VLOG(FULL_NODE_DEBUG) << "Received newBlockCandidate in custom overlay \"" << name_ << "\" from " << src << ": " + << block_id.to_str(); + td::actor::send_closure(full_node_, &FullNode::process_block_candidate_broadcast, block_id, query.catchain_seqno_, + query.validator_set_hash_, std::move(query.data_)); +} + void FullNodeCustomOverlay::receive_broadcast(PublicKeyHash src, td::BufferSlice broadcast) { if (adnl::AdnlNodeIdShort{src} == local_id_) { return; @@ -241,6 +293,19 @@ void FullNodeCustomOverlay::send_broadcast(BlockBroadcast broadcast) { local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok()); } +void FullNodeCustomOverlay::send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::BufferSlice data) { + if (!inited_) { + return; + } + VLOG(FULL_NODE_DEBUG) << "Sending newBlockCandidate in custom overlay \"" << name_ << "\": " << block_id.to_str(); + auto B = create_serialize_tl_object( + create_tl_block_id(block_id), cc_seqno, validator_set_hash, + create_tl_object(Bits256::zero(), td::BufferSlice()), std::move(data)); + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), std::move(B)); +} + void FullNodeCustomOverlay::start_up() { std::sort(nodes_.begin(), nodes_.end()); nodes_.erase(std::unique(nodes_.begin(), nodes_.end()), nodes_.end()); diff --git a/validator/full-node-private-overlay.hpp b/validator/full-node-private-overlay.hpp index 196d6da60..e310824f0 100644 --- a/validator/full-node-private-overlay.hpp +++ b/validator/full-node-private-overlay.hpp @@ -27,6 +27,7 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor { void process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query); template void process_broadcast(PublicKeyHash, T &) { VLOG(FULL_NODE_WARNING) << "dropping unknown broadcast"; @@ -34,6 +35,8 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor { void receive_broadcast(PublicKeyHash src, td::BufferSlice query); void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data); + void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data); void send_broadcast(BlockBroadcast broadcast); void set_config(FullNodeConfig config) { @@ -98,6 +101,7 @@ class FullNodeCustomOverlay : public td::actor::Actor { void process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_externalMessageBroadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query); template void process_broadcast(PublicKeyHash, T &) { VLOG(FULL_NODE_WARNING) << "dropping unknown broadcast"; @@ -106,6 +110,8 @@ class FullNodeCustomOverlay : public td::actor::Actor { void send_external_message(td::BufferSlice data); void send_broadcast(BlockBroadcast broadcast); + void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data); void set_config(FullNodeConfig config) { config_ = std::move(config); diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 6e265cbc6..9b2b321e5 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -17,12 +17,14 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "auto/tl/ton_api.h" +#include "checksum.h" #include "overlays.h" #include "td/utils/SharedSlice.h" #include "full-node-shard.hpp" #include "full-node-shard-queries.hpp" #include "full-node-serializer.hpp" +#include "td/utils/buffer.h" #include "ton/ton-shard.h" #include "ton/ton-tl.hpp" #include "ton/ton-io.hpp" @@ -646,6 +648,22 @@ void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_ne query.block_->cc_seqno_, std::move(query.block_->data_)); } +void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query) { + if (query.data_.size() > FullNode::max_block_size()) { + VLOG(FULL_NODE_WARNING) << "received block candidate with too big size from " << src; + return; + } + BlockIdExt block_id = create_block_id(query.id_); + if (td::sha256_bits256(query.data_.as_slice()) != block_id.file_hash) { + VLOG(FULL_NODE_WARNING) << "received block candidate with incorrect file hash from " << src; + return; + } + // ignore cc_seqno and validator_hash for now + VLOG(FULL_NODE_DEBUG) << "Received newBlockCandidate from " << src << ": " << block_id.to_str(); + td::actor::send_closure(full_node_, &FullNode::process_block_candidate_broadcast, block_id, query.catchain_seqno_, + query.validator_set_hash_, std::move(query.data_)); +} + void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query) { process_block_broadcast(src, query); } @@ -738,6 +756,20 @@ void FullNodeShardImpl::send_shard_block_info(BlockIdExt block_id, CatchainSeqno } } +void FullNodeShardImpl::send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data) { + if (!client_.empty()) { + UNREACHABLE(); + return; + } + VLOG(FULL_NODE_DEBUG) << "Sending newBlockCandidate: " << block_id.to_str(); + auto B = create_serialize_tl_object( + create_tl_block_id(block_id), cc_seqno, validator_set_hash, + create_tl_object(Bits256::zero(), td::BufferSlice()), std::move(data)); + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, adnl_id_, overlay_id_, local_id_, + overlay::Overlays::BroadcastFlagAnySender(), std::move(B)); +} + void FullNodeShardImpl::send_broadcast(BlockBroadcast broadcast) { if (!client_.empty()) { UNREACHABLE(); diff --git a/validator/full-node-shard.h b/validator/full-node-shard.h index 2ed13d660..e89031fe9 100644 --- a/validator/full-node-shard.h +++ b/validator/full-node-shard.h @@ -41,6 +41,8 @@ class FullNodeShard : public td::actor::Actor { virtual void send_ihr_message(td::BufferSlice data) = 0; virtual void send_external_message(td::BufferSlice data) = 0; virtual void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) = 0; + virtual void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data) = 0; virtual void send_broadcast(BlockBroadcast broadcast) = 0; virtual void sign_overlay_certificate(PublicKeyHash signed_key, td::uint32 expiry_at, td::uint32 max_size, td::Promise promise) = 0; diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index 1fdbce1c1..08f48f862 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -18,6 +18,7 @@ */ #pragma once +#include "auto/tl/ton_api.h" #include "full-node-shard.h" #include "td/actor/PromiseFuture.h" #include "td/utils/port/Poll.h" @@ -152,12 +153,15 @@ class FullNodeShardImpl : public FullNodeShard { void process_broadcast(PublicKeyHash src, ton_api::tonNode_ihrMessageBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_externalMessageBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query); void receive_broadcast(PublicKeyHash src, td::BufferSlice query); void check_broadcast(PublicKeyHash src, td::BufferSlice query, td::Promise promise); void send_ihr_message(td::BufferSlice data) override; void send_external_message(td::BufferSlice data) override; void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override; + void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data) override; void send_broadcast(BlockBroadcast broadcast) override; void download_block(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 8d0f26bc9..ff8f1eca0 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -243,6 +243,24 @@ void FullNodeImpl::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_s td::actor::send_closure(shard, &FullNodeShard::send_shard_block_info, block_id, cc_seqno, std::move(data)); } +void FullNodeImpl::send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data) { + send_block_candidate_broadcast_to_custom_overlays(block_id, cc_seqno, validator_set_hash, data); + auto shard = get_shard(ShardIdFull{masterchainId, shardIdAll}); + if (shard.empty()) { + VLOG(FULL_NODE_WARNING) << "dropping OUT shard block info message to unknown shard"; + return; + } + if (!private_block_overlays_.empty()) { + td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateBlockOverlay::send_block_candidate, + block_id, cc_seqno, validator_set_hash, data.clone()); + } + if (broadcast_block_candidates_in_public_overlay_) { + td::actor::send_closure(shard, &FullNodeShard::send_block_candidate, block_id, cc_seqno, validator_set_hash, + std::move(data)); + } +} + void FullNodeImpl::send_broadcast(BlockBroadcast broadcast, bool custom_overlays_only) { send_block_broadcast_to_custom_overlays(broadcast); if (custom_overlays_only) { @@ -253,7 +271,7 @@ void FullNodeImpl::send_broadcast(BlockBroadcast broadcast, bool custom_overlays VLOG(FULL_NODE_WARNING) << "dropping OUT broadcast to unknown shard"; return; } - if (!private_block_overlays_.empty()) { + if (broadcast.block_id.is_masterchain() && !private_block_overlays_.empty()) { td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateBlockOverlay::send_broadcast, broadcast.clone()); } @@ -420,6 +438,14 @@ void FullNodeImpl::process_block_broadcast(BlockBroadcast broadcast) { }); } +void FullNodeImpl::process_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::BufferSlice data) { + send_block_candidate_broadcast_to_custom_overlays(block_id, cc_seqno, validator_set_hash, data); + // ignore cc_seqno and validator_hash for now + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_block_candidate, block_id, + std::move(data)); +} + void FullNodeImpl::start_up() { add_shard(ShardIdFull{masterchainId}); if (local_id_.is_zero()) { @@ -452,6 +478,11 @@ void FullNodeImpl::start_up() { void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override { td::actor::send_closure(id_, &FullNodeImpl::send_shard_block_info, block_id, cc_seqno, std::move(data)); } + void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data) override { + td::actor::send_closure(id_, &FullNodeImpl::send_block_candidate, block_id, cc_seqno, validator_set_hash, + std::move(data)); + } void send_broadcast(BlockBroadcast broadcast, bool custom_overlays_only) override { td::actor::send_closure(id_, &FullNodeImpl::send_broadcast, std::move(broadcast), custom_overlays_only); } @@ -587,6 +618,29 @@ void FullNodeImpl::send_block_broadcast_to_custom_overlays(const BlockBroadcast& } } +void FullNodeImpl::send_block_candidate_broadcast_to_custom_overlays(const BlockIdExt &block_id, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, + const td::BufferSlice &data) { + // Same cache of sent broadcasts as in send_block_broadcast_to_custom_overlays + if (!custom_overlays_sent_broadcasts_.insert(block_id).second) { + return; + } + custom_overlays_sent_broadcasts_lru_.push(block_id); + if (custom_overlays_sent_broadcasts_lru_.size() > 256) { + custom_overlays_sent_broadcasts_.erase(custom_overlays_sent_broadcasts_lru_.front()); + custom_overlays_sent_broadcasts_lru_.pop(); + } + for (auto &private_overlay : custom_overlays_) { + for (auto &actor : private_overlay.second.actors_) { + auto local_id = actor.first; + if (private_overlay.second.params_.block_senders_.count(local_id)) { + td::actor::send_closure(actor.second, &FullNodeCustomOverlay::send_block_candidate, block_id, cc_seqno, + validator_set_hash, data.clone()); + } + } + } +} + FullNodeImpl::FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, FullNodeConfig config, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, diff --git a/validator/full-node.h b/validator/full-node.h index ac260dd78..c3719f678 100644 --- a/validator/full-node.h +++ b/validator/full-node.h @@ -87,6 +87,8 @@ class FullNode : public td::actor::Actor { virtual void del_custom_overlay(std::string name, td::Promise promise) = 0; virtual void process_block_broadcast(BlockBroadcast broadcast) = 0; + virtual void process_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::BufferSlice data) = 0; static constexpr td::uint32 max_block_size() { return 4 << 20; diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 946a1705f..9b78b3e01 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -66,6 +66,8 @@ class FullNodeImpl : public FullNode { void send_ihr_message(AccountIdPrefixFull dst, td::BufferSlice data); void send_ext_message(AccountIdPrefixFull dst, td::BufferSlice data); void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqnp, td::BufferSlice data); + void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data); void send_broadcast(BlockBroadcast broadcast, bool custom_overlays_only); void download_block(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise promise); void download_zero_state(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, @@ -84,6 +86,8 @@ class FullNodeImpl : public FullNode { void new_key_block(BlockHandle handle); void process_block_broadcast(BlockBroadcast broadcast) override; + void process_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data) override; void start_up() override; @@ -124,6 +128,7 @@ class FullNodeImpl : public FullNode { std::map> private_block_overlays_; bool private_block_overlays_enable_compression_ = false; + bool broadcast_block_candidates_in_public_overlay_ = false; struct CustomOverlayInfo { CustomOverlayParams params_; @@ -138,6 +143,8 @@ class FullNodeImpl : public FullNode { void create_private_block_overlay(PublicKeyHash key); void update_custom_overlay(CustomOverlayInfo& overlay); void send_block_broadcast_to_custom_overlays(const BlockBroadcast& broadcast); + void send_block_candidate_broadcast_to_custom_overlays(const BlockIdExt& block_id, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, const td::BufferSlice& data); }; } // namespace fullnode diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index fd4ddd341..ba38ba423 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -5012,12 +5012,13 @@ bool Collator::create_block_candidate() { } // 4. save block candidate LOG(INFO) << "saving new BlockCandidate"; - td::actor::send_closure_later(manager, &ValidatorManager::set_block_candidate, block_candidate->id, - block_candidate->clone(), [self = get_self()](td::Result saved) -> void { - LOG(DEBUG) << "got answer to set_block_candidate"; - td::actor::send_closure_later(std::move(self), &Collator::return_block_candidate, - std::move(saved)); - }); + td::actor::send_closure_later( + manager, &ValidatorManager::set_block_candidate, block_candidate->id, block_candidate->clone(), + validator_set_->get_catchain_seqno(), validator_set_->get_validator_set_hash(), + [self = get_self()](td::Result saved) -> void { + LOG(DEBUG) << "got answer to set_block_candidate"; + td::actor::send_closure_later(std::move(self), &Collator::return_block_candidate, std::move(saved)); + }); // 5. communicate about bad and delayed external messages if (!bad_ext_msgs_.empty() || !delay_ext_msgs_.empty()) { LOG(INFO) << "sending complete_external_messages() to Manager"; diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 8b2723c33..88bc61634 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -6330,7 +6330,8 @@ bool ValidateQuery::save_candidate() { } }); - td::actor::send_closure(manager, &ValidatorManager::set_block_candidate, id_, block_candidate.clone(), std::move(P)); + td::actor::send_closure(manager, &ValidatorManager::set_block_candidate, id_, block_candidate.clone(), + validator_set_->get_catchain_seqno(), validator_set_->get_validator_set_hash(), std::move(P)); return true; } diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index d435b7569..0e9fab73b 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -93,7 +93,8 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void wait_block_signatures_short(BlockIdExt id, td::Timestamp timeout, td::Promise> promise) = 0; - virtual void set_block_candidate(BlockIdExt id, BlockCandidate candidate, td::Promise promise) = 0; + virtual void set_block_candidate(BlockIdExt id, BlockCandidate candidate, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::Promise promise) = 0; virtual void wait_block_state_merge(BlockIdExt left_id, BlockIdExt right_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) = 0; diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index 42e440811..17b793c7a 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -775,7 +775,8 @@ void ValidatorManagerImpl::set_next_block(BlockIdExt block_id, BlockIdExt next, get_block_handle(block_id, true, std::move(P)); } -void ValidatorManagerImpl::set_block_candidate(BlockIdExt id, BlockCandidate candidate, td::Promise promise) { +void ValidatorManagerImpl::set_block_candidate(BlockIdExt id, BlockCandidate candidate, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::Promise promise) { td::actor::send_closure(db_, &Db::store_block_candidate, std::move(candidate), std::move(promise)); } diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 5cf938847..12bd68ef2 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -130,6 +130,8 @@ class ValidatorManagerImpl : public ValidatorManager { } void new_ihr_message(td::BufferSlice data) override; void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override; + void new_block_candidate(BlockIdExt block_id, td::BufferSlice data) override { + } void add_ext_server_id(adnl::AdnlNodeIdShort id) override { UNREACHABLE(); @@ -177,7 +179,8 @@ class ValidatorManagerImpl : public ValidatorManager { void wait_block_signatures_short(BlockIdExt id, td::Timestamp timeout, td::Promise> promise) override; - void set_block_candidate(BlockIdExt id, BlockCandidate candidate, td::Promise promise) override; + void set_block_candidate(BlockIdExt id, BlockCandidate candidate, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::Promise promise) override; void wait_block_state_merge(BlockIdExt left_id, BlockIdExt right_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) override; diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index fb5237d6c..3a124f827 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -152,6 +152,9 @@ class ValidatorManagerImpl : public ValidatorManager { void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override { UNREACHABLE(); } + void new_block_candidate(BlockIdExt block_id, td::BufferSlice data) override { + UNREACHABLE(); + } void add_ext_server_id(adnl::AdnlNodeIdShort id) override { UNREACHABLE(); @@ -215,7 +218,8 @@ class ValidatorManagerImpl : public ValidatorManager { void wait_block_signatures_short(BlockIdExt id, td::Timestamp timeout, td::Promise> promise) override; - void set_block_candidate(BlockIdExt id, BlockCandidate candidate, td::Promise promise) override { + void set_block_candidate(BlockIdExt id, BlockCandidate candidate, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::Promise promise) override { promise.set_value(td::Unit()); } diff --git a/validator/manager.cpp b/validator/manager.cpp index 3d5440037..f91f04a17 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -17,6 +17,8 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "manager.hpp" +#include "checksum.h" +#include "td/utils/buffer.h" #include "validator-group.hpp" #include "adnl/utils.hpp" #include "downloaders/wait-block-state.hpp" @@ -464,6 +466,17 @@ void ValidatorManagerImpl::new_shard_block(BlockIdExt block_id, CatchainSeqno cc actor_id(this), td::Timestamp::in(2.0), std::move(P)); } +void ValidatorManagerImpl::new_block_candidate(BlockIdExt block_id, td::BufferSlice data) { + if (!last_masterchain_block_handle_) { + VLOG(VALIDATOR_DEBUG) << "dropping top shard block broadcast: not inited"; + return; + } + if (!started_) { + return; + } + add_cached_block_candidate(ReceivedBlock{block_id, std::move(data)}); +} + void ValidatorManagerImpl::add_shard_block_description(td::Ref desc) { if (desc->may_be_valid(last_masterchain_block_handle_, last_masterchain_state_)) { auto it = shard_blocks_.find(ShardTopBlockDescriptionId{desc->shard(), desc->catchain_seqno()}); @@ -495,6 +508,36 @@ void ValidatorManagerImpl::add_shard_block_description(td::Refsecond.actor_, &WaitBlockData::got_block_data_from_net, r_block.move_as_ok()); + } + } + } + { + auto it = wait_state_.find(id); + if (it != wait_state_.end()) { + // Proof link is not ready at this point, but this will force WaitBlockState to redo send_get_proof_link_request + td::actor::send_closure(it->second.actor_, &WaitBlockState::after_get_proof_link); + } + } + } + if (cached_block_candidates_lru_.size() > max_cached_candidates()) { + CHECK(cached_block_candidates_.erase(cached_block_candidates_lru_.front())); + cached_block_candidates_lru_.pop_front(); + } +} + void ValidatorManagerImpl::add_ext_server_id(adnl::AdnlNodeIdShort id) { class Cb : public adnl::Adnl::Callback { private: @@ -1189,11 +1232,16 @@ void ValidatorManagerImpl::set_next_block(BlockIdExt block_id, BlockIdExt next, get_block_handle(block_id, true, std::move(P)); } -void ValidatorManagerImpl::set_block_candidate(BlockIdExt id, BlockCandidate candidate, td::Promise promise) { +void ValidatorManagerImpl::set_block_candidate(BlockIdExt id, BlockCandidate candidate, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::Promise promise) { if (!candidates_buffer_.empty()) { td::actor::send_closure(candidates_buffer_, &CandidatesBuffer::add_new_candidate, id, PublicKey{pubkeys::Ed25519{candidate.pubkey.as_bits256()}}, candidate.collated_file_hash); } + if (!id.is_masterchain()) { + add_cached_block_candidate(ReceivedBlock{id, candidate.data.clone()}); + callback_->send_block_candidate(id, cc_seqno, validator_set_hash, candidate.data.clone()); + } td::actor::send_closure(db_, &Db::store_block_candidate, std::move(candidate), std::move(promise)); } @@ -1450,6 +1498,13 @@ void ValidatorManagerImpl::get_last_liteserver_state_block( void ValidatorManagerImpl::send_get_block_request(BlockIdExt id, td::uint32 priority, td::Promise promise) { + { + auto it = cached_block_candidates_.find(id); + if (it != cached_block_candidates_.end()) { + LOG(DEBUG) << "send_get_block_request: got result from candidates cache for " << id.to_str(); + return promise.set_value(it->second.clone()); + } + } callback_->download_block(id, priority, td::Timestamp::in(10.0), std::move(promise)); } @@ -1472,6 +1527,20 @@ void ValidatorManagerImpl::send_get_block_proof_request(BlockIdExt block_id, td: void ValidatorManagerImpl::send_get_block_proof_link_request(BlockIdExt block_id, td::uint32 priority, td::Promise promise) { + if (!block_id.is_masterchain()) { + auto it = cached_block_candidates_.find(block_id); + if (it != cached_block_candidates_.end()) { + // Proof link can be created from the cached block candidate + LOG(DEBUG) << "send_get_block_proof_link_request: creating proof link from cached caniddate for " + << block_id.to_str(); + TRY_RESULT_PROMISE_PREFIX(promise, block_root, vm::std_boc_deserialize(it->second.data), + "failed to create proof link: "); + TRY_RESULT_PROMISE_PREFIX(promise, proof_link, WaitBlockData::generate_proof_link(it->second.id, block_root), + "failed to create proof link: "); + promise.set_result(std::move(proof_link)); + return; + } + } callback_->download_block_proof_link(block_id, priority, td::Timestamp::in(10.0), std::move(promise)); } diff --git a/validator/manager.hpp b/validator/manager.hpp index 65636a94c..be7f18db9 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -18,10 +18,14 @@ */ #pragma once +#include "common/refcnt.hpp" #include "interfaces/validator-manager.h" #include "interfaces/db.h" #include "td/actor/PromiseFuture.h" +#include "td/utils/SharedSlice.h" +#include "td/utils/buffer.h" #include "td/utils/port/Poll.h" +#include "td/utils/port/StdStreams.h" #include "validator-group.hpp" #include "shard-client.hpp" #include "manager-init.h" @@ -220,6 +224,10 @@ class ValidatorManagerImpl : public ValidatorManager { }; // DATA FOR COLLATOR std::map> shard_blocks_; + + std::map cached_block_candidates_; + std::list cached_block_candidates_lru_; + struct ExtMessages { std::map, std::unique_ptr>> ext_messages_; std::map, std::map>> @@ -365,6 +373,7 @@ class ValidatorManagerImpl : public ValidatorManager { void new_ihr_message(td::BufferSlice data) override; void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override; + void new_block_candidate(BlockIdExt block_id, td::BufferSlice data) override; void add_ext_server_id(adnl::AdnlNodeIdShort id) override; void add_ext_server_port(td::uint16 port) override; @@ -409,7 +418,8 @@ class ValidatorManagerImpl : public ValidatorManager { void wait_block_signatures_short(BlockIdExt id, td::Timestamp timeout, td::Promise> promise) override; - void set_block_candidate(BlockIdExt id, BlockCandidate candidate, td::Promise promise) override; + void set_block_candidate(BlockIdExt id, BlockCandidate candidate, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::Promise promise) override; void wait_block_state_merge(BlockIdExt left_id, BlockIdExt right_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) override; @@ -503,6 +513,7 @@ class ValidatorManagerImpl : public ValidatorManager { } void add_shard_block_description(td::Ref desc); + void add_cached_block_candidate(ReceivedBlock block); void register_block_handle(BlockHandle handle); @@ -664,6 +675,9 @@ class ValidatorManagerImpl : public ValidatorManager { double max_mempool_num() const { return opts_->max_mempool_num(); } + size_t max_cached_candidates() const { + return 128; + } private: std::map> shard_client_waiters_; diff --git a/validator/validator.h b/validator/validator.h index 245acbd49..4e3390cc5 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -142,6 +142,8 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void send_ihr_message(AccountIdPrefixFull dst, td::BufferSlice data) = 0; virtual void send_ext_message(AccountIdPrefixFull dst, td::BufferSlice data) = 0; virtual void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) = 0; + virtual void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data) = 0; virtual void send_broadcast(BlockBroadcast broadcast, bool custom_overlays_only = false) = 0; virtual void download_block(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) = 0; @@ -210,6 +212,7 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void check_external_message(td::BufferSlice data, td::Promise> promise) = 0; virtual void new_ihr_message(td::BufferSlice data) = 0; virtual void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) = 0; + virtual void new_block_candidate(BlockIdExt block_id, td::BufferSlice data) = 0; virtual void add_ext_server_id(adnl::AdnlNodeIdShort id) = 0; virtual void add_ext_server_port(td::uint16 port) = 0;