Add candidates cache (#1000)

* Broadcast shardchain block candidates in private overlays, generate proof links from candidates * Disable shardchain block broadcasts in private overlays * Send block candidate broadcasts to custom overlays, allow non-validators to receive candidates --------- Co-authored-by: SpyCheese <[email protected]>
ton-blockchain · May 24, 2024 · 539d5dd · 539d5dd · Tntzh · Sep 30, 2024
1 parent 7a74888
commit 539d5dd
Show file tree

Hide file tree

Showing 26 changed files with 382 additions and 20 deletions.
diff --git a/create-hardfork/create-hardfork.cpp b/create-hardfork/create-hardfork.cpp
@@ -246,6 +246,9 @@ class HardforkCreator : public td::actor::Actor {
       }
       void send_shard_block_info(ton::BlockIdExt block_id, ton::CatchainSeqno cc_seqno, td::BufferSlice data) override {
       }
+      void send_block_candidate(ton::BlockIdExt block_id, ton::CatchainSeqno cc_seqno, td::uint32 validator_set_hash,
+                                td::BufferSlice data) override {
+      }
       void send_broadcast(ton::BlockBroadcast broadcast, bool custom_overlays_only) override {
       }
       void download_block(ton::BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout,

diff --git a/test/test-ton-collator.cpp b/test/test-ton-collator.cpp
@@ -347,6 +347,9 @@ class TestNode : public td::actor::Actor {
           }
         }
       }
+      void send_block_candidate(ton::BlockIdExt block_id, ton::CatchainSeqno cc_seqno, td::uint32 validator_set_hash,
+                                td::BufferSlice data) override {
+      }
       void send_broadcast(ton::BlockBroadcast broadcast, bool custom_overlays_only) override {
       }
       void download_block(ton::BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout,

diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl
@@ -396,6 +396,9 @@ tonNode.blockBroadcastCompressed id:tonNode.blockIdExt catchain_seqno:int valida
 tonNode.ihrMessageBroadcast message:tonNode.ihrMessage = tonNode.Broadcast;
 tonNode.externalMessageBroadcast message:tonNode.externalMessage = tonNode.Broadcast;
 tonNode.newShardBlockBroadcast block:tonNode.newShardBlock = tonNode.Broadcast;
+// signature may be empty, at least for now
+tonNode.newBlockCandidateBroadcast id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int
+              collator_signature:tonNode.blockSignature data:bytes = tonNode.Broadcast;
 
 tonNode.shardPublicOverlayId workchain:int shard:long zero_state_file_hash:int256 = tonNode.ShardPublicOverlayId;
 

diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo
diff --git a/ton/ton-types.h b/ton/ton-types.h
@@ -344,6 +344,10 @@ struct BlockSignature {
 struct ReceivedBlock {
   BlockIdExt id;
   td::BufferSlice data;
+
+  ReceivedBlock clone() const {
+    return ReceivedBlock{id, data.clone()};
+  }
 };
 
 struct BlockBroadcast {

diff --git a/validator/db/files-async.hpp b/validator/db/files-async.hpp
@@ -57,7 +57,7 @@ class WriteFile : public td::actor::Actor {
       status = file.sync();
     }
     if (status.is_error()) {
-      td::unlink(old_name);
+      td::unlink(old_name).ignore();
       promise_.set_error(std::move(status));
       stop();
       return;

diff --git a/validator/downloaders/wait-block-data.cpp b/validator/downloaders/wait-block-data.cpp
@@ -17,10 +17,14 @@
     Copyright 2017-2020 Telegram Systems LLP
 */
 #include "wait-block-data.hpp"
+
+#include "block-parse.h"
+#include "block-auto.h"
 #include "fabric.h"
 #include "adnl/utils.hpp"
 #include "ton/ton-io.hpp"
 #include "common/delay.h"
+#include "vm/cells/MerkleProof.h"
 
 namespace ton {
 
@@ -108,7 +112,7 @@ void WaitBlockData::start() {
         td::actor::send_closure(SelfId, &WaitBlockData::failed_to_get_block_data_from_net,
                                 R.move_as_error_prefix("net error: "));
       } else {
-        td::actor::send_closure(SelfId, &WaitBlockData::got_block_data_from_net, R.move_as_ok());
+        td::actor::send_closure(SelfId, &WaitBlockData::got_data_from_net, R.move_as_ok());
       }
     });
 
@@ -133,13 +137,49 @@ void WaitBlockData::failed_to_get_block_data_from_net(td::Status reason) {
                td::Timestamp::in(0.1));
 }
 
-void WaitBlockData::got_block_data_from_net(ReceivedBlock block) {
+void WaitBlockData::got_data_from_net(ReceivedBlock block) {
   auto X = create_block(std::move(block));
   if (X.is_error()) {
     failed_to_get_block_data_from_net(X.move_as_error_prefix("bad block from net: "));
     return;
   }
-  data_ = X.move_as_ok();
+  got_block_data_from_net(X.move_as_ok());
+}
+
+void WaitBlockData::got_block_data_from_net(td::Ref<BlockData> block) {
+  if (data_.not_null()) {
+    return;
+  }
+  data_ = std::move(block);
+  if (handle_->received()) {
+    finish_query();
+    return;
+  }
+  if (!handle_->id().is_masterchain() && !handle_->inited_proof_link()) {
+    // This can happen if we get block from candidates cache.
+    // Proof link can be derived from the block (but not for masterchain block).
+    auto r_proof_link = generate_proof_link(handle_->id(), data_->root_cell());
+    if (r_proof_link.is_error()) {
+      abort_query(r_proof_link.move_as_error_prefix("failed to create proof link for block: "));
+      return;
+    }
+    td::actor::send_closure(manager_, &ValidatorManager::validate_block_proof_link, handle_->id(),
+                            r_proof_link.move_as_ok(),
+                            [id = handle_->id().id, SelfId = actor_id(this)](td::Result<td::Unit> R) {
+                              if (R.is_error()) {
+                                td::actor::send_closure(SelfId, &WaitBlockData::abort_query,
+                                                        R.move_as_error_prefix("validate proof link error: "));
+                                return;
+                              }
+                              LOG(DEBUG) << "Created and validated proof link for " << id.to_str();
+                              td::actor::send_closure(SelfId, &WaitBlockData::checked_proof_link);
+                            });
+    return;
+  }
+  checked_proof_link();
+}
+
+void WaitBlockData::checked_proof_link() {
   CHECK(handle_->id().is_masterchain() ? handle_->inited_proof() : handle_->inited_proof_link());
   if (!handle_->received()) {
     auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::Unit> R) {
@@ -198,6 +238,41 @@ void WaitBlockData::got_static_file(td::BufferSlice data) {
   run_hardfork_accept_block_query(handle_->id(), data_, manager_, std::move(P));
 }
 
+td::Result<td::BufferSlice> WaitBlockData::generate_proof_link(BlockIdExt id, td::Ref<vm::Cell> block_root) {
+  // Creating proof link. Similar to accept-block.cpp
+  if (id.is_masterchain()) {
+    return td::Status::Error("cannot create proof link for masterchain block");
+  }
+  auto usage_tree = std::make_shared<vm::CellUsageTree>();
+  auto usage_cell = vm::UsageCell::create(block_root, usage_tree->root_ptr());
+
+  block::gen::Block::Record blk;
+  block::gen::BlockInfo::Record info;
+  block::gen::BlockExtra::Record extra;
+  block::gen::ExtBlkRef::Record mcref{};  // _ ExtBlkRef = BlkMasterInfo;
+  ShardIdFull shard;
+  if (!(tlb::unpack_cell(usage_cell, blk) && tlb::unpack_cell(blk.info, info) && !info.version &&
+        block::tlb::t_ShardIdent.unpack(info.shard.write(), shard) &&
+        block::gen::BlkPrevInfo{info.after_merge}.validate_ref(info.prev_ref) &&
+        tlb::unpack_cell(std::move(blk.extra), extra) && block::gen::t_ValueFlow.force_validate_ref(blk.value_flow) &&
+        (!info.not_master || tlb::unpack_cell(info.master_ref, mcref)))) {
+    return td::Status::Error("cannot unpack block header");
+  }
+  vm::CellSlice upd_cs{vm::NoVmSpec(), blk.state_update};
+
+  auto proof = vm::MerkleProof::generate(block_root, usage_tree.get());
+  vm::CellBuilder cb;
+  td::Ref<vm::Cell> bs_cell;
+  if (!(cb.store_long_bool(0xc3, 8)               // block_proof#c3
+        && block::tlb::t_BlockIdExt.pack(cb, id)  // proof_for:BlockIdExt
+        && cb.store_ref_bool(std::move(proof))    // proof:^Cell
+        && cb.store_bool_bool(false)              // signatures:(Maybe ^BlockSignatures)
+        && cb.finalize_to(bs_cell))) {
+    return td::Status::Error("cannot serialize BlockProof");
+  }
+  return std_boc_serialize(bs_cell, 0);
+}
+
 }  // namespace validator
 
 }  // namespace ton
diff --git a/validator/downloaders/wait-block-data.hpp b/validator/downloaders/wait-block-data.hpp
@@ -57,11 +57,15 @@ class WaitBlockData : public td::actor::Actor {
   void set_is_hardfork(bool value);
   void start();
   void got_block_data_from_db(td::Ref<BlockData> data);
-  void got_block_data_from_net(ReceivedBlock data);
+  void got_data_from_net(ReceivedBlock data);
+  void got_block_data_from_net(td::Ref<BlockData> block);
+  void checked_proof_link();
   void failed_to_get_block_data_from_net(td::Status reason);
 
   void got_static_file(td::BufferSlice data);
 
+  static td::Result<td::BufferSlice> generate_proof_link(BlockIdExt id, td::Ref<vm::Cell> block_root);
+
  private:
   BlockHandle handle_;
 

diff --git a/validator/downloaders/wait-block-state.cpp b/validator/downloaders/wait-block-state.cpp
@@ -192,7 +192,8 @@ void WaitBlockState::got_proof_link(td::BufferSlice data) {
       td::actor::send_closure(SelfId, &WaitBlockState::after_get_proof_link);
     } else {
       LOG(INFO) << "received bad proof link: " << R.move_as_error();
-      td::actor::send_closure(SelfId, &WaitBlockState::after_get_proof_link);
+      delay_action([SelfId]() { td::actor::send_closure(SelfId, &WaitBlockState::after_get_proof_link); },
+                   td::Timestamp::in(0.1));
     }
   });
   run_check_proof_link_query(handle_->id(), R.move_as_ok(), manager_, timeout_, std::move(P));

diff --git a/validator/full-node-private-overlay.cpp b/validator/full-node-private-overlay.cpp
@@ -17,6 +17,7 @@
 #include "full-node-private-overlay.hpp"
 #include "ton/ton-tl.hpp"
 #include "common/delay.h"
+#include "common/checksum.h"
 #include "full-node-serializer.hpp"
 
 namespace ton::validator::fullnode {
@@ -49,6 +50,22 @@ void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, ton_api::
                           query.block_->cc_seqno_, std::move(query.block_->data_));
 }
 
+void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src,
+                                                    ton_api::tonNode_newBlockCandidateBroadcast &query) {
+  if (query.data_.size() > FullNode::max_block_size()) {
+    VLOG(FULL_NODE_WARNING) << "received block candidate with too big size from " << src;
+    return;
+  }
+  BlockIdExt block_id = create_block_id(query.id_);
+  if (td::sha256_bits256(query.data_.as_slice()) != block_id.file_hash) {
+    VLOG(FULL_NODE_WARNING) << "received block candidate with incorrect file hash from " << src;
+    return;
+  }
+  VLOG(FULL_NODE_DEBUG) << "Received newBlockCandidate in private overlay from " << src << ": " << block_id.to_str();
+  td::actor::send_closure(full_node_, &FullNode::process_block_candidate_broadcast, block_id, query.catchain_seqno_,
+                          query.validator_set_hash_, std::move(query.data_));
+}
+
 void FullNodePrivateBlockOverlay::receive_broadcast(PublicKeyHash src, td::BufferSlice broadcast) {
   if (adnl::AdnlNodeIdShort{src} == local_id_) {
     return;
@@ -77,6 +94,19 @@ void FullNodePrivateBlockOverlay::send_shard_block_info(BlockIdExt block_id, Cat
   }
 }
 
+void FullNodePrivateBlockOverlay::send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno,
+                                                       td::uint32 validator_set_hash, td::BufferSlice data) {
+  if (!inited_) {
+    return;
+  }
+  VLOG(FULL_NODE_DEBUG) << "Sending newBlockCandidate in private overlay: " << block_id.to_str();
+  auto B = create_serialize_tl_object<ton_api::tonNode_newBlockCandidateBroadcast>(
+      create_tl_block_id(block_id), cc_seqno, validator_set_hash,
+      create_tl_object<ton_api::tonNode_blockSignature>(Bits256::zero(), td::BufferSlice()), std::move(data));
+  td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_,
+                          local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), std::move(B));
+}
+
 void FullNodePrivateBlockOverlay::send_broadcast(BlockBroadcast broadcast) {
   if (!inited_) {
     return;
@@ -199,6 +229,28 @@ void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNod
                           std::move(query.message_->data_), it->second);
 }
 
+void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query) {
+  if (!block_senders_.count(adnl::AdnlNodeIdShort(src))) {
+    VLOG(FULL_NODE_DEBUG) << "Dropping block candidate broadcast in private overlay \"" << name_
+                          << "\" from unauthorized sender " << src;
+    return;
+  }
+  if (query.data_.size() > FullNode::max_block_size()) {
+    VLOG(FULL_NODE_WARNING) << "received block candidate with too big size from " << src;
+    return;
+  }
+  BlockIdExt block_id = create_block_id(query.id_);
+  if (td::sha256_bits256(query.data_.as_slice()) != block_id.file_hash) {
+    VLOG(FULL_NODE_WARNING) << "received block candidate with incorrect file hash from " << src;
+    return;
+  }
+  // ignore cc_seqno and validator_hash for now
+  VLOG(FULL_NODE_DEBUG) << "Received newBlockCandidate in custom overlay \"" << name_ << "\" from " << src << ": "
+                        << block_id.to_str();
+  td::actor::send_closure(full_node_, &FullNode::process_block_candidate_broadcast, block_id, query.catchain_seqno_,
+                          query.validator_set_hash_, std::move(query.data_));
+}
+
 void FullNodeCustomOverlay::receive_broadcast(PublicKeyHash src, td::BufferSlice broadcast) {
   if (adnl::AdnlNodeIdShort{src} == local_id_) {
     return;
@@ -241,6 +293,19 @@ void FullNodeCustomOverlay::send_broadcast(BlockBroadcast broadcast) {
                           local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok());
 }
 
+void FullNodeCustomOverlay::send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno,
+                                                 td::uint32 validator_set_hash, td::BufferSlice data) {
+  if (!inited_) {
+    return;
+  }
+  VLOG(FULL_NODE_DEBUG) << "Sending newBlockCandidate in custom overlay \"" << name_ << "\": " << block_id.to_str();
+  auto B = create_serialize_tl_object<ton_api::tonNode_newBlockCandidateBroadcast>(
+      create_tl_block_id(block_id), cc_seqno, validator_set_hash,
+      create_tl_object<ton_api::tonNode_blockSignature>(Bits256::zero(), td::BufferSlice()), std::move(data));
+  td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_,
+                          local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), std::move(B));
+}
+
 void FullNodeCustomOverlay::start_up() {
   std::sort(nodes_.begin(), nodes_.end());
   nodes_.erase(std::unique(nodes_.begin(), nodes_.end()), nodes_.end());

diff --git a/validator/full-node-private-overlay.hpp b/validator/full-node-private-overlay.hpp
@@ -27,13 +27,16 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor {
   void process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query);
 
   void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query);
+  void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query);
   template <class T>
   void process_broadcast(PublicKeyHash, T &) {
     VLOG(FULL_NODE_WARNING) << "dropping unknown broadcast";
   }
   void receive_broadcast(PublicKeyHash src, td::BufferSlice query);
 
   void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data);
+  void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash,
+                            td::BufferSlice data);
   void send_broadcast(BlockBroadcast broadcast);
 
   void set_config(FullNodeConfig config) {
@@ -98,6 +101,7 @@ class FullNodeCustomOverlay : public td::actor::Actor {
   void process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query);
 
   void process_broadcast(PublicKeyHash src, ton_api::tonNode_externalMessageBroadcast &query);
+  void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query);
   template <class T>
   void process_broadcast(PublicKeyHash, T &) {
     VLOG(FULL_NODE_WARNING) << "dropping unknown broadcast";
@@ -106,6 +110,8 @@ class FullNodeCustomOverlay : public td::actor::Actor {
 
   void send_external_message(td::BufferSlice data);
   void send_broadcast(BlockBroadcast broadcast);
+  void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash,
+                            td::BufferSlice data);
 
   void set_config(FullNodeConfig config) {
     config_ = std::move(config);

diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp
@@ -17,12 +17,14 @@
     Copyright 2017-2020 Telegram Systems LLP
 */
 #include "auto/tl/ton_api.h"
+#include "checksum.h"
 #include "overlays.h"
 #include "td/utils/SharedSlice.h"
 #include "full-node-shard.hpp"
 #include "full-node-shard-queries.hpp"
 #include "full-node-serializer.hpp"
 
+#include "td/utils/buffer.h"
 #include "ton/ton-shard.h"
 #include "ton/ton-tl.hpp"
 #include "ton/ton-io.hpp"
@@ -646,6 +648,22 @@ void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_ne
                           query.block_->cc_seqno_, std::move(query.block_->data_));
 }
 
+void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query) {
+  if (query.data_.size() > FullNode::max_block_size()) {
+    VLOG(FULL_NODE_WARNING) << "received block candidate with too big size from " << src;
+    return;
+  }
+  BlockIdExt block_id = create_block_id(query.id_);
+  if (td::sha256_bits256(query.data_.as_slice()) != block_id.file_hash) {
+    VLOG(FULL_NODE_WARNING) << "received block candidate with incorrect file hash from " << src;
+    return;
+  }
+  // ignore cc_seqno and validator_hash for now
+  VLOG(FULL_NODE_DEBUG) << "Received newBlockCandidate from " << src << ": " << block_id.to_str();
+  td::actor::send_closure(full_node_, &FullNode::process_block_candidate_broadcast, block_id, query.catchain_seqno_,
+                          query.validator_set_hash_, std::move(query.data_));
+}
+
 void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query) {
   process_block_broadcast(src, query);
 }
@@ -738,6 +756,20 @@ void FullNodeShardImpl::send_shard_block_info(BlockIdExt block_id, CatchainSeqno
   }
 }
 
+void FullNodeShardImpl::send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash,
+                                             td::BufferSlice data) {
+  if (!client_.empty()) {
+    UNREACHABLE();
+    return;
+  }
+  VLOG(FULL_NODE_DEBUG) << "Sending newBlockCandidate: " << block_id.to_str();
+  auto B = create_serialize_tl_object<ton_api::tonNode_newBlockCandidateBroadcast>(
+      create_tl_block_id(block_id), cc_seqno, validator_set_hash,
+      create_tl_object<ton_api::tonNode_blockSignature>(Bits256::zero(), td::BufferSlice()), std::move(data));
+  td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, adnl_id_, overlay_id_, local_id_,
+                          overlay::Overlays::BroadcastFlagAnySender(), std::move(B));
+}
+
 void FullNodeShardImpl::send_broadcast(BlockBroadcast broadcast) {
   if (!client_.empty()) {
     UNREACHABLE();

diff --git a/validator/full-node-shard.h b/validator/full-node-shard.h
@@ -41,6 +41,8 @@ class FullNodeShard : public td::actor::Actor {
   virtual void send_ihr_message(td::BufferSlice data) = 0;
   virtual void send_external_message(td::BufferSlice data) = 0;
   virtual void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) = 0;
+  virtual void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash,
+                                    td::BufferSlice data) = 0;
   virtual void send_broadcast(BlockBroadcast broadcast) = 0;
 
   virtual void sign_overlay_certificate(PublicKeyHash signed_key, td::uint32 expiry_at, td::uint32 max_size, td::Promise<td::BufferSlice> promise) = 0;