From ef94a26d30c747e4bcce7db64e0d9b47f05be7c7 Mon Sep 17 00:00:00 2001 From: Dmitry Shtukenberg Date: Thu, 1 Feb 2024 18:08:38 +0200 Subject: [PATCH 01/40] Remp message bodies broadcasts, initial release --- src/validator/message_cache.rs | 302 +++++++++++++++++------- src/validator/reliable_message_queue.rs | 170 +++++++------ src/validator/remp_block_parser.rs | 2 +- src/validator/remp_catchain.rs | 38 +-- src/validator/remp_manager.rs | 93 ++++---- 5 files changed, 366 insertions(+), 239 deletions(-) diff --git a/src/validator/message_cache.rs b/src/validator/message_cache.rs index a23ea2d9..c1e318dd 100644 --- a/src/validator/message_cache.rs +++ b/src/validator/message_cache.rs @@ -43,7 +43,8 @@ use ton_api::{ IntoBoxed, ton::ton_node::{ rempmessagestatus::{RempAccepted, RempIgnored}, - RempMessageStatus, RempMessageLevel + RempMessageStatus, RempMessageLevel, + rempcatchainrecordv2::RempCatchainMessageHeaderV2 } }; @@ -51,6 +52,7 @@ use ton_block::{ Deserializable, Message, Serializable, MsgAddressInt, MsgAddrStd, ExternalInboundMessageHeader, BlockIdExt, UnixTime32 }; + use ton_types::{error, fail, KeyId, SliceData, Result, UInt256}; #[cfg(test)] @@ -62,67 +64,23 @@ pub struct RmqMessage { pub message: Arc, pub message_id: UInt256, pub message_uid: UInt256, - pub source_key: Arc, - pub source_idx: u32, - pub timestamp: u32, } impl RmqMessage { - pub fn new(message: Arc, message_id: UInt256, message_uid: UInt256, source_key: Arc, source_idx: u32) -> Result { - return Ok(RmqMessage { message, message_id, message_uid, source_key, source_idx, timestamp: Self::timestamp_now()? }) - } - - pub fn from_rmq_record(record: &ton_api::ton::ton_node::rempcatchainrecord::RempCatchainMessage) -> Result { - let message= Arc::new(Message::construct_from_bytes(&record.message)?); - Ok(RmqMessage { - message: message.clone(), - message_id: record.message_id.clone(), - message_uid: get_message_uid(&message), - source_key: KeyId::from_data(record.source_key_id.as_slice().clone()), - source_idx: record.source_idx as u32, - timestamp: Self::timestamp_now()? - }) - } - - fn timestamp_now() -> Result { - Ok(SystemTime::now().duration_since(SystemTime::UNIX_EPOCH)?.as_secs() as u32) + pub fn new(message: Arc, message_id: UInt256, message_uid: UInt256) -> Result { + return Ok(RmqMessage { message, message_id, message_uid }) } - pub fn new_with_updated_source_idx(&self, source_idx: u32) -> Self { - RmqMessage { - message: self.message.clone(), - message_id: self.message_id.clone(), - message_uid: self.message_uid.clone(), - source_key: self.source_key.clone(), - source_idx, - timestamp: self.timestamp - } - } - - pub fn has_no_source_key(&self) -> bool { - self.source_key.data().to_vec().iter().all(|x| *x == 0) - } - - pub fn deserialize(raw: &ton_api::ton::bytes) -> Result { - let rmq_record: ton_api::ton::ton_node::RempCatchainRecord = catchain::utils::deserialize_tl_boxed_object(&raw)?; - Ok(rmq_record) - } - - pub fn as_rmq_record(&self, master_cc: u32) -> ton_api::ton::ton_node::RempCatchainRecord { - ton_api::ton::ton_node::rempcatchainrecord::RempCatchainMessage { - message: self.message.write_to_bytes().unwrap().into(), + pub fn as_remp_catchain_record(&self, master_cc: u32, origin: &RempMessageOrigin) -> ton_api::ton::ton_node::RempCatchainRecordV2 { + ton_api::ton::ton_node::rempcatchainrecordv2::RempCatchainMessageHeaderV2 { message_id: self.message_id.clone().into(), - source_key_id: UInt256::from(self.source_key.data()), - source_idx: self.source_idx as i32, + message_uid: self.message_uid.clone().into(), + source_key_id: UInt256::from(origin.source_key.data()), + source_idx: origin.source_idx as i32, masterchain_seqno: master_cc as i32 }.into_boxed() } - pub fn serialize(rmq_record: &ton_api::ton::ton_node::RempCatchainRecord) -> Result { - let rmq_record_serialized = serialize_tl_boxed_object!(rmq_record); - return Ok(rmq_record_serialized) - } - #[allow(dead_code)] pub fn make_test_message(body: &SliceData) -> Result { let address = UInt256::rand(); @@ -151,21 +109,14 @@ impl RmqMessage { ); let (msg_id, msg_uid, msg) = (msg_cell.repr_hash(), get_message_uid(&msg), msg); - RmqMessage::new (Arc::new(msg), msg_id, msg_uid, KeyId::from_data([0; 32]), 0) + RmqMessage::new (Arc::new(msg), msg_id, msg_uid) } } impl fmt::Display for RmqMessage { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let source_key = if self.has_no_source_key() { - "(broadcast)".to_owned() - } - else { - format!("{}", self.source_key) - }; - - write!(f, "id {:x}, uid {:x}, source {}, source_idx {}, ts {}", - self.message_id, self.message_uid, source_key, self.source_idx, self.timestamp + write!(f, "fullmsg, id {:x}, uid {:x}", + self.message_id, self.message_uid ) } } @@ -173,15 +124,56 @@ impl fmt::Display for RmqMessage { #[derive(Debug,PartialEq,Eq)] pub struct RempMessageHeader { pub message_id: UInt256, - pub message_uid: UInt256 + pub message_uid: UInt256, } impl RempMessageHeader { - pub fn new_arc(message_id: &UInt256, message_uid: &UInt256) -> Arc { - Arc::new(RempMessageHeader { + pub fn new(message_id: &UInt256, message_uid: &UInt256) -> Self { + RempMessageHeader { message_id: message_id.clone(), message_uid: message_uid.clone() - }) + } + } + + pub fn new_arc(message_id: &UInt256, message_uid: &UInt256) -> Arc { + Arc::new(Self::new(message_id, message_uid)) + } +/* + pub fn from_rmq_record(record: &ton_api::ton::ton_node::rempcatchainrecordv2::RempCatchainMessageHeaderV2) -> Result<(RempMessageHeader, RempMessageOrigin)> { + let header = RempMessageHeader { + message_id: record.message_id.clone(), + message_uid: record.message_uid.clone(), + }; + let origin = RempMessageOrigin { + source_key: KeyId::from_data(record.source_key_id.as_slice().clone()), + source_idx: record.source_idx as u32, + timestamp: RempMessageOrigin::timestamp_now()? + }; + Ok((header, origin)) + } +*/ + pub fn deserialize(raw: &ton_api::ton::bytes) -> Result { + let rmq_record: ton_api::ton::ton_node::RempCatchainRecordV2 = catchain::utils::deserialize_tl_boxed_object(&raw)?; + Ok(rmq_record) + } +/* + pub fn as_remp_catchain_record(&self, master_cc: u32, origin: &RempMessageOrigin) -> ton_api::ton::ton_node::RempCatchainRecordV2 { + ton_api::ton::ton_node::rempcatchainrecordv2::RempCatchainMessageHeaderV2 { + message_id: self.message_id.clone().into(), + message_uid: self.message_uid.clone().into(), + source_key_id: UInt256::from(origin.source_key.data()), + source_idx: origin.source_idx as i32, + masterchain_seqno: master_cc as i32 + }.into_boxed() + } +*/ + pub fn serialize(rmq_record: &ton_api::ton::ton_node::RempCatchainRecordV2) -> Result { + let rmq_record_serialized = serialize_tl_boxed_object!(rmq_record); + return Ok(rmq_record_serialized) + } + + pub fn from_remp_catchain(record: &RempCatchainMessageHeaderV2) -> Result { + Ok(Self::new(&record.message_id, &record.message_uid)) } } @@ -193,6 +185,95 @@ impl Display for RempMessageHeader { } } +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct RempMessageOrigin { + pub source_key: Arc, + pub source_idx: u32, + pub timestamp: u32, +} + +impl RempMessageOrigin { + pub fn new(source_key: Arc, source_idx: u32) -> Result { + Ok(RempMessageOrigin { + source_key, + source_idx, + timestamp: Self::timestamp_now()? + }) + } + + pub fn new_with_updated_source_idx(&self, source_idx: u32) -> Self { + RempMessageOrigin { + source_key: self.source_key.clone(), + source_idx, + timestamp: self.timestamp + } + } + + pub fn from_remp_catchain(record: &RempCatchainMessageHeaderV2) -> Result { + Self::new( + KeyId::from_data(record.source_key_id.as_slice().clone()), + record.source_idx as u32 + ) + } + + pub fn has_no_source_key(&self) -> bool { + self.source_key.data().to_vec().iter().all(|x| *x == 0) + } + + fn timestamp_now() -> Result { + Ok(SystemTime::now().duration_since(SystemTime::UNIX_EPOCH)?.as_secs() as u32) + } + + pub fn create_empty() -> Result { + Ok(RempMessageOrigin { + source_key: KeyId::from_data([0; 32]), + source_idx: 0, + timestamp: RempMessageOrigin::timestamp_now()? + }) + } +} + +impl Display for RempMessageOrigin { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + let source_key = if self.has_no_source_key() { + "(broadcast)".to_owned() + } + else { + format!("{}", self.source_key) + }; + + write!(f, "source_key {}, source_idx {}, timestamp {}", + source_key, self.source_idx, self.timestamp + ) + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct RempMessageWithOrigin { + pub message: RmqMessage, + pub origin: RempMessageOrigin +} + +impl RempMessageWithOrigin { + pub fn get_message_id(&self) -> &UInt256 { + return &self.message.message_id; + } + + pub fn has_no_source_key(&self) -> bool { + return self.origin.has_no_source_key(); + } + + pub fn as_remp_catchain_record(&self, master_cc: u32) -> ton_api::ton::ton_node::RempCatchainRecordV2 { + return self.message.as_remp_catchain_record(master_cc, &self.origin) + } +} + +impl Display for RempMessageWithOrigin { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "{}, {}", self.message, self.origin) + } +} + pub struct MessageCacheSession { master_cc: u32, @@ -208,6 +289,7 @@ pub struct MessageCacheSession { ids_for_uid: LockfreeMapSet, message_headers: DashMap>, + message_origins: DashMap>, messages: Map>, message_events: LockfreeMapSet, //Map>, message_status: DashMap, @@ -222,7 +304,7 @@ impl Display for MessageCacheSession { } impl MessageCacheSession { - fn insert_message_header(&self, msg_id: &UInt256, msg_hdr: Arc) -> Result<()> { + fn insert_message_header(&self, msg_id: &UInt256, msg_hdr: Arc, msg_origin: Option>) -> Result<()> { if msg_id != &msg_hdr.message_id { fail!("Message with id {:x} and its header {} have different ids", msg_id, msg_hdr); } @@ -239,15 +321,25 @@ impl MessageCacheSession { fail!("Message with id {:x} changed its header from {} to {}", msg_id, old_hdr, msg_hdr); } } + + match msg_origin { + Some(origin) => if let Some(old_origin) = self.message_origins.insert(msg_id.clone(), origin.clone()) { + fail!("Message with id {:x} changed its origin from {} to {}", msg_id, origin, old_origin) + }, + None => if let Some(old_origin) = self.message_origins.get(msg_id) { + fail!("Message with id {:x} already has its origin in cache: {}, although we do not know it at insertion", msg_id, old_origin.value()) + } + } + Ok(()) } - fn insert_message(&self, msg: Arc, msg_hdr: Arc) -> Result<()> { + fn insert_message(&self, msg: Arc, msg_hdr: Arc, msg_origin: Option>) -> Result<()> { if msg.message_uid != msg_hdr.message_uid || msg.message_id != msg_hdr.message_id { fail!("Message with id {:x} and uid {:x} and its header {} have different uids or ids", msg.message_id, msg.message_uid, msg_hdr); } - self.insert_message_header(&msg.message_id, msg_hdr)?; + self.insert_message_header(&msg.message_id, msg_hdr, msg_origin)?; match self.messages.insert(msg.message_id.clone(), msg.clone()) { None => Ok(()), Some(prev) if *prev.val() == msg => Ok(()), @@ -257,6 +349,26 @@ impl MessageCacheSession { } } + fn ensure_same_message(&self, message_id: &UInt256, message: Option>, origin: Option>) -> Result<()> { + if let Some(m1) = &message { + if let Some(m2) = self.messages.get(message_id) { + if m1 != m2.val() { + fail!("Different message body: new body '{}', current cached body '{}'", m1, m2.val()); + } + } + } + + if let Some(o1) = &origin { + if let Some(o2) = self.message_origins.get(message_id) { + if o1 != o2.value() { + fail!("Different message origin: new origin '{}', current cached origin '{}'", o1, o2.value()); + } + } + } + + Ok(()) + } + fn is_message_present(&self, msg_id: &UInt256) -> bool { self.message_headers.contains_key(msg_id) } @@ -305,13 +417,13 @@ impl MessageCacheSession { } fn message_events_to_string(&self, message_id: &UInt256) -> String { - if let Some(msg) = self.messages.get(message_id) { - let base = msg.val().timestamp; + if let Some(msg) = self.message_origins.get(message_id) { + let base = msg.value().timestamp; let events = self.message_events.get_set(message_id); events.iter().map(|x| format!("{} ", (*x) as i64 - base as i64)).collect() } else { - "*events: no message base time*".to_string() + "*events: no message origin*".to_string() } } @@ -374,6 +486,7 @@ impl MessageCacheSession { start_time, ids_for_uid: LockfreeMapSet::default(), message_headers: DashMap::new(), + message_origins: DashMap::new(), message_events: LockfreeMapSet::default(), messages: Map::default(), message_status: DashMap::default(), @@ -487,29 +600,34 @@ impl MessageCache { } } - pub fn get_message_with_status(&self, message_id: &UInt256) -> Result, RempMessageStatus)>> { - self.get_message_with_status_cc(message_id).map(|e| e.map(|(m,s,_c)| (m,s))) - } - - pub fn get_message_with_status_cc(&self, message_id: &UInt256) -> Result, RempMessageStatus, u32)>> { + pub fn get_message_with_origin_status_cc(&self, message_id: &UInt256) -> Result, Arc, RempMessageStatus, u32)>> { let session = match self.get_session_for_message(message_id) { None => return Ok(None), Some(s) => s }; - let (msg, status) = ( + let (msg, origin, status) = ( session.messages.get(message_id).map(|m| m.val().clone()), + session.message_origins.get(message_id).map(|m| m.value().clone()), session.message_status.get(message_id).map(|m| m.value().clone()) ); - match (msg, status) { - (None, Some(_)) => Ok(None), // Bare message info (retrieved from finalized block) - (Some(m), Some (h)) => Ok(Some((m.clone(),h.clone(),session.master_cc))), // Full message info - (m, None) => fail!("Message {:x} has no status, body = {:?}", message_id, m) + match (msg, origin, status) { + (None, _, Some(_)) => Ok(None), // Bare message info (retrieved from finalized block) + (Some(m), Some(o), Some (h)) => Ok(Some((m.clone(), o.clone(), h.clone(), session.master_cc))), // Full message info + (m, o, None) => fail!("Message {:x} has no status, body = {:?}, origin = {:?}", message_id, m, o), + (Some(m), None, Some(h)) => fail!("Message {:x} has no orgin, but has body {} and status {}", message_id, m, h) } } - fn insert_message(&self, session: Arc, message: Arc, message_header: Arc, status: &RempMessageStatus) -> Result<()> { + pub fn get_message_origin(&self, message_id: &UInt256) -> Result>> { + match self.get_session_for_message(message_id) { + None => Ok(None), + Some(s) => Ok(s.message_origins.get(message_id).map(|m| m.value().clone())) + } + } + + fn insert_message(&self, session: Arc, message: Arc, message_header: Arc, message_origin: Arc, status: &RempMessageStatus) -> Result<()> { if message.message_id != message_header.message_id { fail!("Inconsistent message: message {} and message_header {} have different message_id", message, message_header) } @@ -521,18 +639,18 @@ impl MessageCache { } session.message_status.insert(message_id.clone(), status.clone()); - session.insert_message(message, message_header)?; + session.insert_message(message, message_header, Some(message_origin))?; Ok(()) } - fn insert_message_header(&self, session: Arc, message_header: Arc, status: &RempMessageStatus) -> Result<()> { + fn insert_message_header(&self, session: Arc, message_header: Arc, message_origin: Option>, status: &RempMessageStatus) -> Result<()> { let message_id = message_header.message_id.clone(); if session.is_message_present(&message_id) { fail!("Inconsistent message cache contents: message header {:x} present in cache, although should not", message_id) } session.message_status.insert(message_id.clone(), status.clone()); - session.insert_message_header(&message_id, message_header)?; + session.insert_message_header(&message_id, message_header, message_origin)?; Ok(()) } @@ -542,7 +660,7 @@ impl MessageCache { /// Actual -- get it as granted ("imprinting") /// Returns old status and new (added) status pub async fn add_external_message_status(&self, - message_id: &UInt256, message_uid: &UInt256, message: Option>, + message_id: &UInt256, message_uid: &UInt256, message: Option>, message_origin: Option>, status_if_new: RempMessageStatus, status_updater: F, master_cc: u32 ) -> Result<(Option,RempMessageStatus)> @@ -561,13 +679,17 @@ impl MessageCache { message_uid ); - match message { - None => self.insert_message_header( session, header, &status_if_new)?, - Some(message) => self.insert_message(session, message, header, &status_if_new)? + match (message, &message_origin) { + (None,_) => self.insert_message_header( session, header, message_origin, &status_if_new)?, + (Some(message), Some(origin)) => self.insert_message(session, message, header, origin.clone(), &status_if_new)?, + (Some(message), None) => fail!("Incorrect options for add_external_message_status: message header {}, body {}, but no message origin", header, message) }; Ok((None, status_if_new)) }, Some(session) => { + if let Err(e) = session.ensure_same_message(&message_id, message, message_origin) { + log::error!(target: "remp", "Different cache contents for external message {}: {}", message_id, e); + } let (old_status, final_status) = session.alter_message_status(&message_id, |old| status_updater(old,&status_if_new))?; Ok((Some(old_status), final_status)) diff --git a/src/validator/reliable_message_queue.rs b/src/validator/reliable_message_queue.rs index 7baddebe..c817daa5 100644 --- a/src/validator/reliable_message_queue.rs +++ b/src/validator/reliable_message_queue.rs @@ -12,18 +12,21 @@ */ use std::{ + cmp::Reverse, collections::{BinaryHeap, HashMap}, fmt, fmt::Formatter, + ops::RangeInclusive, sync::Arc, - time::Duration, time::SystemTime + time::{Duration, SystemTime} }; -use std::cmp::Reverse; -use std::ops::RangeInclusive; use dashmap::DashMap; use ton_api::ton::ton_node::{ RempMessageStatus, RempMessageLevel, - rempmessagestatus::{RempAccepted, RempIgnored, RempRejected}, RempCatchainRecord + rempmessagestatus::{RempAccepted, RempIgnored, RempRejected}, + RempMessageStatus::TonNode_RempRejected, + RempCatchainRecordV2, + rempcatchainrecordv2::{RempCatchainMessageHeaderV2, RempCatchainMessageDigestV2} }; use ton_block::{ShardIdent, Message, BlockIdExt, ValidatorDescr}; use ton_types::{UInt256, Result, fail}; @@ -33,7 +36,7 @@ use crate::{ ext_messages::{get_level_and_level_change, is_finally_accepted, is_finally_rejected}, validator::{ mutex_wrapper::MutexWrapper, - message_cache::RmqMessage, + message_cache::{RmqMessage, RempMessageHeader, RempMessageOrigin, RempMessageWithOrigin}, remp_manager::RempManager, remp_block_parser::{process_block_messages_by_blockid, BlockProcessor}, remp_catchain::{RempCatchainInfo, RempCatchainInstance}, @@ -42,8 +45,6 @@ use crate::{ } }; use failure::err_msg; -use ton_api::ton::ton_node::rempcatchainrecord::{RempCatchainMessage, RempCatchainMessageDigest}; -use ton_api::ton::ton_node::RempMessageStatus::TonNode_RempRejected; use crate::block::BlockIdExtExtention; use crate::engine_traits::RempDuplicateStatus; @@ -158,26 +159,26 @@ impl MessageQueue { }).await } - pub fn send_response_to_fullnode(&self, rmq_message: Arc, status: RempMessageStatus) { - log::debug!(target: "remp", "RMQ {}: queueing response to fullnode {}, status {}", - self, rmq_message, status + pub fn send_response_to_fullnode(&self, message_id: &UInt256, origin: Arc, status: RempMessageStatus) { + log::debug!(target: "remp", "RMQ {}: queueing response to fullnode {:x}, status {}", + self, message_id, status ); - if rmq_message.has_no_source_key() { - log::trace!(target: "remp", "RMQ {}: message {} was broadcast and has no source key, no response", self, rmq_message) + if origin.has_no_source_key() { + log::trace!(target: "remp", "RMQ {}: message {:x} was broadcast and has no source key, no response", self, message_id) } else if let Err(e) = self.remp_manager.queue_response_to_fullnode( - self.catchain_info.local_key_id.clone(), rmq_message.clone(), status.clone() + self.catchain_info.local_key_id.clone(), message_id.clone(), origin.clone(), status.clone() ) { - log::error!(target: "remp", "RMQ {}: cannot queue response to fullnode: {}, {}, local key {:x}, error `{}`", - self, rmq_message, status, self.catchain_info.local_key_id, e + log::error!(target: "remp", "RMQ {}: cannot queue response to fullnode: message id {} from {}, {}, local key {:x}, error `{}`", + self, message_id, origin, status, self.catchain_info.local_key_id, e ); } } - pub fn update_status_send_response(&self, msgid: &UInt256, message: Arc, new_status: RempMessageStatus) { + pub fn update_status_send_response(&self, msgid: &UInt256, origin: Arc, new_status: RempMessageStatus) { match self.remp_manager.message_cache.update_message_status(&msgid, new_status.clone()) { - Ok(Some(final_status)) => self.send_response_to_fullnode(message.clone(), final_status), + Ok(Some(final_status)) => self.send_response_to_fullnode(msgid, origin, final_status), Ok(None) => (), // Send nothing, no status update is requested Err(e) => log::error!(target: "remp", "RMQ {}: Cannot update status for {:x}, new status {}, error {}", @@ -187,10 +188,9 @@ impl MessageQueue { } pub async fn update_status_send_response_by_id(&self, msgid: &UInt256, new_status: RempMessageStatus) -> Result> { - let message = self.get_message(msgid)?; - match &message { - Some(rm) => { - self.update_status_send_response(msgid, rm.clone(), new_status.clone()); + match &self.remp_manager.message_cache.get_message_with_origin_status_cc(msgid)? { + Some((rm,origin,_,_)) => { + self.update_status_send_response(msgid, origin.clone(), new_status.clone()); Ok(rm.clone()) }, None => @@ -250,15 +250,15 @@ impl MessageQueue { } } - pub async fn put_message_to_rmq(&self, old_message: Arc) -> Result<()> { - if self.queues.execute_sync(|q| q.pending_collation_set.contains_key(&old_message.message_id)).await { - log::trace!(target: "remp", "Point 3. RMQ {}; computing message {} delay --- already have it in local queue, should be skipped", self, old_message); + pub async fn put_message_to_rmq(&self, msg: Arc) -> Result<()> { + if self.queues.execute_sync(|q| q.pending_collation_set.contains_key(msg.get_message_id())).await { + log::trace!(target: "remp", "Point 3. RMQ {}; computing message {} delay --- already have it in local queue, should be skipped", self, msg); return Ok(()) } - let msg = Arc::new(old_message.new_with_updated_source_idx(self.catchain_info.local_idx as u32)); - log::trace!(target: "remp", "Point 3. Pushing to RMQ {}; message {}", self, msg); - self.catchain_instance.pending_messages_queue_send(msg.as_rmq_record(self.catchain_info.get_master_cc_seqno()))?; + let origin_with_idx = Arc::new(msg.origin.new_with_updated_source_idx(self.catchain_info.local_idx as u32)); + log::trace!(target: "remp", "Point 3. Pushing to RMQ {}; message {}, {}", self, msg, origin_with_idx); + self.catchain_instance.pending_messages_queue_send(msg.as_remp_catchain_record(self.catchain_info.get_master_cc_seqno()))?; #[cfg(feature = "telemetry")] self.engine.remp_core_telemetry().in_channel_to_catchain( @@ -269,7 +269,7 @@ impl MessageQueue { session.request_new_block(SystemTime::now() + RMQ_REQUEST_NEW_BLOCK_INTERVAL); // Temporary status "New" --- message is not registered yet - self.send_response_to_fullnode(msg, RempMessageStatus::TonNode_RempNew); + self.send_response_to_fullnode(msg.get_message_id(), origin_with_idx, RempMessageStatus::TonNode_RempNew); Ok(()) } else { @@ -278,10 +278,10 @@ impl MessageQueue { } } - async fn add_pending_collation(&self, rmq_message: Arc, status_to_send: Option) -> Result<()> { + async fn add_pending_collation(&self, message_id: &UInt256, remp_message_origin: Arc, status_to_send: Option) -> Result<()> { let (added_to_queue, _len) = self.queues.execute_sync( |catchain| catchain.add_to_collation_queue( - &rmq_message.message_id, rmq_message.timestamp, true + message_id, remp_message_origin.timestamp, true ) ).await?; @@ -293,11 +293,11 @@ impl MessageQueue { if added_to_queue { log::trace!(target: "remp", - "Point 5. RMQ {}: adding message {} to collator queue", self, rmq_message + "Point 5. RMQ {}: adding message {:x} to collator queue", self, message_id ); - self.remp_manager.message_cache.mark_collation_attempt(&rmq_message.message_id)?; + self.remp_manager.message_cache.mark_collation_attempt(message_id)?; if let Some(status) = status_to_send { - self.send_response_to_fullnode(rmq_message.clone(), status); + self.send_response_to_fullnode(message_id, remp_message_origin, status); } } @@ -353,14 +353,15 @@ impl MessageQueue { self.catchain_instance.is_session_active() } - async fn process_pending_remp_catchain_message(&self, rmq_record_message: &RempCatchainMessage) -> Result<()> { - let rmq_message = Arc::new(RmqMessage::from_rmq_record(rmq_record_message)?); - let rmq_message_master_seqno = rmq_record_message.masterchain_seqno as u32; - let forwarded = self.catchain_info.get_master_cc_seqno() > rmq_message_master_seqno; + async fn process_pending_remp_catchain_message(&self, catchain_record: &RempCatchainMessageHeaderV2) -> Result<()> { + let remp_message_header = Arc::new(RempMessageHeader::from_remp_catchain(catchain_record)?); + let remp_message_origin = Arc::new(RempMessageOrigin::from_remp_catchain(catchain_record)?); + let message_master_seqno = catchain_record.masterchain_seqno as u32; + let forwarded = self.catchain_info.get_master_cc_seqno() > message_master_seqno; log::trace!(target: "remp", - "Point 4. RMQ {}: inserting pending message {} from RMQ into message_cache, forwarded {}, message_master_cc {}", - self, rmq_message, forwarded, rmq_message_master_seqno + "Point 4. RMQ {}: inserting pending message {}, {} from RMQ into message_cache, forwarded {}, message_master_cc {}", + self, remp_message_header, remp_message_origin, forwarded, message_master_seqno ); let status_if_new = if let Some((_msg, status)) = self.is_queue_overloaded().await { @@ -378,9 +379,10 @@ impl MessageQueue { // 1c. We know it with some normal status -- leave in place our knowledge. // 1d. We know only forwarded message status -- replace it with new status. let added = self.remp_manager.message_cache.add_external_message_status( - &rmq_message.message_id, - &rmq_message.message_uid, - Some(rmq_message.clone()), + &remp_message_header.message_id, + &remp_message_header.message_uid, + None, + Some(remp_message_origin.clone()), status_if_new, |old_status, new_status| { if Self::is_forwarded_status(old_status) { @@ -398,7 +400,7 @@ impl MessageQueue { None => { log::error!(target: "remp", "RMQ {}: cannot increment level {:?} for message_id {:x}", - self, old_status, rmq_message.message_id + self, old_status, remp_message_header.message_id ); RempMessageLevel::TonNode_RempMasterchain } @@ -412,20 +414,20 @@ impl MessageQueue { } old_status.clone() }, - rmq_message_master_seqno + message_master_seqno ).await; match added { Err(e) => { log::error!(target: "remp", "Point 4. RMQ {}: cannot insert new message {} into message_cache, error: `{}`", - self, rmq_message, e + self, remp_message_header, e ); }, Ok((Some(_),new_status)) if Self::is_final_status(&new_status) => { log::trace!(target: "remp", "Point 4. RMQ {}. Message {:x} master_cc_seqno {} from validator {} has final status {}, skipping", - self, rmq_message.message_id, rmq_message_master_seqno, rmq_message.source_idx, new_status + self, remp_message_header.message_id, message_master_seqno, remp_message_origin.source_idx, new_status ); #[cfg(feature = "telemetry")] self.engine.remp_core_telemetry().add_to_cache_attempt(false); @@ -433,13 +435,13 @@ impl MessageQueue { Ok((old_status,new_status)) => { log::trace!(target: "remp", "Point 4. RMQ {}. Message {:x} master_cc_seqno {} from validator {} has non-final status {}{}, will be collated", - self, rmq_message.message_id, rmq_message_master_seqno, rmq_message.source_idx, new_status, + self, remp_message_header.message_id, message_master_seqno, remp_message_origin.source_idx, new_status, match &old_status { None => format!(" (no old status)"), Some(x) => format!(" (old status {})", x) } ); - self.add_pending_collation(rmq_message, Some(new_status)).await?; + self.add_pending_collation(&remp_message_header.message_id, remp_message_origin, Some(new_status)).await?; #[cfg(feature = "telemetry")] self.engine.remp_core_telemetry().add_to_cache_attempt(true); } @@ -447,7 +449,7 @@ impl MessageQueue { Ok(()) } - async fn process_pending_remp_catchain_digest(&self, reject_digest: &RempCatchainMessageDigest) -> Result<()> { + async fn process_pending_remp_catchain_digest(&self, reject_digest: &RempCatchainMessageDigestV2) -> Result<()> { if !self.catchain_info.master_cc_range.contains(&(reject_digest.masterchain_seqno as u32)) { log::error!(target: "remp", "Point 4. RMQ {}. Message digest (masterchain_seqno = {}, len = {}) does not fit to RMQ master cc range {}", @@ -462,7 +464,7 @@ impl MessageQueue { for message_ids in reject_digest.messages.iter() { self.remp_manager.message_cache.add_external_message_status( &message_ids.id, &message_ids.uid, - None, + None, None, Self::forwarded_rejected_status().clone(), // Forwarded reject cannot replace any status: if we know something // to grant re-collation for the message, then this knowledge is @@ -476,11 +478,11 @@ impl MessageQueue { Ok(()) } - async fn process_pending_remp_catchain_record(&self, remp_catchain_record: &RempCatchainRecord) -> Result<()> { + async fn process_pending_remp_catchain_record(&self, remp_catchain_record: &RempCatchainRecordV2) -> Result<()> { match remp_catchain_record { - RempCatchainRecord::TonNode_RempCatchainMessage(msg) => + RempCatchainRecordV2::TonNode_RempCatchainMessageHeaderV2(msg) => self.process_pending_remp_catchain_message(msg).await, - RempCatchainRecord::TonNode_RempCatchainMessageDigest(digest) => + RempCatchainRecordV2::TonNode_RempCatchainMessageDigestV2(digest) => self.process_pending_remp_catchain_digest(digest).await } } @@ -523,7 +525,7 @@ impl MessageQueue { log::trace!(target: "remp", "RMQ {}: collecting messages for collation", self); let mut cnt = 0; while let Some((msgid, _timestamp)) = self.queues.execute_sync(|x| x.take_first_for_collation()).await? { - let (status, message) = match self.remp_manager.message_cache.get_message_with_status(&msgid) { + let (message, origin, status) = match self.remp_manager.message_cache.get_message_with_origin_status_cc(&msgid) { Err(e) => { log::error!( target: "remp", @@ -535,12 +537,12 @@ impl MessageQueue { Ok(None) => { log::error!( target: "remp", - "Point 5. RMQ {}: message {:x} found in pending_collation queue, but not in messages/message_statuses", + "Point 5. RMQ {}: message {:x} found in pending_collation queue, but has no body/origin", self, msgid ); continue }, - Ok(Some((m, s))) => (s, m) + Ok(Some((m, o, s, _cc))) => (m, o, s) }; match status.clone() { @@ -569,7 +571,7 @@ impl MessageQueue { block_id: BlockIdExt::default(), error: duplicate_info }; - self.update_status_send_response(&msgid, message.clone(), RempMessageStatus::TonNode_RempRejected(rejected)); + self.update_status_send_response(&msgid, origin.clone(), RempMessageStatus::TonNode_RempRejected(rejected)); } else { log::error!(target: "remp", "Point 5. RMQ {}: message {:x} must not have a final status {:?}", @@ -597,7 +599,7 @@ impl MessageQueue { block_id: BlockIdExt::default(), master_id: BlockIdExt::default() }); - self.update_status_send_response(&msgid, message.clone(), new_status); + self.update_status_send_response(&msgid, origin.clone(), new_status); cnt = cnt + 1; } } @@ -639,9 +641,9 @@ impl MessageQueue { /// Returns message to collation queue of the current collator. /// Message must alreaedy present in the queue. pub async fn return_to_collation_queue(&self, message_id: &UInt256) -> Result<()> { - if let Some(message) = self.remp_manager.message_cache.get_message(message_id)? { + if let Some(origin) = self.remp_manager.message_cache.get_message_origin(message_id)? { self.queues.execute_sync( - |catchain| catchain.add_to_collation_queue(message_id, message.timestamp, false) + |catchain| catchain.add_to_collation_queue(message_id, origin.timestamp, false) ).await?; Ok(()) } @@ -726,10 +728,6 @@ impl MessageQueue { ); } - pub fn get_message(&self, id: &UInt256) -> Result>> { - self.remp_manager.message_cache.get_message(id) - } - async fn get_queue_len(&self) -> usize { self.queues.execute_sync(|q| q.pending_collation_set.len()).await } @@ -800,14 +798,14 @@ struct StatusUpdater { #[async_trait::async_trait] impl BlockProcessor for StatusUpdater { async fn process_message(&self, message_id: &UInt256, _message_uid: &UInt256) { - match self.queue.get_message(message_id) { - Err(e) => log::error!(target: "remp", "Cannot get message {:x} from cache: {}", message_id, e), - Ok(None) => log::warn!(target: "remp", "Cannot find message {:x} in cache", message_id), - Ok(Some(message)) => { - log::trace!(target: "remp", "Point 7. RMQ {} shard accepted message {}, new status {}", - self.queue, message, self.new_status + match self.queue.remp_manager.message_cache.get_message_with_origin_status_cc(message_id) { + Err(e) => log::error!(target: "remp", "Point 7. Cannot get message {:x} from cache: {}", message_id, e), + Ok(None) => log::warn!(target: "remp", "Point 7. Message {:x} is not stored in cache (body or/and origin is missing)", message_id), + Ok(Some((message,origin,_,_))) => { + log::trace!(target: "remp", "Point 7. RMQ {} shard accepted message {}, {}, new status {}", + self.queue, message, origin, self.new_status ); - self.queue.update_status_send_response(message_id, message, self.new_status.clone()) + self.queue.update_status_send_response(message_id, origin, self.new_status.clone()) } } } @@ -954,10 +952,10 @@ impl RmqQueueManager { } //let mut rejected_message_digests: HashMap = HashMap::default(); - let mut rejected_message_digests: Vec = Vec::new(); + let mut rejected_message_digests: Vec = Vec::new(); for msgid in to_forward.iter() { - let (message, message_status, message_cc) = match self.remp_manager.message_cache.get_message_with_status_cc(msgid) { + let (message, origin, message_status, message_cc) = match self.remp_manager.message_cache.get_message_with_origin_status_cc(msgid) { Err(e) => { log::error!( target: "remp", @@ -974,8 +972,8 @@ impl RmqQueueManager { ); continue }, - Ok(Some((_m, status, _cc))) if status == RempMessageStatus::TonNode_RempTimeout => continue, - Ok(Some((_m, _status, cc))) if !new_cc_range.contains(&cc) => { + Ok(Some((_m, _origin, status, _cc))) if status == RempMessageStatus::TonNode_RempTimeout => continue, + Ok(Some((_m, _origin, _status, cc))) if !new_cc_range.contains(&cc) => { if *new_cc_range.end() < cc { log::error!(target: "remp", "Point 5a. RMQ {}: message {:x} is younger (cc={}) than next_cc_range end {}..={} -- impossible", self, msgid, cc, new_cc_range.start(), new_cc_range.end() @@ -983,7 +981,7 @@ impl RmqQueueManager { } continue }, - Ok(Some((m, status, cc))) => (m, status, cc) + Ok(Some((m, origin, status, cc))) => (m, origin, status, cc) }; // Forwarding: @@ -1009,7 +1007,7 @@ impl RmqQueueManager { rejected_message_digests.insert(message_cc, digest); } */ - let mut digest = RempCatchainMessageDigest::default(); + let mut digest = RempCatchainMessageDigestV2::default(); digest.masterchain_seqno = message_cc as i32; digest.messages.0.push(ton_api::ton::ton_node::rempcatchainmessageids::RempCatchainMessageIds { id: message.message_id.clone(), @@ -1026,7 +1024,7 @@ impl RmqQueueManager { } for new in next_queues.iter() { - if let Err(x) = new.catchain_instance.pending_messages_queue_send(message.as_rmq_record(message_cc)) { + if let Err(x) = new.catchain_instance.pending_messages_queue_send(message.as_remp_catchain_record(message_cc, &origin)) { log::error!(target: "remp", "Point 5a. RMQ {}: message {:x} cannot be put to new queue {}: `{}`", self, msgid, new, x @@ -1055,7 +1053,7 @@ impl RmqQueueManager { } */ let digest_len = digest.messages.0.len(); - let msg = ton_api::ton::ton_node::RempCatchainRecord::TonNode_RempCatchainMessageDigest(digest); + let msg = ton_api::ton::ton_node::RempCatchainRecordV2::TonNode_RempCatchainMessageDigestV2(digest); for new in next_queues.iter() { if let Err(x) = new.catchain_instance.pending_messages_queue_send(msg.clone()) { log::error!(target: "remp", @@ -1107,7 +1105,7 @@ impl RmqQueueManager { log::trace!(target: "remp", "Stopping RMQ {} finished, next queues [{}] will be stopped by GC", self, next_queues); } - pub async fn put_message_to_rmq(&self, message: Arc) -> Result<()> { + pub async fn put_message_to_rmq(&self, message: Arc) -> Result<()> { if let Some(cur_queue) = &self.cur_queue { cur_queue.clone().put_message_to_rmq(message).await } @@ -1190,17 +1188,17 @@ impl RmqQueueManager { let mut cnt_rejected_overload = 0; 'a: loop { match self.remp_manager.poll_incoming(&self.shard).await { - (Some(rmq_message), _) => { + (Some(msg), _) => { if let Some((overload_message, status)) = cur_queue.is_queue_overloaded().await { - log::warn!(target: "remp", "Point 3. RMQ {}: {}, ignoring incoming message {}", self, overload_message, rmq_message); - cur_queue.send_response_to_fullnode(rmq_message, status); + log::warn!(target: "remp", "Point 3. RMQ {}: {}, ignoring incoming message {}", self, overload_message, msg); + cur_queue.send_response_to_fullnode(&msg.get_message_id(), Arc::new(msg.origin.clone()), status); cnt_rejected_overload+=1; } - else if let Err(e) = self.put_message_to_rmq(rmq_message.clone()).await { + else if let Err(e) = self.put_message_to_rmq(msg.clone()).await { log::warn!(target: "remp", "Point 3. Error sending RMQ {} message {:?}: {}; returning back to incoming queue", - self, rmq_message, e + self, msg, e ); - self.remp_manager.return_to_incoming(rmq_message, &self.shard).await; + self.remp_manager.return_to_incoming(msg, &self.shard).await; break 'a; } else { diff --git a/src/validator/remp_block_parser.rs b/src/validator/remp_block_parser.rs index bb91a202..6e81820f 100644 --- a/src/validator/remp_block_parser.rs +++ b/src/validator/remp_block_parser.rs @@ -89,7 +89,7 @@ impl BlockProcessor for RempMasterBlockIndexingProcessor { }; if let Err(e) = self.message_cache.add_external_message_status( - message_id, message_uid, None, RempMessageStatus::TonNode_RempAccepted(accepted), + message_id, message_uid, None, None, RempMessageStatus::TonNode_RempAccepted(accepted), |_o,n| n.clone(), self.masterchain_seqno ).await { log::warn!(target: "remp", "Update message {:x}, uid {:x} status failed: {}", message_id, message_uid, e); diff --git a/src/validator/remp_catchain.rs b/src/validator/remp_catchain.rs index c0ebfeb7..62ec6592 100644 --- a/src/validator/remp_catchain.rs +++ b/src/validator/remp_catchain.rs @@ -18,7 +18,7 @@ use std::ops::RangeInclusive; use crate::{ engine_traits::EngineOperations, validator::{ - catchain_overlay::CatchainOverlayManagerImpl, message_cache::RmqMessage, + catchain_overlay::CatchainOverlayManagerImpl, message_cache::RempMessageHeader, sessions_computing::GeneralSessionInfo, mutex_wrapper::MutexWrapper, remp_manager::RempManager, validator_utils::{ @@ -35,18 +35,18 @@ use catchain::{ PublicKey, PublicKeyHash }; use ton_api::{ - IntoBoxed, ton::ton_node::RempCatchainRecord + IntoBoxed, ton::ton_node::RempCatchainRecordV2 }; use ton_block::ValidatorDescr; use ton_types::{error, fail, KeyId, Result, UInt256}; const REMP_CATCHAIN_START_POLLING_INTERVAL: Duration = Duration::from_millis(50); -fn get_remp_catchain_record_info(r: &RempCatchainRecord) -> String { +fn get_remp_catchain_record_info(r: &RempCatchainRecordV2) -> String { match r { - RempCatchainRecord::TonNode_RempCatchainMessage(msg) => + RempCatchainRecordV2::TonNode_RempCatchainMessageHeaderV2(msg) => format!("msg_id: {:x}", msg.message_id), - RempCatchainRecord::TonNode_RempCatchainMessageDigest(msg) => + RempCatchainRecordV2::TonNode_RempCatchainMessageDigestV2(msg) => format!("digest, master_seqno={}, len={}", msg.masterchain_seqno, msg.messages.len()) } } @@ -54,11 +54,11 @@ fn get_remp_catchain_record_info(r: &RempCatchainRecord) -> String { pub struct RempCatchainInstanceImpl { pub catchain_ptr: CatchainPtr, - pending_messages_queue_receiver: crossbeam_channel::Receiver, - pub pending_messages_queue_sender: crossbeam_channel::Sender, + pending_messages_queue_receiver: crossbeam_channel::Receiver, + pub pending_messages_queue_sender: crossbeam_channel::Sender, - pub rmq_catchain_receiver: crossbeam_channel::Receiver, - rmq_catchain_sender: crossbeam_channel::Sender + pub rmq_catchain_receiver: crossbeam_channel::Receiver, + rmq_catchain_sender: crossbeam_channel::Sender } impl RempCatchainInstanceImpl { @@ -106,7 +106,7 @@ impl RempCatchainInstance { self.instance_impl.load().map(|inst| inst.catchain_ptr.clone()) } - pub fn pending_messages_queue_send(&self, msg: RempCatchainRecord) -> Result<()> { + pub fn pending_messages_queue_send(&self, msg: RempCatchainRecordV2) -> Result<()> { let instance = self.get_instance_impl()?; match instance.pending_messages_queue_sender.send(msg) { Ok(()) => Ok(()), @@ -120,7 +120,7 @@ impl RempCatchainInstance { Ok(instance.pending_messages_queue_sender.len()) } - pub fn pending_messages_queue_try_recv(&self) -> Result> { + pub fn pending_messages_queue_try_recv(&self) -> Result> { let instance = self.get_instance_impl()?; match instance.pending_messages_queue_receiver.try_recv() { Ok(x) => Ok(Some(x)), @@ -134,7 +134,7 @@ impl RempCatchainInstance { Ok(instance.rmq_catchain_receiver.len()) } - pub fn rmq_catchain_try_recv(&self) -> Result> { + pub fn rmq_catchain_try_recv(&self) -> Result> { let instance = self.get_instance_impl()?; match instance.rmq_catchain_receiver.try_recv() { Ok(x) => Ok(Some(x)), @@ -143,7 +143,7 @@ impl RempCatchainInstance { } } - pub fn rmq_catchain_send(&self, msg: RempCatchainRecord) -> Result<()> { + pub fn rmq_catchain_send(&self, msg: RempCatchainRecordV2) -> Result<()> { let instance = self.get_instance_impl()?; match instance.rmq_catchain_sender.send(msg) { Ok(()) => Ok(()), @@ -385,19 +385,19 @@ impl RempCatchain { for msgbx in pld.actions.0.iter() { match msgbx { ::ton_api::ton::validator_session::round::Message::ValidatorSession_Message_Commit(msg) => { - match RmqMessage::deserialize(&msg.signature) { - Ok(unpacked_message) => { + match RempMessageHeader::deserialize(&msg.signature) { + Ok(record) => { #[cfg(feature = "telemetry")] { total += 1; } log::trace!(target: "remp", "Point 4. Message received from RMQ {}: {:?}, decoded {:?}, put to rmq_catchain queue", - self, msg.signature.0, unpacked_message //catchain.received_messages.len() + self, msg.signature.0, record //catchain.received_messages.len() ); - if let Err(e) = self.instance.rmq_catchain_send(unpacked_message.clone()) { + if let Err(e) = self.instance.rmq_catchain_send(record.clone()) { log::error!( target: "remp", "Point 4. Cannot put message {:?} from RMQ {} to queue: {}", - unpacked_message, self, e + record, self, e ) } }, @@ -447,7 +447,7 @@ impl CatchainListener for RempCatchain { let msg_body = ::ton_api::ton::validator_session::round::validator_session::message::message::Commit { round: 0, candidate: Default::default(), - signature: RmqMessage::serialize(&msg).unwrap() + signature: RempMessageHeader::serialize(&msg).unwrap() }.into_boxed(); log::trace!(target: "remp", "Point 3. RMQ {} sending message: {:?}, decoded {:?}", self, msg_body, msg diff --git a/src/validator/remp_manager.rs b/src/validator/remp_manager.rs index 5bae2886..62240abd 100644 --- a/src/validator/remp_manager.rs +++ b/src/validator/remp_manager.rs @@ -29,7 +29,7 @@ use crate::{ config::RempConfig, engine_traits::{EngineOperations, RempCoreInterface, RempDuplicateStatus}, validator::{ - message_cache::{RmqMessage, MessageCache}, mutex_wrapper::MutexWrapper, + message_cache::{RmqMessage, RempMessageOrigin, RempMessageWithOrigin, MessageCache}, mutex_wrapper::MutexWrapper, remp_catchain::RempCatchainStore, validator_utils::{get_message_uid, get_shard_by_message} } @@ -49,24 +49,24 @@ pub struct RempInterfaceQueues { runtime: Arc, pub engine: Arc, pub incoming_sender: - crossbeam_channel::Sender>, + crossbeam_channel::Sender>, pub response_receiver: - crossbeam_channel::Receiver<(UInt256, Arc, RempMessageStatus)> + crossbeam_channel::Receiver<(UInt256, UInt256, Arc, RempMessageStatus)> } pub struct RempDelayer { max_incoming_broadcast_delay_millis: u32, random_seed: u64, - pub incoming_receiver: crossbeam_channel::Receiver>, - pub delayed_incoming_sender: crossbeam_channel::Sender>, - delay_heap: MutexWrapper<(BinaryHeap<(Reverse, UInt256)>, HashMap>)>, + pub incoming_receiver: crossbeam_channel::Receiver>, + pub delayed_incoming_sender: crossbeam_channel::Sender>, + delay_heap: MutexWrapper<(BinaryHeap<(Reverse, UInt256)>, HashMap>)>, } impl RempDelayer { pub fn new (random_seed: u64, options: &RempConfig, - incoming_receiver: crossbeam_channel::Receiver>, - delayed_incoming_sender: crossbeam_channel::Sender>) -> Self { + incoming_receiver: crossbeam_channel::Receiver>, + delayed_incoming_sender: crossbeam_channel::Sender>) -> Self { Self { max_incoming_broadcast_delay_millis: options.get_max_incoming_broadcast_delay_millis(), random_seed, @@ -76,21 +76,22 @@ impl RempDelayer { } } - async fn push_delayer(&self, activation_time: SystemTime, msg: Arc) { + async fn push_delayer(&self, activation_time: SystemTime, msg: Arc) { let tm: DateTime = activation_time.into(); self.delay_heap.execute_sync(|(h,m)| { - if !m.contains_key(&msg.message_id) { + let message_id = msg.get_message_id(); + if !m.contains_key(message_id) { log::trace!(target: "remp", "Delaying REMP message {} till {}", msg, tm.format("%T")); - h.push((Reverse(activation_time), msg.message_id.clone())); - m.insert(msg.message_id.clone(), msg); + h.push((Reverse(activation_time), message_id.clone())); + m.insert(message_id.clone(), msg); } else { - log::trace!(target: "remp", "REMP message {} is already waiting", msg); + log::trace!(target: "remp", "Delayer: REMP message {} is already waiting", msg); } }).await } - async fn pop_delayer(&self) -> Option> { + async fn pop_delayer(&self) -> Option> { let now = SystemTime::now(); let res = self.delay_heap.execute_sync(|(h,m)| { match h.peek() { @@ -107,7 +108,7 @@ impl RempDelayer { if let Some((tm,msg)) = res { let tm: DateTime = tm.0.into(); - log::trace!(target: "remp", "Sending further REMP message {}, delayed till {}", msg, tm.format("%T")); + log::trace!(target: "remp", "Resuming REMP message {}, delayed till {}", msg, tm.format("%T")); Some(msg) } else { @@ -123,7 +124,7 @@ impl RempDelayer { max (h,m) } - async fn poll_incoming_once(&self) -> Result>> { + async fn poll_incoming_once(&self) -> Result>> { if let Some(msg) = self.pop_delayer().await { return Ok(Some(msg)) } @@ -132,7 +133,7 @@ impl RempDelayer { match self.incoming_receiver.try_recv() { Ok(msg) => { if msg.has_no_source_key() && self.max_incoming_broadcast_delay_millis > 0 { - let delay = (msg.message_id.first_u64() + self.random_seed) % (self.max_incoming_broadcast_delay_millis as u64); + let delay = (msg.get_message_id().first_u64() + self.random_seed) % (self.max_incoming_broadcast_delay_millis as u64); self.push_delayer(SystemTime::now() + Duration::from_millis(delay), msg).await; } else { @@ -321,21 +322,21 @@ impl> RempQueueDispatcher { pub struct RempIncomingQueue { engine: Arc, - pub incoming_receiver: crossbeam_channel::Receiver> + pub incoming_receiver: crossbeam_channel::Receiver> } impl RempIncomingQueue { pub fn new( engine: Arc, - incoming_receiver: crossbeam_channel::Receiver> + incoming_receiver: crossbeam_channel::Receiver> ) -> Self { RempIncomingQueue { engine, incoming_receiver } } } #[async_trait::async_trait] -impl RempQueue for RempIncomingQueue { - fn receive_message(&self) -> Result>> { +impl RempQueue for RempIncomingQueue { + fn receive_message(&self) -> Result>> { match self.incoming_receiver.try_recv() { Ok(x) => Ok(Some(x)), Err(crossbeam_channel::TryRecvError::Empty) => @@ -345,8 +346,8 @@ impl RempQueue for RempIncomingQueue { } } - async fn compute_shard(&self, msg: Arc) -> Result { - get_shard_by_message(self.engine.clone(), msg.message.clone()).await + async fn compute_shard(&self, msg: Arc) -> Result { + get_shard_by_message(self.engine.clone(), msg.message.message.clone()).await } } @@ -439,9 +440,9 @@ pub struct RempManager { pub catchain_store: Arc, pub message_cache: Arc, incoming_delayer: RempDelayer, - incoming_dispatcher: RempQueueDispatcher, + incoming_dispatcher: RempQueueDispatcher, pub collator_receipt_dispatcher: RempQueueDispatcher, - pub response_sender: crossbeam_channel::Sender<(UInt256, Arc, RempMessageStatus)> + pub response_sender: crossbeam_channel::Sender<(UInt256 /* local_id */, UInt256 /* message_id */, Arc, RempMessageStatus)> } impl RempManager { @@ -503,7 +504,7 @@ impl RempManager { self.collator_receipt_dispatcher.remove_actual_shard(shard).await; } - pub async fn poll_incoming(&self, shard: &ShardIdent) -> (Option>, usize) { + pub async fn poll_incoming(&self, shard: &ShardIdent) -> (Option>, usize) { match self.incoming_delayer.poll_incoming().await { Ok(n) => { log::trace!(target: "remp", "Polling REMP incoming delayer queue: {} messages processed", n); @@ -515,12 +516,12 @@ impl RempManager { return self.incoming_dispatcher.poll(shard).await; } - pub async fn return_to_incoming(&self, message: Arc, shard: &ShardIdent) { + pub async fn return_to_incoming(&self, message: Arc, shard: &ShardIdent) { self.incoming_dispatcher.return_back(message, shard).await; } - pub fn queue_response_to_fullnode(&self, local_key_id: UInt256, rmq_message: Arc, status: RempMessageStatus) -> Result<()> { - self.response_sender.send((local_key_id, rmq_message, status))?; + pub fn queue_response_to_fullnode(&self, local_key_id: UInt256, message_id: UInt256, origin: Arc, status: RempMessageStatus) -> Result<()> { + self.response_sender.send((local_key_id, message_id, origin, status))?; Ok(()) } @@ -553,8 +554,11 @@ impl RempManager { #[allow(dead_code)] impl RempInterfaceQueues { - pub fn make_test_message(&self) -> Result { - RmqMessage::make_test_message(&SliceData::new_empty()) + pub fn make_test_message(&self) -> Result { + Ok(RempMessageWithOrigin { + message: RmqMessage::make_test_message(&SliceData::new_empty())?, + origin: RempMessageOrigin::create_empty()? + }) } /** @@ -568,8 +572,8 @@ impl RempInterfaceQueues { loop { match self.make_test_message() { Err(e) => log::error!(target: "remp", "Cannot make test REMP message: `{}`", e), - Ok(test_message) => { - if let Err(x) = self.incoming_sender.send(Arc::new(test_message)) { + Ok(msg) => { + if let Err(x) = self.incoming_sender.send(Arc::new(msg)) { log::error!(target: "remp", "Cannot send test REMP message to RMQ: {}", x ); @@ -586,11 +590,11 @@ impl RempInterfaceQueues { } pub async fn send_response_to_fullnode( - &self, local_key_id: UInt256, rmq_message: Arc, status: RempMessageStatus + &self, local_key_id: UInt256, message_id: UInt256, origin: Arc, status: RempMessageStatus ) { let receipt = ton_api::ton::ton_node::RempReceipt::TonNode_RempReceipt ( ton_api::ton::ton_node::rempreceipt::RempReceipt { - message_id: rmq_message.message_id.clone().into(), + message_id: message_id.clone().into(), status: status.clone(), timestamp: 0, source_id: local_key_id.into() @@ -599,15 +603,15 @@ impl RempInterfaceQueues { let (engine,runtime) = (self.engine.clone(), self.runtime.clone()); runtime.clone().spawn(async move { - if let Err(e) = engine.send_remp_receipt(rmq_message.source_key.clone(), receipt).await { + if let Err(e) = engine.send_remp_receipt(origin.source_key.clone(), receipt).await { log::error!(target: "remp", "Cannot send {} response message {:x} to {}: {}", - status, rmq_message.message_id, rmq_message.source_key, e + status, message_id, origin.source_key, e ) } else { log::trace!(target: "remp", "Sending {} response for message {:x} to {}", - status, rmq_message.message_id, rmq_message.source_idx + status, message_id, origin.source_idx ) } }); @@ -616,8 +620,8 @@ impl RempInterfaceQueues { pub async fn poll_responses_loop(&self) { loop { match self.response_receiver.try_recv() { - Ok((local_key_id, msg,status)) => - self.send_response_to_fullnode(local_key_id, msg, status).await, + Ok((local_key_id, hdr, origin, status)) => + self.send_response_to_fullnode(local_key_id, hdr, origin, status).await, Err(crossbeam_channel::TryRecvError::Empty) => tokio::time::sleep(Duration::from_millis(1)).await, Err(crossbeam_channel::TryRecvError::Disconnected) => return @@ -632,13 +636,16 @@ impl RempCoreInterface for RempInterfaceQueues { let arc_message = Arc::new(message.clone()); // build message - let remp_message = Arc::new(RmqMessage::new ( + let remp_message = RmqMessage::new ( arc_message, message_id.clone(), get_message_uid(&message), + )?; + + let remp_message_origin = RempMessageOrigin::new ( source, 0 - )?); + )?; if self.message_cache.get_message(&message_id)?.is_some() { log::trace!(target: "remp", @@ -648,7 +655,7 @@ impl RempCoreInterface for RempInterfaceQueues { } else { log::trace!(target: "remp", "Point 1. Adding incoming message {} to incoming queue", remp_message); - self.incoming_sender.send(remp_message)?; + self.incoming_sender.send(Arc::new(RempMessageWithOrigin { message: remp_message, origin: remp_message_origin }))?; #[cfg(feature = "telemetry")] self.engine.remp_core_telemetry().in_channel_from_fullnode(self.incoming_sender.len()); } From 52bea5a17ef8deee845ff8b84c3d66b2cbe899bc Mon Sep 17 00:00:00 2001 From: Dmitry Shtukenberg Date: Fri, 2 Feb 2024 11:46:06 +0200 Subject: [PATCH 02/40] Updated manifests, some midroad broadcast changes --- Cargo.toml | 40 ++++++++++++------------- catchain/Cargo.toml | 11 +++---- src/validator/message_cache.rs | 13 ++++++++ src/validator/reliable_message_queue.rs | 4 +-- src/validator/remp_catchain.rs | 8 +++-- storage/Cargo.toml | 12 ++++---- validator-session/Cargo.toml | 10 +++---- 7 files changed, 57 insertions(+), 41 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 9325471b..4b0fc47b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,21 +2,25 @@ build = 'common/build/build.rs' edition = '2021' name = 'ton_node' -version = '0.55.90' +version = '0.55.91' [workspace] members = [ 'storage' ] [dependencies] +adnl = { features = [ 'client', 'node', 'server' ], git = 'https://github.com/tonlabs/ever-adnl.git', tag = '0.9.22' } arc-swap = '0.3.11' async-recursion = '0.3.2' async-trait = '0.1.22' bitflags = '1.2.1' +catchain = { path = 'catchain' } chrono = '=0.4.19' clap = '2.33.1' colored = '1.9.3' crossbeam-channel = '0.4.2' +ctrlc = { features = [ 'termination' ], version = '3.4.0' } dashmap = '5.4.0' +dht = { git = 'https://github.com/tonlabs/ever-dht.git', tag = '0.6.87' } dirs = '2.0.2' enum-as-inner = '=0.5.1' env_logger = '0.7.1' @@ -25,42 +29,38 @@ futures = '0.3.1' futures-timer = '3.0.1' hex = '0.4' lazy_static = '1.4.0' +lockfree = { git = 'https://github.com/tonlabs/lockfree.git' } log = '0.4' log4rs = '1.2' log4rs-rolling-file = '0.2.0' metrics = '0.21.0' +metrics-exporter-prometheus = { optional = true, version = '0.12.1' } +metrics-exporter-statsd = { optional = true, version = '0.5.0' } num-bigint = '0.4' num_cpus = '1.13' openssl = '0.10.35' +overlay = { git = 'https://github.com/tonlabs/ever-overlay.git', tag = '0.7.19' } parking_lot = '0.12' rand = '0.7' regex = '^1.3.0' +rldp = { git = 'https://github.com/tonlabs/ever-rldp.git', tag = '0.8.15' } serde = '1.0.105' serde_derive = '1.0.105' serde_json = '1.0.64' spin = '0.7.1' -stream-cancel = '0.8.0' -string-builder = '^0.2.0' -tokio-util = '0.7' -adnl = { features = [ 'client', 'node', 'server' ], git = 'https://github.com/tonlabs/ever-adnl.git', tag = '0.9.20' } -catchain = { path = 'catchain' } -ctrlc = { features = [ 'termination' ], version = '3.4.0' } -dht = { git = 'https://github.com/tonlabs/ever-dht.git', tag = '0.6.85' } -lockfree = { git = 'https://github.com/tonlabs/lockfree.git' } -metrics-exporter-prometheus = { optional = true, version = '0.12.1' } -metrics-exporter-statsd = { optional = true, version = '0.5.0' } -overlay = { git = 'https://github.com/tonlabs/ever-overlay.git', tag = '0.7.17' } -rldp = { git = 'https://github.com/tonlabs/ever-rldp.git', tag = '0.8.13' } statsd = { optional = true, version = '0.15' } storage = { path = 'storage' } +stream-cancel = '0.8.0' +string-builder = '^0.2.0' tokio = { features = [ 'rt-multi-thread' ], version = '1.5' } -ton_abi = { git = 'https://github.com/tonlabs/ever-abi.git', optional = true, tag = '2.4.13' } -ton_api = { git = 'https://github.com/tonlabs/ever-tl.git', package = 'ton_api', tag = '0.3.57' } -ton_block = { git = 'https://github.com/tonlabs/ever-block.git', tag = '1.9.121' } -ton_block_json = { git = 'https://github.com/tonlabs/ever-block-json.git', tag = '0.7.209' } -ton_executor = { git = 'https://github.com/tonlabs/ever-executor.git', tag = '1.16.108' } -ton_types = { git = 'https://github.com/tonlabs/ever-types.git', tag = '2.0.31' } -ton_vm = { git = 'https://github.com/tonlabs/ever-vm.git', tag = '1.9.5' } +tokio-util = '0.7' +ton_abi = { git = 'https://github.com/tonlabs/ever-abi.git', optional = true, tag = '2.4.14' } +ton_api = { git = 'https://github.com/tonlabs/ever-tl.git', package = 'ton_api', tag = '0.3.59' } +ton_block = { git = 'https://github.com/tonlabs/ever-block.git', tag = '1.9.122' } +ton_block_json = { git = 'https://github.com/tonlabs/ever-block-json.git', tag = '0.7.211' } +ton_executor = { git = 'https://github.com/tonlabs/ever-executor.git', tag = '1.16.109' } +ton_types = { git = 'https://github.com/tonlabs/ever-types.git', tag = '2.0.32' } +ton_vm = { git = 'https://github.com/tonlabs/ever-vm.git', tag = '1.9.6' } validator_session = { path = 'validator-session' } [dev-dependencies] diff --git a/catchain/Cargo.toml b/catchain/Cargo.toml index 2429febd..762eafb5 100644 --- a/catchain/Cargo.toml +++ b/catchain/Cargo.toml @@ -5,6 +5,7 @@ name = 'catchain' version = '0.1.0' [dependencies] +adnl = { features = [ 'node' ], git = 'https://github.com/tonlabs/ever-adnl.git', tag = '0.9.22' } chrono = '0.4.10' crossbeam = '0.7.3' failure = '0.1' @@ -15,16 +16,16 @@ log = '0.4' metrics = '0.21.0' metrics-core = '0.5.2' metrics-util = '0.15.0' +overlay = { git = 'https://github.com/tonlabs/ever-overlay.git', tag = '0.7.19' } quanta = '0.11.1' rand = '0.8' regex = '1.3.1' -adnl = { features = [ 'node' ], git = 'https://github.com/tonlabs/ever-adnl.git', tag = '0.9.20' } -overlay = { git = 'https://github.com/tonlabs/ever-overlay.git', tag = '0.7.17' } -rldp = { git = 'https://github.com/tonlabs/ever-rldp.git', tag = '0.8.13' } +rldp = { git = 'https://github.com/tonlabs/ever-rldp.git', tag = '0.8.15' } storage = { path = '../storage' } tokio = { features = [ 'rt-multi-thread' ], version = '1.5' } -ton_api = { git = 'https://github.com/tonlabs/ever-tl.git', package = 'ton_api', tag = '0.3.57' } -ton_types = { git = 'https://github.com/tonlabs/ever-types.git', tag = '2.0.31' } +ton_api = { git = 'https://github.com/tonlabs/ever-tl.git', package = 'ton_api', tag = '0.3.59' } +ton_types = { git = 'https://github.com/tonlabs/ever-types.git', tag = '2.0.32' } +weak-self = '1.0.2' [dev-dependencies] chrono = '0.4.10' diff --git a/src/validator/message_cache.rs b/src/validator/message_cache.rs index c1e318dd..f060e864 100644 --- a/src/validator/message_cache.rs +++ b/src/validator/message_cache.rs @@ -81,6 +81,19 @@ impl RmqMessage { }.into_boxed() } + pub fn as_rmq_record(&self) -> ton_api::ton::ton_node::RmqRecord { + unimplemented!("TBI") + } + + pub fn deserialize(raw: &ton_api::ton::bytes) -> Result { + let rmq_record: ton_api::ton::ton_node::RmqRecord = catchain::utils::deserialize_tl_boxed_object(&raw)?; + Ok(rmq_record) + } + + pub fn from_rmq_record(record: &ton_api::ton::ton_node::RmqRecord) -> Result { + unimplemented!("TBI") + } + #[allow(dead_code)] pub fn make_test_message(body: &SliceData) -> Result { let address = UInt256::rand(); diff --git a/src/validator/reliable_message_queue.rs b/src/validator/reliable_message_queue.rs index c817daa5..4b84303d 100644 --- a/src/validator/reliable_message_queue.rs +++ b/src/validator/reliable_message_queue.rs @@ -535,9 +535,9 @@ impl MessageQueue { continue } Ok(None) => { - log::error!( + log::warn!( target: "remp", - "Point 5. RMQ {}: message {:x} found in pending_collation queue, but has no body/origin", + "Point 5. RMQ {}: message {:x} found in pending_collation queue, but has no body/origin yet; will not be collated", self, msgid ); continue diff --git a/src/validator/remp_catchain.rs b/src/validator/remp_catchain.rs index 62ec6592..d802fba6 100644 --- a/src/validator/remp_catchain.rs +++ b/src/validator/remp_catchain.rs @@ -18,7 +18,7 @@ use std::ops::RangeInclusive; use crate::{ engine_traits::EngineOperations, validator::{ - catchain_overlay::CatchainOverlayManagerImpl, message_cache::RempMessageHeader, + catchain_overlay::CatchainOverlayManagerImpl, message_cache::{RmqMessage, RempMessageHeader}, sessions_computing::GeneralSessionInfo, mutex_wrapper::MutexWrapper, remp_manager::RempManager, validator_utils::{ @@ -493,8 +493,10 @@ impl CatchainListener for RempCatchain { log::trace!(target: "remp", "MessageQueue {} started", self) } - fn process_broadcast(&self, _source_id: PublicKeyHash, _data: BlockPayloadPtr) { - log::trace!(target: "remp", "MessageQueue {} process broadcast", self) + fn process_broadcast(&self, source_id: PublicKeyHash, data: BlockPayloadPtr) { + log::trace!(target: "remp", "MessageQueue {} process broadcast from {}", self, source_id); + + let message = RmqMessage::deserialize(data.data()); } fn process_query(&self, source_id: PublicKeyHash, data: BlockPayloadPtr, _callback: ExternalQueryResponseCallback) { diff --git a/storage/Cargo.toml b/storage/Cargo.toml index 70fd1dbd..b0cfaa9a 100644 --- a/storage/Cargo.toml +++ b/storage/Cargo.toml @@ -4,6 +4,7 @@ name = 'storage' version = '0.5.0' [dependencies] +adnl = { git = 'https://github.com/tonlabs/ever-adnl.git', tag = '0.9.22' } ahash = '0.8' async-trait = '0.1.31' bytes = '1.1.0' @@ -12,25 +13,24 @@ fnv = '1.0.6' futures = '0.3.4' hex = '0.4' lazy_static = '1.4.0' +lockfree = { git = 'https://github.com/tonlabs/lockfree.git' } log = '0.4' log4rs = '1.2' lru = '0.11.0' metrics = '0.21.1' parking_lot = '0.12.1' quick_cache = '0.4.0' +rand = { features = [ 'small_rng' ], version = '0.8' } rocksdb = '0.21' serde = '1.0.114' serde_cbor = '0.11.1' serde_derive = '1.0.114' strum = '0.18.0' strum_macros = '0.18.0' -adnl = { git = 'https://github.com/tonlabs/ever-adnl.git', tag = '0.9.20' } -lockfree = { git = 'https://github.com/tonlabs/lockfree.git' } -rand = { features = [ 'small_rng' ], version = '0.8' } tokio = { features = [ 'fs', 'rt-multi-thread' ], version = '1.5' } -ton_api = { git = 'https://github.com/tonlabs/ever-tl.git', package = 'ton_api', tag = '0.3.57' } -ton_block = { git = 'https://github.com/tonlabs/ever-block.git', tag = '1.9.121' } -ton_types = { git = 'https://github.com/tonlabs/ever-types.git', tag = '2.0.31' } +ton_api = { git = 'https://github.com/tonlabs/ever-tl.git', package = 'ton_api', tag = '0.3.59' } +ton_block = { git = 'https://github.com/tonlabs/ever-block.git', tag = '1.9.122' } +ton_types = { git = 'https://github.com/tonlabs/ever-types.git', tag = '2.0.32' } [build-dependencies] cc = { features = [ 'parallel' ], version = '1.0.61' } diff --git a/validator-session/Cargo.toml b/validator-session/Cargo.toml index 57b4947a..de6787b5 100644 --- a/validator-session/Cargo.toml +++ b/validator-session/Cargo.toml @@ -6,6 +6,7 @@ version = '0.0.2' [dependencies] backtrace = '0.3' +catchain = { path = '../catchain' } crc = '3.0' crossbeam = '0.7' failure = '0.1' @@ -14,13 +15,12 @@ lazy_static = '1.4' log = '0.4' metrics = '0.21.0' metrics-core = '0.5' +overlay = { git = 'https://github.com/tonlabs/ever-overlay.git', tag = '0.7.19' } rand = '0.8' -catchain = { path = '../catchain' } -overlay = { git = 'https://github.com/tonlabs/ever-overlay.git', tag = '0.7.17' } storage = { path = '../storage' } -ton_api = { git = 'https://github.com/tonlabs/ever-tl.git', package = 'ton_api', tag = '0.3.57' } -ton_block = { git = 'https://github.com/tonlabs/ever-block.git', tag = '1.9.121' } -ton_types = { git = 'https://github.com/tonlabs/ever-types.git', tag = '2.0.31' } +ton_api = { git = 'https://github.com/tonlabs/ever-tl.git', package = 'ton_api', tag = '0.3.59' } +ton_block = { git = 'https://github.com/tonlabs/ever-block.git', tag = '1.9.122' } +ton_types = { git = 'https://github.com/tonlabs/ever-types.git', tag = '2.0.32' } [dev-dependencies] chrono = '0.4' From e2161675541211a8fc68fea21d64882c48d06a56 Mon Sep 17 00:00:00 2001 From: Dmitry Shtukenberg Date: Sat, 3 Feb 2024 00:16:59 +0200 Subject: [PATCH 03/40] Proper conversion of RmqMessage (temporary data structure for broadcasts; later to be replaced with something smaller) --- src/validator/message_cache.rs | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/validator/message_cache.rs b/src/validator/message_cache.rs index f060e864..a0e101c1 100644 --- a/src/validator/message_cache.rs +++ b/src/validator/message_cache.rs @@ -82,7 +82,13 @@ impl RmqMessage { } pub fn as_rmq_record(&self) -> ton_api::ton::ton_node::RmqRecord { - unimplemented!("TBI") + ton_api::ton::ton_node::rmqrecord::RmqMessage { + message: self.message.write_to_bytes().unwrap().into(), + message_id: self.message_id.clone().into(), + source_key_id: UInt256::default(), + source_idx: 0, + masterchain_seqno: 0 + }.into_boxed() } pub fn deserialize(raw: &ton_api::ton::bytes) -> Result { @@ -91,7 +97,17 @@ impl RmqMessage { } pub fn from_rmq_record(record: &ton_api::ton::ton_node::RmqRecord) -> Result { - unimplemented!("TBI") + match record { + ton_api::ton::ton_node::RmqRecord::TonNode_RmqMessage(rec) => { + let message_body = Arc::new(Message::construct_from_bytes(&rec.message)?); + Ok(Self { + message: message_body.clone(), + message_id: rec.message_id.clone().into(), + message_uid: get_message_uid(&message_body) + }) + }, + _ => fail!("message_cache::from_rmq_record: not an RmqMessage is given: {:?}", record) + } } #[allow(dead_code)] From beb4f2df97927767f8dac8232f8ca3b82c0efd81 Mon Sep 17 00:00:00 2001 From: Dmitry Shtukenberg Date: Thu, 8 Feb 2024 14:31:30 +0300 Subject: [PATCH 04/40] Fixed message bodies broadcasts --- src/validator/message_cache.rs | 31 +++++++++++++++++----- src/validator/reliable_message_queue.rs | 17 +++++++++---- src/validator/remp_block_parser.rs | 2 +- src/validator/remp_catchain.rs | 34 +++++++++++++++++++++---- 4 files changed, 66 insertions(+), 18 deletions(-) diff --git a/src/validator/message_cache.rs b/src/validator/message_cache.rs index a0e101c1..8db35328 100644 --- a/src/validator/message_cache.rs +++ b/src/validator/message_cache.rs @@ -295,6 +295,10 @@ impl RempMessageWithOrigin { pub fn as_remp_catchain_record(&self, master_cc: u32) -> ton_api::ton::ton_node::RempCatchainRecordV2 { return self.message.as_remp_catchain_record(master_cc, &self.origin) } + + pub fn as_rmq_record(&self) -> ton_api::ton::ton_node::RmqRecord { + return self.message.as_rmq_record() + } } impl Display for RempMessageWithOrigin { @@ -656,7 +660,7 @@ impl MessageCache { } } - fn insert_message(&self, session: Arc, message: Arc, message_header: Arc, message_origin: Arc, status: &RempMessageStatus) -> Result<()> { + fn insert_message(&self, session: Arc, message: Arc, message_header: Arc, message_origin: Option>, status: &RempMessageStatus) -> Result<()> { if message.message_id != message_header.message_id { fail!("Inconsistent message: message {} and message_header {} have different message_id", message, message_header) } @@ -668,7 +672,7 @@ impl MessageCache { } session.message_status.insert(message_id.clone(), status.clone()); - session.insert_message(message, message_header, Some(message_origin))?; + session.insert_message(message, message_header, message_origin)?; Ok(()) } @@ -688,7 +692,7 @@ impl MessageCache { /// If we do not know anything -- TODO: if all reject, then 'Rejected'. Otherwise 'New' /// Actual -- get it as granted ("imprinting") /// Returns old status and new (added) status - pub async fn add_external_message_status(&self, + pub fn add_external_message_status(&self, message_id: &UInt256, message_uid: &UInt256, message: Option>, message_origin: Option>, status_if_new: RempMessageStatus, status_updater: F, master_cc: u32 @@ -708,10 +712,11 @@ impl MessageCache { message_uid ); - match (message, &message_origin) { - (None,_) => self.insert_message_header( session, header, message_origin, &status_if_new)?, - (Some(message), Some(origin)) => self.insert_message(session, message, header, origin.clone(), &status_if_new)?, - (Some(message), None) => fail!("Incorrect options for add_external_message_status: message header {}, body {}, but no message origin", header, message) + match message { + None => + self.insert_message_header( session, header, message_origin, &status_if_new)?, + Some(message) => + self.insert_message(session, message, header, message_origin.clone(), &status_if_new)? }; Ok((None, status_if_new)) }, @@ -726,6 +731,18 @@ impl MessageCache { } } + pub fn update_message_body(&self, data: Arc) -> Result<()> { + self.add_external_message_status( + &data.message_id, + &data.message_uid, + Some(data.clone()), + None, + RempMessageStatus::TonNode_RempNew, |old,_new| old.clone(), + self.master_cc_seqno_curr.load(Ordering::Relaxed) + )?; + Ok(()) + } + /// Checks whether message msg_id is accepted by collator; /// if true, changes its status to ignored pub fn change_accepted_by_collator_to_ignored(&self, msg_id: &UInt256) -> Result { diff --git a/src/validator/reliable_message_queue.rs b/src/validator/reliable_message_queue.rs index 4b84303d..bfa42457 100644 --- a/src/validator/reliable_message_queue.rs +++ b/src/validator/reliable_message_queue.rs @@ -257,8 +257,11 @@ impl MessageQueue { } let origin_with_idx = Arc::new(msg.origin.new_with_updated_source_idx(self.catchain_info.local_idx as u32)); - log::trace!(target: "remp", "Point 3. Pushing to RMQ {}; message {}, {}", self, msg, origin_with_idx); - self.catchain_instance.pending_messages_queue_send(msg.as_remp_catchain_record(self.catchain_info.get_master_cc_seqno()))?; + log::trace!(target: "remp", "Point 3. Pushing to RMQ {}; message {}, {} + broadcast", self, msg, origin_with_idx); + self.catchain_instance.pending_messages_broadcast_send( + msg.as_remp_catchain_record(self.catchain_info.get_master_cc_seqno()), + msg.as_rmq_record() + )?; #[cfg(feature = "telemetry")] self.engine.remp_core_telemetry().in_channel_to_catchain( @@ -415,7 +418,7 @@ impl MessageQueue { old_status.clone() }, message_master_seqno - ).await; + ); match added { Err(e) => { @@ -472,7 +475,7 @@ impl MessageQueue { // validator. |old,_new| { old.clone() }, reject_digest.masterchain_seqno as u32 - ).await?; + )?; } } Ok(()) @@ -1024,7 +1027,11 @@ impl RmqQueueManager { } for new in next_queues.iter() { - if let Err(x) = new.catchain_instance.pending_messages_queue_send(message.as_remp_catchain_record(message_cc, &origin)) { + if let Err(x) = new.catchain_instance.pending_messages_broadcast_send( + message.as_remp_catchain_record(message_cc, &origin), + message.as_rmq_record() + ) + { log::error!(target: "remp", "Point 5a. RMQ {}: message {:x} cannot be put to new queue {}: `{}`", self, msgid, new, x diff --git a/src/validator/remp_block_parser.rs b/src/validator/remp_block_parser.rs index 6e81820f..c2f26725 100644 --- a/src/validator/remp_block_parser.rs +++ b/src/validator/remp_block_parser.rs @@ -91,7 +91,7 @@ impl BlockProcessor for RempMasterBlockIndexingProcessor { if let Err(e) = self.message_cache.add_external_message_status( message_id, message_uid, None, None, RempMessageStatus::TonNode_RempAccepted(accepted), |_o,n| n.clone(), self.masterchain_seqno - ).await { + ) { log::warn!(target: "remp", "Update message {:x}, uid {:x} status failed: {}", message_id, message_uid, e); } } diff --git a/src/validator/remp_catchain.rs b/src/validator/remp_catchain.rs index d802fba6..63a610c0 100644 --- a/src/validator/remp_catchain.rs +++ b/src/validator/remp_catchain.rs @@ -37,6 +37,7 @@ use catchain::{ use ton_api::{ IntoBoxed, ton::ton_node::RempCatchainRecordV2 }; +use ton_api::ton::ton_node::{rmqrecord, RmqRecord}; use ton_block::ValidatorDescr; use ton_types::{error, fail, KeyId, Result, UInt256}; @@ -58,7 +59,10 @@ pub struct RempCatchainInstanceImpl { pub pending_messages_queue_sender: crossbeam_channel::Sender, pub rmq_catchain_receiver: crossbeam_channel::Receiver, - rmq_catchain_sender: crossbeam_channel::Sender + rmq_catchain_sender: crossbeam_channel::Sender, + + pub remp_messages_receiver: crossbeam_channel::Receiver, + pending_messages_broadcast_sender: crossbeam_channel::Sender } impl RempCatchainInstanceImpl { @@ -67,10 +71,12 @@ impl RempCatchainInstanceImpl { crossbeam_channel::unbounded(); let (rmq_catchain_sender, rmq_catchain_receiver) = crossbeam_channel::unbounded(); + let (remp_messages_sender, remp_messages_receiver) = crossbeam_channel::unbounded(); Self { catchain_ptr, pending_messages_queue_sender, pending_messages_queue_receiver, - rmq_catchain_sender, rmq_catchain_receiver + rmq_catchain_sender, rmq_catchain_receiver, + pending_messages_broadcast_sender: remp_messages_sender, remp_messages_receiver } } } @@ -114,6 +120,15 @@ impl RempCatchainInstance { } } + pub fn pending_messages_broadcast_send(&self, msg: RempCatchainRecordV2, msg_body: RmqRecord) -> Result<()> { + let instance = self.get_instance_impl()?; + match (instance.pending_messages_queue_sender.send(msg), instance.pending_messages_broadcast_sender.send(msg_body)) { + (Ok(()), Ok(())) => Ok(()), + (Err(e), _) => fail!("pending_messages_queue_sender: send error {}", e), + (_, Err(e)) => fail!("pending_messages_broadcast_sender: send error {}", e) + } + } + #[cfg(feature = "telemetry")] pub fn pending_messages_queue_len(&self) -> Result { let instance = self.get_instance_impl()?; @@ -420,6 +435,14 @@ impl RempCatchain { Err(e) => log::error!(target: "remp", "Cannot deserialize RMQ {} message: {}", self, e) } } + + fn unpack_broadcast(&self, payload: &BlockPayloadPtr) -> Result<()> { + let message = RmqMessage::deserialize(payload.data())?; + let rmqrecord = RmqMessage::from_rmq_record(&message)?; + self.remp_manager.message_cache.update_message_body(Arc::new(rmqrecord))?; + + Ok(()) + } } impl fmt::Display for RempCatchain { @@ -494,9 +517,10 @@ impl CatchainListener for RempCatchain { } fn process_broadcast(&self, source_id: PublicKeyHash, data: BlockPayloadPtr) { - log::trace!(target: "remp", "MessageQueue {} process broadcast from {}", self, source_id); - - let message = RmqMessage::deserialize(data.data()); + log::trace!(target: "remp", "MessageQueue {} process broadcast from {}", self, source_id); + if let Err(e) = self.unpack_broadcast(&data) { + log::error!(target: "remp", "Error processing broadcast from {}, message body will be ignored: `{}`", source_id, e); + } } fn process_query(&self, source_id: PublicKeyHash, data: BlockPayloadPtr, _callback: ExternalQueryResponseCallback) { From f4a72405c8f0da93a7c2c4bc78e780a872d15759 Mon Sep 17 00:00:00 2001 From: Dmitry Shtukenberg Date: Mon, 12 Feb 2024 16:08:27 +0300 Subject: [PATCH 05/40] Polling and sending broadcasts --- src/validator/reliable_message_queue.rs | 7 +++-- src/validator/remp_catchain.rs | 37 +++++++++++++++++++++++-- 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/src/validator/reliable_message_queue.rs b/src/validator/reliable_message_queue.rs index bfa42457..c4e1c8e0 100644 --- a/src/validator/reliable_message_queue.rs +++ b/src/validator/reliable_message_queue.rs @@ -17,7 +17,7 @@ use std::{ fmt, fmt::Formatter, ops::RangeInclusive, sync::Arc, - time::{Duration, SystemTime} + time::Duration }; use dashmap::DashMap; @@ -52,6 +52,7 @@ use crate::engine_traits::RempDuplicateStatus; enum MessageQueueStatus { Created, Starting, Active, Stopping } const RMQ_STOP_POLLING_INTERVAL: Duration = Duration::from_millis(50); const RMQ_REQUEST_NEW_BLOCK_INTERVAL: Duration = Duration::from_millis(50); +const RMQ_MAXIMAL_BROADCASTS_IN_PACK: u32 = 1000; struct MessageQueueImpl { status: MessageQueueStatus, @@ -267,9 +268,9 @@ impl MessageQueue { self.engine.remp_core_telemetry().in_channel_to_catchain( &self.catchain_info.general_session_info.shard, self.catchain_instance.pending_messages_queue_len()?); - if let Some(session) = &self.remp_manager.catchain_store.get_catchain_session(&self.catchain_info.queue_id).await { + if self.catchain_instance.is_session_active() { log::trace!(target: "remp", "Point 3. Activating RMQ {} processing", self); - session.request_new_block(SystemTime::now() + RMQ_REQUEST_NEW_BLOCK_INTERVAL); + self.catchain_instance.activate_exchange(RMQ_REQUEST_NEW_BLOCK_INTERVAL, RMQ_MAXIMAL_BROADCASTS_IN_PACK)?; // Temporary status "New" --- message is not registered yet self.send_response_to_fullnode(msg.get_message_id(), origin_with_idx, RempMessageStatus::TonNode_RempNew); diff --git a/src/validator/remp_catchain.rs b/src/validator/remp_catchain.rs index 63a610c0..be1ae717 100644 --- a/src/validator/remp_catchain.rs +++ b/src/validator/remp_catchain.rs @@ -37,7 +37,7 @@ use catchain::{ use ton_api::{ IntoBoxed, ton::ton_node::RempCatchainRecordV2 }; -use ton_api::ton::ton_node::{rmqrecord, RmqRecord}; +use ton_api::ton::ton_node::RmqRecord; use ton_block::ValidatorDescr; use ton_types::{error, fail, KeyId, Result, UInt256}; @@ -61,7 +61,7 @@ pub struct RempCatchainInstanceImpl { pub rmq_catchain_receiver: crossbeam_channel::Receiver, rmq_catchain_sender: crossbeam_channel::Sender, - pub remp_messages_receiver: crossbeam_channel::Receiver, + pub pending_messages_broadcast_receiver: crossbeam_channel::Receiver, pending_messages_broadcast_sender: crossbeam_channel::Sender } @@ -76,7 +76,8 @@ impl RempCatchainInstanceImpl { catchain_ptr, pending_messages_queue_sender, pending_messages_queue_receiver, rmq_catchain_sender, rmq_catchain_receiver, - pending_messages_broadcast_sender: remp_messages_sender, remp_messages_receiver + pending_messages_broadcast_sender: remp_messages_sender, + pending_messages_broadcast_receiver: remp_messages_receiver } } } @@ -144,6 +145,15 @@ impl RempCatchainInstance { } } + pub fn pending_messages_broadcast_try_recv(&self) -> Result> { + let instance = self.get_instance_impl()?; + match instance.pending_messages_broadcast_receiver.try_recv() { + Ok(x) => Ok(Some(x)), + Err(crossbeam_channel::TryRecvError::Empty) => Ok(None), + Err(crossbeam_channel::TryRecvError::Disconnected) => fail!("channel disconnected") + } + } + pub fn rmq_catchain_receiver_len(&self) -> Result { let instance = self.get_instance_impl()?; Ok(instance.rmq_catchain_receiver.len()) @@ -169,6 +179,27 @@ impl RempCatchainInstance { pub fn get_id(&self) -> u128 { self.id.duration_since(UNIX_EPOCH).unwrap().as_micros() } + + fn poll_pending_broadcasts(&self, max_broadcasts: u32) -> Result<()> { + for _msg_count in 0..max_broadcasts { + match self.pending_messages_broadcast_try_recv()? { + None => break, + Some(msg) => { + let block_payload = catchain::CatchainFactory::create_block_payload( + serialize_tl_boxed_object!(&msg), + ); + self.get_instance_impl()?.catchain_ptr.send_broadcast(block_payload) + } + } + } + Ok(()) + } + + pub fn activate_exchange(&self, delay: Duration, max_broadcasts: u32) -> Result<()> { + let session = &self.get_instance_impl()?.catchain_ptr; + session.request_new_block(SystemTime::now() + delay); + self.poll_pending_broadcasts(max_broadcasts) + } } impl fmt::Display for RempCatchainInstance { From 394efb4ecaefb1d4624d3d106855789a6ffa1ed7 Mon Sep 17 00:00:00 2001 From: Dmitry Shtukenberg Date: Mon, 12 Feb 2024 18:26:38 +0300 Subject: [PATCH 06/40] Partial updating of message body/origin --- src/validator/message_cache.rs | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/validator/message_cache.rs b/src/validator/message_cache.rs index 8db35328..92965fc7 100644 --- a/src/validator/message_cache.rs +++ b/src/validator/message_cache.rs @@ -389,6 +389,11 @@ impl MessageCacheSession { fail!("Different message body: new body '{}', current cached body '{}'", m1, m2.val()); } } + else { + if let Some(m2) = self.messages.insert(message_id.clone(), m1.clone()) { + fail!("Parallel updating of message body: new body '{}', another body '{}'", m1, m2.val()) + } + } } if let Some(o1) = &origin { @@ -397,6 +402,11 @@ impl MessageCacheSession { fail!("Different message origin: new origin '{}', current cached origin '{}'", o1, o2.value()); } } + else { + if let Some(o2) = self.message_origins.insert(message_id.clone(), o1.clone()) { + fail!("Parallel updating of message origin: new origin '{}', another origin '{}'", o1, o2) + } + } } Ok(()) @@ -646,10 +656,9 @@ impl MessageCache { ); match (msg, origin, status) { - (None, _, Some(_)) => Ok(None), // Bare message info (retrieved from finalized block) + (None, _, Some(_)) | (_, None, Some(_)) => Ok(None), // Bare message info (retrieved from finalized block/not received from broadcast) (Some(m), Some(o), Some (h)) => Ok(Some((m.clone(), o.clone(), h.clone(), session.master_cc))), // Full message info (m, o, None) => fail!("Message {:x} has no status, body = {:?}, origin = {:?}", message_id, m, o), - (Some(m), None, Some(h)) => fail!("Message {:x} has no orgin, but has body {} and status {}", message_id, m, h) } } @@ -740,6 +749,10 @@ impl MessageCache { RempMessageStatus::TonNode_RempNew, |old,_new| old.clone(), self.master_cc_seqno_curr.load(Ordering::Relaxed) )?; + match self.get_message_with_origin_status_cc(&data.message_id)? { + Some((msg, msg_origin, x, seqno)) => log::info!(target: "remp", "Message info added: {}, {}, {}, {}", msg, msg_origin, x, seqno), + None => log::info!(target: "remp", "Message {} was not found in cache", &data.message_id) + } Ok(()) } From 1c1402a027a562a9d78b14ac4f6e4a8f5217a869 Mon Sep 17 00:00:00 2001 From: Dmitry Shtukenberg Date: Mon, 12 Feb 2024 15:30:49 +0000 Subject: [PATCH 07/40] Updated standalone node test --- tests/setup_test_paths.sh | 5 +++++ tests/test_run_net/test_run_net.sh | 8 +++++--- 2 files changed, 10 insertions(+), 3 deletions(-) create mode 100755 tests/setup_test_paths.sh mode change 100644 => 100755 tests/test_run_net/test_run_net.sh diff --git a/tests/setup_test_paths.sh b/tests/setup_test_paths.sh new file mode 100755 index 00000000..c96c767e --- /dev/null +++ b/tests/setup_test_paths.sh @@ -0,0 +1,5 @@ +TESTS_DIR=$(realpath $(dirname "${BASH_SOURCE:-$0}")) +BASE_DIR=${TESTS_DIR}/../../ +TOOLS_DIR=${BASE_DIR}/ever-node-tools/target/release/ +CONFIG_PATH=${BASE_DIR}/ever-node/target/release/ +CONSOLE_CONFIG_0=${CONFIG_PATH}configs_0/console.json \ No newline at end of file diff --git a/tests/test_run_net/test_run_net.sh b/tests/test_run_net/test_run_net.sh old mode 100644 new mode 100755 index 79e22bc4..95a43fe3 --- a/tests/test_run_net/test_run_net.sh +++ b/tests/test_run_net/test_run_net.sh @@ -44,13 +44,15 @@ then fi cd ../../../ -if ! [ -d "ever-node-tools-private" ] +if ! [ -d "ever-node-tools" ] then git clone "https://github.com/tonlabs/ever-node-tools" - cd ever-node-tools-private + cd ever-node-tools git checkout "$CURRENT_BRANCH" || echo "Use default branch" + git submodule init + git submodule update else - cd ever-node-tools-private + cd ever-node-tools fi TOOLS_ROOT=$(pwd) From b92e454910b59ead458a774ec0876514767881e8 Mon Sep 17 00:00:00 2001 From: Dmitry Shtukenberg Date: Wed, 14 Feb 2024 11:57:28 +0300 Subject: [PATCH 08/40] Cosmetic improvement, method renaming --- src/validator/message_cache.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/validator/message_cache.rs b/src/validator/message_cache.rs index 92965fc7..920205e5 100644 --- a/src/validator/message_cache.rs +++ b/src/validator/message_cache.rs @@ -382,7 +382,7 @@ impl MessageCacheSession { } } - fn ensure_same_message(&self, message_id: &UInt256, message: Option>, origin: Option>) -> Result<()> { + fn update_missing_fields(&self, message_id: &UInt256, message: Option>, origin: Option>) -> Result<()> { if let Some(m1) = &message { if let Some(m2) = self.messages.get(message_id) { if m1 != m2.val() { @@ -730,7 +730,7 @@ impl MessageCache { Ok((None, status_if_new)) }, Some(session) => { - if let Err(e) = session.ensure_same_message(&message_id, message, message_origin) { + if let Err(e) = session.update_missing_fields(&message_id, message, message_origin) { log::error!(target: "remp", "Different cache contents for external message {}: {}", message_id, e); } let (old_status, final_status) = From 1d23374e36a5acf40595c46197ba238b6e795c2e Mon Sep 17 00:00:00 2001 From: Dmitry Shtukenberg Date: Wed, 14 Feb 2024 15:11:24 +0000 Subject: [PATCH 09/40] Added remp tests --- tests/setup_test_paths.sh | 2 +- tests/test_high_load/.gitignore | 2 + tests/test_high_load/Brazil.abi.json | 126 ++++++ tests/test_high_load/Brazil.sol | 173 +++++++ tests/test_high_load/Brazil.tvc | Bin 0 -> 2647 bytes .../SafeMultisigWallet.abi.json | 124 +++++ .../SetcodeMultisigWallet.abi.json | 160 +++++++ tests/test_high_load/addrBrazil | 3 + tests/test_high_load/deploy.keys.2.json | 4 + tests/test_high_load/deploy.keys.json | 4 + tests/test_high_load/msig.keys.json | 4 + tests/test_high_load/remove_junk.sh | 1 + tests/test_high_load/seedphrase | 3 + tests/test_high_load/test_console.sh | 2 + tests/test_high_load/test_high_load.sh | 169 +++++++ tests/test_high_load/test_one_message.sh | 66 +++ tests/test_remp/.gitignore | 6 + tests/test_remp/no_replay.sol | 27 ++ tests/test_remp/test_remp.sh | 422 ++++++++++++++++++ tests/test_run_net/log_cfg.yml | 2 +- 20 files changed, 1298 insertions(+), 2 deletions(-) create mode 100644 tests/test_high_load/.gitignore create mode 100644 tests/test_high_load/Brazil.abi.json create mode 100644 tests/test_high_load/Brazil.sol create mode 100644 tests/test_high_load/Brazil.tvc create mode 100644 tests/test_high_load/SafeMultisigWallet.abi.json create mode 100644 tests/test_high_load/SetcodeMultisigWallet.abi.json create mode 100644 tests/test_high_load/addrBrazil create mode 100644 tests/test_high_load/deploy.keys.2.json create mode 100644 tests/test_high_load/deploy.keys.json create mode 100644 tests/test_high_load/msig.keys.json create mode 100755 tests/test_high_load/remove_junk.sh create mode 100644 tests/test_high_load/seedphrase create mode 100755 tests/test_high_load/test_console.sh create mode 100755 tests/test_high_load/test_high_load.sh create mode 100755 tests/test_high_load/test_one_message.sh create mode 100644 tests/test_remp/.gitignore create mode 100644 tests/test_remp/no_replay.sol create mode 100755 tests/test_remp/test_remp.sh diff --git a/tests/setup_test_paths.sh b/tests/setup_test_paths.sh index c96c767e..f2fd4115 100755 --- a/tests/setup_test_paths.sh +++ b/tests/setup_test_paths.sh @@ -2,4 +2,4 @@ TESTS_DIR=$(realpath $(dirname "${BASH_SOURCE:-$0}")) BASE_DIR=${TESTS_DIR}/../../ TOOLS_DIR=${BASE_DIR}/ever-node-tools/target/release/ CONFIG_PATH=${BASE_DIR}/ever-node/target/release/ -CONSOLE_CONFIG_0=${CONFIG_PATH}configs_0/console.json \ No newline at end of file +CONSOLE_CONFIG_0=${CONFIG_PATH}configs_0/console.json diff --git a/tests/test_high_load/.gitignore b/tests/test_high_load/.gitignore new file mode 100644 index 00000000..01380c91 --- /dev/null +++ b/tests/test_high_load/.gitignore @@ -0,0 +1,2 @@ +*.boc +logs/* \ No newline at end of file diff --git a/tests/test_high_load/Brazil.abi.json b/tests/test_high_load/Brazil.abi.json new file mode 100644 index 00000000..8ea7e666 --- /dev/null +++ b/tests/test_high_load/Brazil.abi.json @@ -0,0 +1,126 @@ +{ + "ABI version": 2, + "header": ["time"], + "functions": [ + { + "name": "constructor", + "inputs": [ + ], + "outputs": [ + ] + }, + { + "name": "setC", + "inputs": [ + {"name":"counter","type":"uint128"} + ], + "outputs": [ + ] + }, + { + "name": "deployWorker", + "inputs": [ + ], + "outputs": [ + ] + }, + { + "name": "stopWorker", + "inputs": [ + ], + "outputs": [ + ] + }, + { + "name": "NeedMoney", + "inputs": [ + {"name":"id","type":"uint128"} + ], + "outputs": [ + ] + }, + { + "name": "build", + "inputs": [ + ], + "outputs": [ + ] + }, + { + "name": "init", + "inputs": [ + ], + "outputs": [ + ] + }, + { + "name": "setNumber", + "inputs": [ + {"name":"number","type":"uint128"} + ], + "outputs": [ + ] + }, + { + "name": "setCode2", + "inputs": [ + {"name":"c","type":"cell"} + ], + "outputs": [ + ] + }, + { + "name": "setData2", + "inputs": [ + {"name":"c","type":"cell"} + ], + "outputs": [ + ] + }, + { + "name": "setContract2", + "inputs": [ + {"name":"c","type":"cell"} + ], + "outputs": [ + ] + }, + { + "name": "setKey", + "inputs": [ + {"name":"key","type":"uint256"} + ], + "outputs": [ + ] + }, + { + "name": "setdGrant", + "inputs": [ + {"name":"dgrant","type":"uint128"} + ], + "outputs": [ + ] + }, + { + "name": "upgrade", + "inputs": [ + {"name":"newcode","type":"cell"} + ], + "outputs": [ + ] + }, + { + "name": "grant", + "inputs": [ + {"name":"addr","type":"address"}, + {"name":"value","type":"uint128"} + ], + "outputs": [ + ] + } + ], + "data": [ + ], + "events": [ + ] +} diff --git a/tests/test_high_load/Brazil.sol b/tests/test_high_load/Brazil.sol new file mode 100644 index 00000000..03da1b5b --- /dev/null +++ b/tests/test_high_load/Brazil.sol @@ -0,0 +1,173 @@ +/* */ +pragma solidity >= 0.6.0; + +import "Worker.sol"; + +contract Brazil is IBrazil { + + uint128 _counter = 0; + uint128 _deployed = 0; + uint128 _number = 10; + TvmCell _contract2; + TvmCell _code2; + TvmCell _data2; + + uint _key; + uint128 _dgrant = 2e9; + uint128 _giveMoney = 3e9; + + address _one; + address _two; + address _three; + + address[] _DeployedArray; + address[] _WorkerHelps; + + modifier alwaysAccept { + tvm.accept(); + _; + } + + constructor() public alwaysAccept { + } + + + function setC(uint128 counter) public alwaysAccept { + if (counter < 0) { + counter = 0; + } + _counter = counter; + while (_counter > _deployed){ + deployWorker(); + } + while (_deployed > _counter){ + stopWorker(); + } + } + + function _isSameShard(address a1, address a2) private pure returns (bool) { + uint sh1 = a1.value >> 252; + uint sh2 = a2.value >> 252; + return (sh1 == sh2); + } + + function deployWorker() public alwaysAccept { + if (_DeployedArray.length > _counter){ + while (_deployed < _counter){ + Worker(_DeployedArray[_deployed]).start(); + Worker(_WorkerHelps[_deployed]).start(); + _deployed++; + } + return; + } + if (_DeployedArray.length > _deployed){ + while (_deployed < _DeployedArray.length){ + Worker(_DeployedArray[_deployed]).start(); + Worker(_WorkerHelps[_deployed]).start(); + _deployed++; + } + return; + } + address a1; + address a2; + TvmCell s1; + TvmCell s2; + + _key = _key + rnd.next(3); + s1 = tvm.insertPubkey(_contract2, _key); + a1 = address(tvm.hash(s1)); + + a2 = a1; + while (_isSameShard(a2, a1)) { + _key++; + s2 = tvm.insertPubkey(_contract2, _key); + a2 = address(tvm.hash(s2)); + } + _DeployedArray.push(a1); + _WorkerHelps.push(a2); + _deployed++; + new Worker {stateInit:s1, value:_dgrant} (a2, _deployed); + new Worker {stateInit:s2, value:_dgrant} (a1, _deployed); + } + + function stopWorker() public alwaysAccept { + while (_deployed > _counter){ + Worker(_DeployedArray[_deployed - 1]).stop(); + Worker(_WorkerHelps[_deployed- 1]).stop(); + _deployed--; + } + } + + function NeedMoney(uint128 id) external override alwaysAccept { +// if (_DeployedArray[id - 1].balance() > 1e9){ +// return; +// } + if (id <= _deployed){ + _DeployedArray[id - 1].transfer(_giveMoney); + _WorkerHelps[id - 1].transfer(_giveMoney); + } + } + + + function build() public alwaysAccept { + _contract2 = tvm.buildStateInit(_code2, _data2); + } + + function init() public alwaysAccept { + _counter = 0; + _deployed = 0; + _key = rnd.next(3); + _contract2 = tvm.buildStateInit(_code2, _data2); + } + + + /* Setters */ + + function setNumber(uint128 number) public alwaysAccept { + _number = number; + } + + + function setCode2(TvmCell c) public alwaysAccept { + _code2 = c; + } + + function setData2(TvmCell c) public alwaysAccept { + _data2 = c; + } + + function setContract2(TvmCell c) public alwaysAccept { + _contract2 = c; + } + + function setKey(uint key) public alwaysAccept { + _key = key; + } + + function setdGrant(uint128 dgrant) public alwaysAccept { + _dgrant = dgrant; + } + + function setgMoney(uint128 money) public alwaysAccept { + _giveMoney = money; + } + + /* fallback/receive */ + receive() external { + + } + + function upgrade(TvmCell newcode) public { +// require(msg.pubkey() == tvm.pubkey(), 101); + tvm.accept(); + tvm.commit(); + tvm.setcode(newcode); + tvm.setCurrentCode(newcode); +// onCodeUpgrade(); + } + + function grant(address addr, uint128 value) external { + tvm.accept(); + addr.transfer(value, false, 3); + } +} diff --git a/tests/test_high_load/Brazil.tvc b/tests/test_high_load/Brazil.tvc new file mode 100644 index 0000000000000000000000000000000000000000..95f1b080ea1b32dbf1efada4cd7599f839c8945e GIT binary patch literal 2647 zcmc&$3rtg27(S;hP^e;og2?l>P&ai>pheuoQe+|zxA_>F}-R~EE!-9W?gby6u~BW;H09nt~1Mph%eHUa%I4f^16ys*#uAUa4NqzZhj z3YHGoaC;mbZDi;mS)e{p6(|d!0~%?6)jYjF?XLsh$oS1s49$OJBx$enMnja$hZd*= zdI9a5Y0~>PnCg#7q6QwFZ@xAGqU>Gf3zT|qy*KS6^HKTe1#^!{I+dMe16JG~Nun6* zCusx=60H1vXU+%H!UIV^ zvI@6@M_j9~ca=T`lYZ&{-0u|Lu5g3}Gv>^OE;2xPLRYQkwm(?GRn;_fJ~lEIdEBC; zsq3WTs<~I}@O`}C_l#P2|E+GKH`xfaDQ3nTX9l-=%<3`unA{v{<7_Npp&fYPg69gP zL}jQHmE1z*X0^Tb^z`x=q(H@}2yI7&s6dUjpj@;S<#@wpt!Bs!U}F^v^!gK{Faxa| zjw6s_~Ke>PXEsc%!Pa$0m@U2^8BGE z&IKQ(w9kXIcK2bFic;FNC3u=?hHCs2(Rycw&`h&4#iz-UojVa0lokLzFp|{vK#Zia z-T{IC|6mD%PY!UirxEfwe>gL-F?E@`&{`$((qS6Bjn)dgO+qt}7(iVqDL-0D#tY4% z5|S4>^p`Xk(oO}$3OGe^#BSarAq5WERd`4fW2O4B&=f}U7*UV3+s1(+g-9c}B}5V? z8z8#GKShhpAjSX=hvWUC8}Jw(E$)fI@}cl9ZWcF6I0{VUejxvm>FSn-Ne+p^DOL!J z)@nvjS~x5b37gm(ajE**d5+S4#EDp4v{C$Oz&EjzlO(e2(+{Y0g<3IF8mf%{Z|CepH2E&5HsXIrYsXxERC3 zz>>46r(&q;KCvbA@|lwqh^P9uSb`XeWU&tx&KHAK`r*|^UU|Jn_2esqn^A47NURKZ z)4AzfD?Q)6TB1=hS7RYhKgwXES75kS(~lCZ%2Jezi`r4nufo&+#bLF;eq;_Ugg0$2 z3RIv#g$lGC)4B-vV%$p>IHJL|)v$Nv8RkP3`PITBRz$l?ASeWjE6E zEP%Y2c~>BFs#*qYcAfWw-)i-Oq)pq3vha1;sNzqbJ)RLUuJWgCp-sSdi4^jA8$MxU0jpTPc92p0E zuaEMK`|`j9luPgF@(lM3!)+6q#mFsgTw=i9F-t+@Io-x}lg#{@aZq!}NlozKSr>8F zqzWqrsh&aVmwQwgHApp5#}78T!x7vX*>%(%Czm0y;tdRJ`I|t}YAT#v>BN-ZVMkKA z)PI*_TezU5ysuyq@3wJ}GA;U9Du{2+4};5PxgJA>*Nv0zN-**_wv|h RmVvv%Yq;Ypz3*k0`k%CgS!n= 0.55.0; +//pragma AbiHeader time; + +// This contract demonstrates custom replay protection functionality. +contract CustomReplaySample { + + uint public value; + + function addValue(uint num) public { + tvm.accept(); + value += num; + } + + function getValue() public view returns (uint) { + return value; + } + + // Function with predefined name which is used to replace custom replay protection. + function afterSignatureCheck(TvmSlice body, TvmCell) private pure inline returns (TvmSlice) { + // Via TvmSlice methods we read header fields from the message body + + body.decode(uint64); // The first 64 bits contain timestamp which is usually used to differentiate messages. + + // After reading message headers this function must return the rest of the body slice. + return body; + } +} \ No newline at end of file diff --git a/tests/test_remp/test_remp.sh b/tests/test_remp/test_remp.sh new file mode 100755 index 00000000..2ea5c62b --- /dev/null +++ b/tests/test_remp/test_remp.sh @@ -0,0 +1,422 @@ +source "../setup_test_paths.sh" + +contracts_count=20 +messages_count=50 +timeout_sec=0.5 +prime_numbers=( + 1091 1093 1097 1103 1109 1117 1123 1129 1151 1153 1163 1171 1181 1187 1193 1201 1213 1217 1223 + 1231 1237 1249 1259 1277 1279 1283 1289 1291 1297 1301 1303 1307 1319 1321 1327 1361 1367 1373 + 1399 1409 1423 1427 1429 1433 1439 1447 1451 1453 1459 1471 1481 1483 1487 1489 1493 1499 1511 + 1531 1543 1549 1553 1559 1567 1571 1579 1583 1597 1601 1607 1609 1613 1619 1621 1627 1637 1657 + 1667 1669 1693 1697 1699 1709 1721 1723 1733 1741 1747 1753 1759 1777 1783 1787 1789 1801 1811 + 1831 1847 1861 1867 1871 1873 1877 1879 1889 1901 1907 1913 1931 1933 1949 1951 1973 1979 1987 + 1997 1999 2003 2011 2017 2027 2029 2039 2053 2063 2069 2081 2083 2087 2089 2099 2111 2113 2129 + 2137 2141 2143 2153 2161 2179 2203 2207 2213 2221 2237 2239 2243 2251 2267 2269 2273 2281 2287 + 2297 2309 2311 2333 2339 2341 2347 2351 2357 2371 2377 2381 2383 2389 2393 2399 2411 2417 2423 + 2441 2447 2459 2467 2473 2477 2503 2521 2531 2539 2543 2549 2551 2557 2579 2591 2593 2609 2617 + 2633 2647 2657 2659 2663 2671 2677 2683 2687 2689 2693 2699 2707 2711 2713 2719 2729 2731 2741 + 2753 2767 2777 2789 2791 2797 2801 2803 2819 2833 2837 2843 2851 2857 2861 2879 2887 2897 2903 + 2917 2927 2939 2953 2957 2963 2969 2971 2999 3001 3011 3019 3023 3037 3041 3049 3061 3067 3079 + 3089 3109 3119 3121 3137 3163 3167 3169 3181 3187 3191 3203 3209 3217 3221 3229 3251 3253 3257 + 3271 3299 3301 3307 3313 3319 3323 3329 3331 3343 3347 3359 3361 3371 3373 3389 3391 3407 3413 + 3449 3457 3461 3463 3467 3469 3491 3499 3511 3517 3527 3529 3533 3539 3541 3547 3557 3559 3571 + 3583 3593 3607 3613 3617 3623 3631 3637 3643 3659 3671 3673 3677 3691 3697 3701 3709 3719 3727 + 3739 3761 3767 3769 3779 3793 3797 3803 3821 3823 3833 3847 3851 3853 3863 3877 3881 3889 3907 + 3917 3919 3923 3929 3931 3943 3947 3967 3989 4001 4003 4007 4013 4019 4021 4027 4049 4051 4057 + 4079 4091 4093 4099 4111 4127 4129 4133 4139 4153 4157 4159 4177 4201 4211 4217 4219 4229 4231 + 4243 4253 4259 4261 4271 4273 4283 4289 4297 4327 4337 4339 4349 4357 4363 4373 4391 4397 4409 + 4423 4441 4447 4451 4457 4463 4481 4483 4493 4507 4513 4517 4519 4523 4547 4549 4561 4567 4583 + 4597 4603 4621 4637 4639 4643 4649 4651 4657 4663 4673 4679 4691 4703 4721 4723 4729 4733 4751 + 4783 4787 4789 4793 4799 4801 4813 4817 4831 4861 4871 4877 4889 4903 4909 4919 4931 4933 4937 + 4951 4957 4967 4969 4973 4987 4993 4999 5003 5009 5011 5021 5023 5039 5051 5059 5077 5081 5087 + 5101 5107 5113 5119 5147 5153 5167 5171 5179 5189 5197 5209 5227 5231 5233 5237 5261 5273 5279 + 5297 5303 5309 5323 5333 5347 5351 5381 5387 5393 5399 5407 5413 5417 5419 5431 5437 5441 5443 + 5471 5477 5479 5483 5501 5503 5507 5519 5521 5527 5531 5557 5563 5569 5573 5581 5591 5623 5639 + 5647 5651 5653 5657 5659 5669 5683 5689 5693 5701 5711 5717 5737 5741 5743 5749 5779 5783 5791 + 5807 5813 5821 5827 5839 5843 5849 5851 5857 5861 5867 5869 5879 5881 5897 5903 5923 5927 5939 + 5981 5987 6007 6011 6029 6037 6043 6047 6053 6067 6073 6079 6089 6091 6101 6113 6121 6131 6133 + 6151 6163 6173 6197 6199 6203 6211 6217 6221 6229 6247 6257 6263 6269 6271 6277 6287 6299 6301 + 6317 6323 6329 6337 6343 6353 6359 6361 6367 6373 6379 6389 6397 6421 6427 6449 6451 6469 6473 + 6491 6521 6529 6547 6551 6553 6563 6569 6571 6577 6581 6599 6607 6619 6637 6653 6659 6661 6673 + 6689 6691 6701 6703 6709 6719 6733 6737 6761 6763 6779 6781 6791 6793 6803 6823 6827 6829 6833 + 6857 6863 6869 6871 6883 6899 6907 6911 6917 6947 6949 6959 6961 6967 6971 6977 6983 6991 6997 + 7013 7019 7027 7039 7043 7057 7069 7079 7103 7109 7121 7127 7129 7151 7159 7177 7187 7193 7207 + 7213 7219 7229 7237 7243 7247 7253 7283 7297 7307 7309 7321 7331 7333 7349 7351 7369 7393 7411 + 7433 7451 7457 7459 7477 7481 7487 7489 7499 7507 7517 7523 7529 7537 7541 7547 7549 7559 7561 + 7577 7583 7589 7591 7603 7607 7621 7639 7643 7649 7669 7673 7681 7687 7691 7699 7703 7717 7723 + 7727 7741 7753 7757 7759 7789 7793 7817 7823 7829 7841 7853 7867 7873 7877 7879 7883 7901 7907 +) +sold=${BASE_DIR}/TVM-Solidity-Compiler/target/release/sold +tvm_linker=${BASE_DIR}/TVM-linker/target/release/tvm_linker +lib="${BASE_DIR}/TVM-Solidity-Compiler/lib" +tonos_cli="${BASE_DIR}/tonos-cli/target/release/tonos-cli" +giver_keys=${TESTS_DIR}/test_high_load/msig.keys.json +giver_abi=${TESTS_DIR}/test_high_load/SafeMultisigWallet.abi.json +console=${TOOLS_DIR}/console +console_config=${CONSOLE_CONFIG_0} +contract_src="no_replay.sol" +abi_file="${contract_src%.sol}.abi.json" +node_log="/shared/output_0.log" + +# $1 number of contract $2 tvc_file +function pre_deploy { + local n=$1 + local tvc_file=$2 + local msg_boc="msg.$n.boc" + + # Generate address, keys and seed phrase + echo "Generating address, keys and seed phrase..." + local keys_file="${contract_src%.sol}.$n.keys.json" + if ! output="$($tonos_cli genaddr $tvc_file --abi $abi_file --genkey $keys_file)" ; then + echo "ERROR: $output" + return 1 + fi + local address=$(echo "$output" | grep "Raw address" | cut -c 14-) + echo $address > "${n}__address" + + # Send money to contract's address + # prepare message + echo "Preparing message to send money..." + if ! output="$( $tonos_cli \ + message "-1:7777777777777777777777777777777777777777777777777777777777777777" \ + submitTransaction '{ + "dest" : "'$address'", + "value":"10000000000", + "bounce":false, + "allBalance":false, + "payload":"" + }' \ + --abi "$giver_abi" \ + --sign "$giver_keys" \ + --output "$msg_boc" \ + --raw )" + then + echo "ERROR: $output" + return 1 + fi + local message_id=$( echo "$output" | grep "MessageId" | cut -c 12- ) + echo "Message id: " $message_id + # send it + echo "Sending money to $address..." + if ! output="$($console -C $console_config -c "sendmessage $msg_boc")" ; then + echo "ERROR: $output" + return 1 + fi + look_after_message $message_id & + sleep 20 + + # Ask contract's status to check have it got money + for (( attempt=0; attempt < 10; attempt++ )); do + echo "Ask contract's status to check have it got money..." + if ! output="$( $console -C $console_config -c "getaccount $address" )" ; then + echo "ERROR: $output" + return 1 + fi + # echo "$output" + if [ $(echo "$output" | grep "Uninit" | wc -l) != "1" ] ; then + if [ $attempt -eq 9 ] ; then + echo "ERROR can't find uninit account with money." + return 1 + else + echo "WARN can't find uninit account with money." + fi + else + break + fi + sleep 5 + done + + echo "$address successfully got money to future deploy." +} + +# $1 number of contract $2 tvc_file $3 address +function deploy { + + local n=$1 + local tvc_file=$2 + local address=$3 + local msg_boc="msg.$n.boc" + local keys_file="${contract_src%.sol}.$n.keys.json" + + # Deploy contract + # prepare message + echo "Preparing message to deploy contract..." + if ! output="$( $tonos_cli \ + deploy_message "$tvc_file" \ + --abi "$abi_file" \ + --sign "$keys_file" {} \ + --raw \ + --output "$msg_boc" )" + then + echo "ERROR: $output" + return 1 + fi + local message_id=$( echo "$output" | grep "MessageId" | cut -c 12- ) + echo "Message id: " $message_id + # send it + echo "Deploying $address..." + if ! output="$( $console -C $console_config -c "sendmessage $msg_boc" )" ; then + echo "ERROR: $output" + return 1 + fi + look_after_message $message_id & + sleep 20 + + # Ask contract's status to check have it been deployed + echo "Ask contract's status to check have it been deployed..." + for (( attempt=0; attempt < 10; attempt++ )); do + echo "Ask contract's status to check have it got money..." + if ! output="$( $console -C $console_config -c "getaccount $address" )" ; then + echo "ERROR: $output" + return 1 + fi + echo "$output" + if [ $(echo "$output" | grep "Active" | wc -l) != "1" ] ; then + if [ $attempt -eq 9 ] ; then + echo "ERROR can't find uninit account with money." + return 1 + else + echo "WARN can't find uninit account with money." + fi + else + break + fi + sleep 5 + done + + echo "$address successfully deployed." +} + +# $1 message_id +function look_after_message { + local message_id=$1 + + for (( attempt=0; attempt < 25; attempt++ )); do + sleep 10 + + local output="$( cat $node_log | grep "New processing stage for external message $message_id" )" + if [ $(echo "$output" | grep "TonNode_RempAccepted(RempAccepted { level: TonNode_RempMasterchain" | wc -l) != "1" ] ; then + if [ $attempt -eq 24 ] ; then + echo "ERROR message $message_id was not accepted yet! Current statuses: " + echo "$output" + return 1 + else + echo "WARN message $message_id was not accepted yet! Current statuses: " + echo "$output" + fi + else + echo "Message $message_id was accepted! Statuses: " + echo "$output" + break + fi + done +} + +# $1 address $2 keys $3 number of message +function send_message { + local address=$1 + local keys_file=$2 + local message_number=$3 + local abi_file="${contract_src%.sol}.abi.json" + local msg_boc="msg.$n.boc" + local n=${prime_numbers[$message_number]} + + # prepare message + echo "Preparing message..." + if ! output="$( $tonos_cli \ + message "$address" \ + addValue '{ "num": '$n'}' \ + --abi "$abi_file" \ + --sign "$keys_file" \ + --output "$msg_boc" \ + --raw )" + then + echo "ERROR: $output" + return 1 + fi + local message_id=$( echo "$output" | grep "MessageId" | cut -c 12- ) + echo "Message id: " $message_id + + # send it + echo "Send to $address..." + if ! output="$($console -C $console_config -c "sendmessage $msg_boc")" ; then + echo "ERROR: $output" + return 1 + fi + + look_after_message $message_id & +} + +# 1 address $2 number of contract +function check_contract { + local address=$1 + local n=$2 + + # Calculate expected value + local expacted_sum=0 + for (( n=0; n < messages_count; n++ )); do + let "expacted_sum = expacted_sum + ${prime_numbers[$n]}" + done + + # local call (get method) + for (( attempt=0; attempt < 10; attempt++ )); do + echo "Obtaining contract's boc..." + if ! output="$( $console -C $console_config -c "getaccountstate $address $n.boc" )" ; then + echo "ERROR: $output" + return 1 + fi + echo "Calling contract's method locally..." + if ! output="$( $tonos_cli \ + run $n.boc \ + --abi "$abi_file" \ + --boc \ + getValue '{ }' )" + then + echo "ERROR $output" + return 1 + fi + # echo "$output" + + local hex=$(printf "%x" $expacted_sum) + if [ $(echo "$output" | grep $hex | wc -l) != "1" ] ; then + echo "$output" + echo "expacted_sum: "$expacted_sum" hex: "$hex + if [ $attempt -eq 9 ] ; then + echo "ERROR contract's value != expected one." + return 1 + else + echo "WARN contract's value != expected one." + fi + else + break + fi + sleep 10 + done +} + +# $1 number of contract $2 tvc_file +function one_contract_scenario { + local n=$1 + local tvc_file=$2 + local keys_file="${contract_src%.sol}.$n.keys.json" + local address=$(cat ${n}__address) + + if ! deploy $n $tvc_file $address ; then + return 1 + fi + + for (( mn=0; mn < messages_count; mn++ )); do + # $1 address $2 keys $3 number of message + if ! send_message $address $keys_file $mn ; then + return 1 + fi + sleep $timeout_sec + done + + echo "waiting..." + sleep 30 + + if ! check_contract $address $n ; then + return 1 + fi + + echo "$address succesfully finished their scenario!" + + exit 0 +} + +function cleanup { + rm *.log + rm *.boc + rm *__address + rm *.json + rm *.code + rm *.tvc +} + +# +# TEST REMP +# Pre deploy: send money to contracts (one by one) +# In parallel: +# - deploy contract +# - send messages - each message contains next prime number, +# contract stores sum of all got numbers. +# - check contract state - check if all messages was delivered exactly ones. +# We will check if contract's sum is equal to sum of sent numbers. + +cleanup + +if [ $messages_count -gt ${#prime_numbers[*]} ]; then + echo "messages_count value is too big, max=${#prime_numbers[*]}" + exit 1 +fi + +# Don't forget to deploy giver first! + +# compile & link test contract +echo "Compile..." +if ! $sold $contract_src ; then exit 1 ; fi +#code_file="${contract_src%.sol}.tvc" + +#echo "Linking..." +#if ! output="$($tvm_linker compile $code_file --lib $lib/stdlib_sol.tvm)" ; then +# echo "ERROR $output" +# exit 1 +#fi +#tvc_file=$(echo "$output" | grep "Saved contract to file" | cut -c 24-) +tvc_file="${contract_src%.sol}.tvc" + +# +echo "Obtaining blockchain's config..." +if ! output="$( $console -C $console_config -c "getblockchainconfig" )" ; then + echo "ERROR $output" + exit 1 +fi +echo $output | grep b5ee9c72 | xxd -r -p > config.boc + +# +echo "Send money to contracts (one by one)" +rm pre_deploy.log +for (( n=0; n < contracts_count; n++ )); do + echo "Sending money to #$n..." + echo " + + + Sending money to #$n... + + " >> pre_deploy.log + if ! pre_deploy $n $tvc_file >> pre_deploy.log ; then + # if ! output="$( pre_deploy $n $tvc_file )" ; then + # echo "ERROR $output" + exit 1 + else + echo "Sent money to #$n" + fi +done + +# run test scenario +echo "Run test scenario..." +for (( n=0; n < contracts_count; n++ )); do + one_contract_scenario $n $tvc_file > $n.log & +done + +wait + +# check log files of each contract +result=0 +for (( n=0; n < contracts_count; n++ )); do + if err="$( cat $n.log | grep ERROR )"; then + echo "Contract #$n has errors in their test scenario" + echo err + echo + result=1 + else + echo "$n succesfully finished their scenario!" + fi +done + +if [ $result -eq 0 ]; then + echo "Succesfully finished!" +fi + +exit $result diff --git a/tests/test_run_net/log_cfg.yml b/tests/test_run_net/log_cfg.yml index 9ef3ad15..43a0035d 100644 --- a/tests/test_run_net/log_cfg.yml +++ b/tests/test_run_net/log_cfg.yml @@ -25,7 +25,7 @@ appenders: limit: 5 gb roller: kind: fixed_window - pattern: '/shared/output_NODE_NUM_{}.log' + pattern: '/home/ruser/TON/ever-node/tests/test_high_load/logs/output_NODE_NUM_{}.log' base: 1 count: 10 From 34ae0b73175c7fc040468dbd0aa16ee8b86e7b7d Mon Sep 17 00:00:00 2001 From: Dmitry Shtukenberg Date: Thu, 15 Feb 2024 16:27:12 +0300 Subject: [PATCH 10/40] Better debugging info --- src/validator/message_cache.rs | 62 ++++++++++++++++++------------ src/validator/remp_catchain.rs | 6 --- src/validator/validator_manager.rs | 2 +- 3 files changed, 39 insertions(+), 31 deletions(-) diff --git a/src/validator/message_cache.rs b/src/validator/message_cache.rs index 920205e5..51418c5b 100644 --- a/src/validator/message_cache.rs +++ b/src/validator/message_cache.rs @@ -323,7 +323,7 @@ pub struct MessageCacheSession { ids_for_uid: LockfreeMapSet, message_headers: DashMap>, message_origins: DashMap>, - messages: Map>, + messages: DashMap>, message_events: LockfreeMapSet, //Map>, message_status: DashMap, @@ -375,9 +375,9 @@ impl MessageCacheSession { self.insert_message_header(&msg.message_id, msg_hdr, msg_origin)?; match self.messages.insert(msg.message_id.clone(), msg.clone()) { None => Ok(()), - Some(prev) if *prev.val() == msg => Ok(()), + Some(prev) if prev == msg => Ok(()), Some(p) => fail!("Different messages for same id {:x}, replacing {} with {}", - p.key(), p.val(), msg + msg.message_id, p, msg ) } } @@ -385,26 +385,31 @@ impl MessageCacheSession { fn update_missing_fields(&self, message_id: &UInt256, message: Option>, origin: Option>) -> Result<()> { if let Some(m1) = &message { if let Some(m2) = self.messages.get(message_id) { - if m1 != m2.val() { - fail!("Different message body: new body '{}', current cached body '{}'", m1, m2.val()); + if m1 != m2.value() { + fail!("Different message body: new body '{}', current cached body '{}'", m1, m2.value()); } } else { if let Some(m2) = self.messages.insert(message_id.clone(), m1.clone()) { - fail!("Parallel updating of message body: new body '{}', another body '{}'", m1, m2.val()) + if m1 != &m2 { + fail!("Parallel updating of message body: new body '{}', another body '{}'", m1, m2) + } } } } + // If origin is different from the stored one, that's not a big deal. if let Some(o1) = &origin { if let Some(o2) = self.message_origins.get(message_id) { if o1 != o2.value() { - fail!("Different message origin: new origin '{}', current cached origin '{}'", o1, o2.value()); + log::trace!("Different message origin: new origin '{}', current cached origin '{}'", o1, o2.value()); } } else { if let Some(o2) = self.message_origins.insert(message_id.clone(), o1.clone()) { - fail!("Parallel updating of message origin: new origin '{}', another origin '{}'", o1, o2) + if o1 != &o2 { + fail!("Different message origin: new origin '{}', current cached origin '{}'", o1, o2); + } } } } @@ -431,8 +436,8 @@ impl MessageCacheSession { self.messages.get(msg_id).is_some() } - fn all_messages_count(&self) -> usize { - self.message_headers.len() + fn all_messages_count(&self) -> (usize,usize,usize) { + (self.message_headers.len(), self.messages.len(), self.message_origins.len()) } fn update_message_status(&self, message_id: &UInt256, new_status: RempMessageStatus) -> Result { @@ -514,7 +519,7 @@ impl MessageCacheSession { (m, h) => { log::error!(target: "remp", "Record for message {:?} is in incorrect state: msg = {:?}, status = {:?}", - id, m.map(|x| x.val().clone()), h.map(|x| x.value().clone()) + id, m.map(|x| x.value().clone()), h.map(|x| x.value().clone()) ); stats.incorrect += 1 } @@ -531,7 +536,7 @@ impl MessageCacheSession { message_headers: DashMap::new(), message_origins: DashMap::new(), message_events: LockfreeMapSet::default(), - messages: Map::default(), + messages: DashMap::default(), message_status: DashMap::default(), inf_shards: HashSet::from_iter(inf_shards.into_iter()), blocks_processed: DashSet::default(), @@ -550,23 +555,31 @@ pub struct MessageCache { cache_size_metric: Arc, } -#[allow(dead_code)] impl MessageCache { - pub fn cc_expired(&self, old_cc_seqno: u32) -> bool { - old_cc_seqno < self.master_cc_seqno_lwb.load(Ordering::Relaxed) - } - - pub fn all_messages_count(&self) -> usize { + /// Returns stats about available messages' info in cache, three counts: + /// (total, with_bodies, with_origins). + /// total --- total messages in cache count; + /// with_bodies --- messages that have bodies (that is, broadcasted/received from full node); + /// with_origins --- messages that have origins (that is, received through catchain/from full node). + /// + /// Some messages never have body/origin (those that were taken from accepted master blocks), + /// others may have either body or origin missing due to delays or losses in network. + pub fn all_messages_count(&self) -> (usize,usize,usize) { let range = self.get_master_cc_stored_range(); let mut result: usize = 0; + let mut with_bodies: usize = 0; + let mut with_origins: usize = 0; for cc in range { if let Some(s) = self.sessions.get(&cc) { - result += s.val().all_messages_count(); + let (r,b,o) = s.val().all_messages_count(); + result += r; + with_bodies += b; + with_origins += o; } } - result + (result, with_bodies, with_origins) } /// Returns new message status, if it worths reporting (final statuses do not need to be reported) @@ -617,7 +630,7 @@ impl MessageCache { pub fn get_message(&self, message_id: &UInt256) -> Result>> { let msg = self.get_session_for_message(message_id).map( |session| session.messages.get(message_id).map( - |m| m.val().clone() + |m| m.value().clone() ) ); Ok(msg.flatten()) @@ -650,7 +663,7 @@ impl MessageCache { }; let (msg, origin, status) = ( - session.messages.get(message_id).map(|m| m.val().clone()), + session.messages.get(message_id).map(|m| m.value().clone()), session.message_origins.get(message_id).map(|m| m.value().clone()), session.message_status.get(message_id).map(|m| m.value().clone()) ); @@ -868,7 +881,8 @@ impl MessageCache { } pub fn message_stats(&self) -> String { - format!("All REMP messages count = {}", self.all_messages_count()) + let (count,with_origins,with_bodies) = self.all_messages_count(); + format!("All REMP messages count = {}, of them: with origins (via catchain) = {}, with bodies (broadcasted) = {}", count, with_origins, with_bodies) } pub fn try_set_master_cc_start_time(&self, master_cc: u32, start_time: UnixTime32, inf_blocks: Vec) -> Result<()> { @@ -1063,7 +1077,7 @@ impl MessageCache { stats.add(&session.val().gc_all()); #[cfg(feature = "telemetry")] - self.cache_size_metric.update(self.all_messages_count() as u64); + self.cache_size_metric.update(self.all_messages_count().0 as u64); } self.master_cc_seqno_stored.store(cc_to_remove+1, Relaxed); } diff --git a/src/validator/remp_catchain.rs b/src/validator/remp_catchain.rs index be1ae717..30e0bf28 100644 --- a/src/validator/remp_catchain.rs +++ b/src/validator/remp_catchain.rs @@ -730,12 +730,6 @@ impl RempCatchainStore { Ok(()) } - pub async fn get_catchain_session(&self, session_id: &UInt256) -> Option { - self.catchains.execute_sync(|x| { - x.get(session_id).map(|rcw| rcw.info.instance.get_session()).flatten() - }).await - } - pub async fn list_catchain_sessions(&self) -> Vec<(String, UInt256)> { let sessions_to_list = self.catchains.execute_sync(|x| { let mut sessions_to_list = Vec::new(); diff --git a/src/validator/validator_manager.rs b/src/validator/validator_manager.rs index a0363595..9dec8cab 100644 --- a/src/validator/validator_manager.rs +++ b/src/validator/validator_manager.rs @@ -1286,7 +1286,7 @@ impl ValidatorManagerImpl { } if let Some(rm) = &self.remp_manager { - metrics::gauge!("remp_message_cache_size", rm.message_cache.all_messages_count() as f64); + metrics::gauge!("remp_message_cache_size", rm.message_cache.all_messages_count().0 as f64); log::info!(target: "validator_manager", "Remp message cache stats: {}", rm.message_cache.message_stats()); for (s, shard_ident) in rm.catchain_store.list_catchain_sessions().await.iter() { From 4c0be2ede5d92e1e11fed183d00e2614e2b758c2 Mon Sep 17 00:00:00 2001 From: Dmitry Shtukenberg Date: Wed, 21 Feb 2024 17:05:01 +0300 Subject: [PATCH 11/40] Cache and debug info improvements --- src/validator/message_cache.rs | 53 ++++++++++++++++++++++++---------- 1 file changed, 38 insertions(+), 15 deletions(-) diff --git a/src/validator/message_cache.rs b/src/validator/message_cache.rs index 51418c5b..0f46747b 100644 --- a/src/validator/message_cache.rs +++ b/src/validator/message_cache.rs @@ -432,10 +432,6 @@ impl MessageCacheSession { return true } - fn is_full_message(&self, msg_id: &UInt256) -> bool { - self.messages.get(msg_id).is_some() - } - fn all_messages_count(&self) -> (usize,usize,usize) { (self.message_headers.len(), self.messages.len(), self.message_origins.len()) } @@ -465,14 +461,15 @@ impl MessageCacheSession { } fn message_events_to_string(&self, message_id: &UInt256) -> String { - if let Some(msg) = self.message_origins.get(message_id) { + let events = self.message_events.get_set(message_id); + let result = if let Some(msg) = self.message_origins.get(message_id) { let base = msg.value().timestamp; - let events = self.message_events.get_set(message_id); events.iter().map(|x| format!("{} ", (*x) as i64 - base as i64)).collect() } else { "*events: no message origin*".to_string() - } + }; + format!("{} ({} events)", result, events.len()) } fn message_info(&self, message_id: &UInt256) -> String { @@ -485,14 +482,30 @@ impl MessageCacheSession { .map(|x| format!("{:?}", x.value())) .unwrap_or_else(|| "*error: no status in cache*".to_owned()); - let collation_history = if self.is_full_message(message_id) { - self.message_events_to_string(message_id) + let has_additional_info = self.messages.contains_key(message_id) + || self.message_origins.contains_key(message_id) + || !self.message_events.get_set(message_id).is_empty(); + + let additional_info = if has_additional_info { + let body = self.messages + .get(message_id).map(|_| "has body".to_owned()) + .unwrap_or_else(|| "*error: no body*".to_owned()); + + let origin = self.message_origins + .get(message_id).map(|x| x.value().to_string()) + .unwrap_or_else(|| "*error: no origin*".to_owned()); + + let collation_history = self.message_events_to_string(message_id); + + format!("{}, {}, {}", body, origin, collation_history) } else { "header only".to_owned() }; - format!("id {:x}, cc {}, {}, status: {}, {}", message_id, self.master_cc, header, status, collation_history) + format!("id {:x}, cc {}, {}, status: {}, {}", + message_id, self.master_cc, header, status, additional_info + ) } fn mark_collation_attempt(&self, msg_id: &UInt256) -> Result<()> { @@ -682,6 +695,13 @@ impl MessageCache { } } + fn get_message_info(&self, message_id: &UInt256) -> Result { + match self.get_session_for_message(message_id) { + None => Ok("absent".to_string()), + Some(s) => Ok(s.message_info(message_id)) + } + } + fn insert_message(&self, session: Arc, message: Arc, message_header: Arc, message_origin: Option>, status: &RempMessageStatus) -> Result<()> { if message.message_id != message_header.message_id { fail!("Inconsistent message: message {} and message_header {} have different message_id", message, message_header) @@ -754,7 +774,7 @@ impl MessageCache { } pub fn update_message_body(&self, data: Arc) -> Result<()> { - self.add_external_message_status( + let (old_status,new_status) = self.add_external_message_status( &data.message_id, &data.message_uid, Some(data.clone()), @@ -762,10 +782,13 @@ impl MessageCache { RempMessageStatus::TonNode_RempNew, |old,_new| old.clone(), self.master_cc_seqno_curr.load(Ordering::Relaxed) )?; - match self.get_message_with_origin_status_cc(&data.message_id)? { - Some((msg, msg_origin, x, seqno)) => log::info!(target: "remp", "Message info added: {}, {}, {}, {}", msg, msg_origin, x, seqno), - None => log::info!(target: "remp", "Message {} was not found in cache", &data.message_id) - } + let info = self.get_message_info(&data.message_id)?; + log::trace!(target: "remp", "Message {}, tried to update message body: old status {}, new status {}, full message cache info {}", + &data, + match old_status { Some(m) => format!("{}",m), None => format!("None") }, + new_status, + info + ); Ok(()) } From c32e38edd524fa511061ca5e8664433bfc8d9604 Mon Sep 17 00:00:00 2001 From: Dmitry Shtukenberg Date: Wed, 21 Feb 2024 22:45:21 +0300 Subject: [PATCH 12/40] Additional logs about broadcasts processing --- src/validator/remp_catchain.rs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/validator/remp_catchain.rs b/src/validator/remp_catchain.rs index 30e0bf28..789caeab 100644 --- a/src/validator/remp_catchain.rs +++ b/src/validator/remp_catchain.rs @@ -467,12 +467,12 @@ impl RempCatchain { } } - fn unpack_broadcast(&self, payload: &BlockPayloadPtr) -> Result<()> { + fn unpack_broadcast(&self, payload: &BlockPayloadPtr) -> Result { let message = RmqMessage::deserialize(payload.data())?; - let rmqrecord = RmqMessage::from_rmq_record(&message)?; - self.remp_manager.message_cache.update_message_body(Arc::new(rmqrecord))?; + let rmqrecord = Arc::new(RmqMessage::from_rmq_record(&message)?); + self.remp_manager.message_cache.update_message_body(rmqrecord.clone())?; - Ok(()) + Ok(rmqrecord.message_id.clone()) } } @@ -548,9 +548,11 @@ impl CatchainListener for RempCatchain { } fn process_broadcast(&self, source_id: PublicKeyHash, data: BlockPayloadPtr) { - log::trace!(target: "remp", "MessageQueue {} process broadcast from {}", self, source_id); - if let Err(e) = self.unpack_broadcast(&data) { - log::error!(target: "remp", "Error processing broadcast from {}, message body will be ignored: `{}`", source_id, e); + let len = data.data().len(); + log::trace!(target: "remp", "MessageQueue {} process broadcast from {}, len {}", self, source_id, len); + match self.unpack_broadcast(&data) { + Err(e) => log::error!(target: "remp", "MessageQueue {}, error processing broadcast from {}, message body will be ignored: `{}`", self, source_id, e), + Ok(id) => log::trace!(target: "remp", "MessageQueue {} broadcast message: id {:x}, len {}", self, id, len) } } From 558c40a5756650a25d370f75386f0df35adb893a Mon Sep 17 00:00:00 2001 From: Dmitry Shtukenberg Date: Tue, 5 Mar 2024 16:26:42 +0300 Subject: [PATCH 13/40] Added broadcast requests --- src/ext_messages.rs | 2 +- src/validator/message_cache.rs | 35 +++++---- src/validator/reliable_message_queue.rs | 76 +++++++++++++++---- src/validator/remp_catchain.rs | 97 +++++++++++++++++++++++-- 4 files changed, 170 insertions(+), 40 deletions(-) diff --git a/src/ext_messages.rs b/src/ext_messages.rs index 3421afe1..6073e8dd 100644 --- a/src/ext_messages.rs +++ b/src/ext_messages.rs @@ -27,7 +27,7 @@ const MESSAGE_LIFETIME: u32 = 600; // seconds const MESSAGE_MAX_GENERATIONS: u8 = 3; const MAX_EXTERNAL_MESSAGE_DEPTH: u16 = 512; -const MAX_EXTERNAL_MESSAGE_SIZE: usize = 65535; +pub const MAX_EXTERNAL_MESSAGE_SIZE: usize = 65535; pub const EXT_MESSAGES_TRACE_TARGET: &str = "ext_messages"; diff --git a/src/validator/message_cache.rs b/src/validator/message_cache.rs index 0f46747b..85114cb0 100644 --- a/src/validator/message_cache.rs +++ b/src/validator/message_cache.rs @@ -71,16 +71,6 @@ impl RmqMessage { return Ok(RmqMessage { message, message_id, message_uid }) } - pub fn as_remp_catchain_record(&self, master_cc: u32, origin: &RempMessageOrigin) -> ton_api::ton::ton_node::RempCatchainRecordV2 { - ton_api::ton::ton_node::rempcatchainrecordv2::RempCatchainMessageHeaderV2 { - message_id: self.message_id.clone().into(), - message_uid: self.message_uid.clone().into(), - source_key_id: UInt256::from(origin.source_key.data()), - source_idx: origin.source_idx as i32, - masterchain_seqno: master_cc as i32 - }.into_boxed() - } - pub fn as_rmq_record(&self) -> ton_api::ton::ton_node::RmqRecord { ton_api::ton::ton_node::rmqrecord::RmqMessage { message: self.message.write_to_bytes().unwrap().into(), @@ -238,6 +228,16 @@ impl RempMessageOrigin { } } + pub fn as_remp_catchain_record(&self, message_id: &UInt256, message_uid: &UInt256, master_cc: u32) -> ton_api::ton::ton_node::RempCatchainRecordV2 { + ton_api::ton::ton_node::rempcatchainrecordv2::RempCatchainMessageHeaderV2 { + message_id: message_id.clone().into(), + message_uid: message_uid.clone().into(), + source_key_id: UInt256::from(self.source_key.data()), + source_idx: self.source_idx as i32, + masterchain_seqno: master_cc as i32 + }.into_boxed() + } + pub fn from_remp_catchain(record: &RempCatchainMessageHeaderV2) -> Result { Self::new( KeyId::from_data(record.source_key_id.as_slice().clone()), @@ -293,7 +293,7 @@ impl RempMessageWithOrigin { } pub fn as_remp_catchain_record(&self, master_cc: u32) -> ton_api::ton::ton_node::RempCatchainRecordV2 { - return self.message.as_remp_catchain_record(master_cc, &self.origin) + return self.origin.as_remp_catchain_record(&self.message.message_id, &self.message.message_uid, master_cc) } pub fn as_rmq_record(&self) -> ton_api::ton::ton_node::RmqRecord { @@ -669,22 +669,25 @@ impl MessageCache { } } - pub fn get_message_with_origin_status_cc(&self, message_id: &UInt256) -> Result, Arc, RempMessageStatus, u32)>> { + pub fn get_message_with_origin_status_cc(&self, message_id: &UInt256) -> Result>, Arc, Arc, RempMessageStatus, u32)>> { let session = match self.get_session_for_message(message_id) { None => return Ok(None), Some(s) => s }; - let (msg, origin, status) = ( + let (header, msg, origin, status) = ( + session.message_headers.get(message_id).ok_or_else(|| error!("Message {:x} has no header", message_id))?.value().clone(), session.messages.get(message_id).map(|m| m.value().clone()), session.message_origins.get(message_id).map(|m| m.value().clone()), session.message_status.get(message_id).map(|m| m.value().clone()) ); match (msg, origin, status) { - (None, _, Some(_)) | (_, None, Some(_)) => Ok(None), // Bare message info (retrieved from finalized block/not received from broadcast) - (Some(m), Some(o), Some (h)) => Ok(Some((m.clone(), o.clone(), h.clone(), session.master_cc))), // Full message info - (m, o, None) => fail!("Message {:x} has no status, body = {:?}, origin = {:?}", message_id, m, o), + (_, None, Some(_)) => Ok(None), // Bare message info (retrieved from finalized block/not received from broadcast) + (m, Some(o), Some (s)) => + Ok(Some((m.clone(), header.clone(), o.clone(), s.clone(), session.master_cc))), // Full message info + (m, o, None) => + fail!("Message {:x} has no status, body = {:?}, origin = {:?}", message_id, m, o), } } diff --git a/src/validator/reliable_message_queue.rs b/src/validator/reliable_message_queue.rs index c4e1c8e0..3601b062 100644 --- a/src/validator/reliable_message_queue.rs +++ b/src/validator/reliable_message_queue.rs @@ -33,7 +33,7 @@ use ton_types::{UInt256, Result, fail}; use catchain::{PrivateKey, PublicKey}; use crate::{ engine_traits::EngineOperations, - ext_messages::{get_level_and_level_change, is_finally_accepted, is_finally_rejected}, + ext_messages::{get_level_and_level_change, is_finally_accepted, is_finally_rejected, MAX_EXTERNAL_MESSAGE_SIZE}, validator::{ mutex_wrapper::MutexWrapper, message_cache::{RmqMessage, RempMessageHeader, RempMessageOrigin, RempMessageWithOrigin}, @@ -53,6 +53,8 @@ enum MessageQueueStatus { Created, Starting, Active, Stopping } const RMQ_STOP_POLLING_INTERVAL: Duration = Duration::from_millis(50); const RMQ_REQUEST_NEW_BLOCK_INTERVAL: Duration = Duration::from_millis(50); const RMQ_MAXIMAL_BROADCASTS_IN_PACK: u32 = 1000; +const RMQ_MAXIMAL_QUERIES_IN_PACK: u32 = 1000; +const RMQ_MESSAGE_QUERY_TIMEOUT: Duration = Duration::from_millis(2000); struct MessageQueueImpl { status: MessageQueueStatus, @@ -190,13 +192,13 @@ impl MessageQueue { pub async fn update_status_send_response_by_id(&self, msgid: &UInt256, new_status: RempMessageStatus) -> Result> { match &self.remp_manager.message_cache.get_message_with_origin_status_cc(msgid)? { - Some((rm,origin,_,_)) => { + Some((Some(rm),_header,origin,_,_)) => { self.update_status_send_response(msgid, origin.clone(), new_status.clone()); Ok(rm.clone()) }, - None => + Some((None,_,_,_,_)) | None => fail!( - "RMQ {}: cannot find message {:x} in RMQ messages hash! (new status {})", + "RMQ {}: cannot find message {:x} body in RMQ messages hash! (new status {})", self, msgid, new_status ) } @@ -270,7 +272,13 @@ impl MessageQueue { if self.catchain_instance.is_session_active() { log::trace!(target: "remp", "Point 3. Activating RMQ {} processing", self); - self.catchain_instance.activate_exchange(RMQ_REQUEST_NEW_BLOCK_INTERVAL, RMQ_MAXIMAL_BROADCASTS_IN_PACK)?; + self.catchain_instance.activate_exchange( + RMQ_REQUEST_NEW_BLOCK_INTERVAL, + RMQ_MAXIMAL_BROADCASTS_IN_PACK, + RMQ_MAXIMAL_QUERIES_IN_PACK, + RMQ_MESSAGE_QUERY_TIMEOUT, + MAX_EXTERNAL_MESSAGE_SIZE as u64 + 1000 + )?; // Temporary status "New" --- message is not registered yet self.send_response_to_fullnode(msg.get_message_id(), origin_with_idx, RempMessageStatus::TonNode_RempNew); @@ -495,7 +503,7 @@ impl MessageQueue { /// hash map. Check status of all old messages in the hash map. pub async fn poll(&self) -> Result<()> { log::debug!(target: "remp", "Point 4. RMQ {}: polling; total {} messages in cache, {} messages in rmq_queue", - self, + self, self.received_messages_count().await, self.catchain_instance.rmq_catchain_receiver_len()? ); @@ -521,6 +529,23 @@ impl MessageQueue { } } + // Process responses for messages with not received bodies + 'responses_loop: loop { + match self.catchain_instance.query_response_receiver_try_recv() { + Err(e) => { + log::error!(target: "remp", "RMQ {}: error receiving from query_response_receiver queue: `{}`", self, e); + break 'responses_loop; + }, + Ok(None) => { + log::trace!(target: "remp", "RMQ {}: no more query responses", self); + }, + Ok(Some(rmq_record)) => { + let rmq_message = Arc::new(RmqMessage::from_rmq_record(&rmq_record)?); + self.remp_manager.message_cache.update_message_body(rmq_message.clone())?; + } + } + } + Ok(()) } @@ -538,15 +563,27 @@ impl MessageQueue { ); continue } + Ok(Some((None, h, o, s, cc))) => { + log::trace!( + target: "remp", + "Point 5. RMQ {}: message {:x} with status {:?} from {} found in pending_collation queue, but has no body yet; requesting body again", + self, msgid, s, o + ); + self.catchain_instance.pending_messages_queries_send( + o.source_key.clone(), + o.as_remp_catchain_record(&msgid, &h.message_uid, cc) + )?; + continue + }, Ok(None) => { log::warn!( target: "remp", - "Point 5. RMQ {}: message {:x} found in pending_collation queue, but has no body/origin yet; will not be collated", + "Point 5. RMQ {}: message {:x} found in pending_collation queue, but has no origin; will not be collated", self, msgid ); continue }, - Ok(Some((m, o, s, _cc))) => (m, o, s) + Ok(Some((Some(m), _h, o, s, _cc))) => (m, o, s) }; match status.clone() { @@ -804,8 +841,9 @@ impl BlockProcessor for StatusUpdater { async fn process_message(&self, message_id: &UInt256, _message_uid: &UInt256) { match self.queue.remp_manager.message_cache.get_message_with_origin_status_cc(message_id) { Err(e) => log::error!(target: "remp", "Point 7. Cannot get message {:x} from cache: {}", message_id, e), - Ok(None) => log::warn!(target: "remp", "Point 7. Message {:x} is not stored in cache (body or/and origin is missing)", message_id), - Ok(Some((message,origin,_,_))) => { + Ok(None) => log::warn!(target: "remp", "Point 7. Message {:x} is not stored in cache (origin is missing)", message_id), + Ok(Some((None, _, _, _, _))) => log::warn!(target: "remp", "Point 7. Message {:x} is not stored in cache (body is missing)", message_id), + Ok(Some((Some(message),_header,origin,_,_))) => { log::trace!(target: "remp", "Point 7. RMQ {} shard accepted message {}, {}, new status {}", self.queue, message, origin, self.new_status ); @@ -971,13 +1009,13 @@ impl RmqQueueManager { Ok(None) => { log::error!( target: "remp", - "Point 5a. RMQ {}: message {:x} found in pending_collation queue, but not in messages/message_statuses", + "Point 5a. RMQ {}: message {:x} found in pending_collation queue, but not in messages/message_statuses, not forwarding", self, msgid ); continue }, - Ok(Some((_m, _origin, status, _cc))) if status == RempMessageStatus::TonNode_RempTimeout => continue, - Ok(Some((_m, _origin, _status, cc))) if !new_cc_range.contains(&cc) => { + Ok(Some((_m, _header, _origin, status, _cc))) if status == RempMessageStatus::TonNode_RempTimeout => continue, + Ok(Some((_m, _header, _origin, _status, cc))) if !new_cc_range.contains(&cc) => { if *new_cc_range.end() < cc { log::error!(target: "remp", "Point 5a. RMQ {}: message {:x} is younger (cc={}) than next_cc_range end {}..={} -- impossible", self, msgid, cc, new_cc_range.start(), new_cc_range.end() @@ -985,7 +1023,15 @@ impl RmqQueueManager { } continue }, - Ok(Some((m, origin, status, cc))) => (m, origin, status, cc) + Ok(Some((None, _header, _origin, _status, _cc))) => { + log::error!( + target: "remp", + "Point 5a. RMQ {}: message {:x} found in pending_collation queue, but has no message body, not forwarding", + self, msgid + ); + continue + } + Ok(Some((Some(m), _header, origin, status, cc))) => (m, origin, status, cc) }; // Forwarding: @@ -1029,7 +1075,7 @@ impl RmqQueueManager { for new in next_queues.iter() { if let Err(x) = new.catchain_instance.pending_messages_broadcast_send( - message.as_remp_catchain_record(message_cc, &origin), + origin.as_remp_catchain_record(&message.message_id, &message.message_uid, message_cc), message.as_rmq_record() ) { diff --git a/src/validator/remp_catchain.rs b/src/validator/remp_catchain.rs index 789caeab..ece8f005 100644 --- a/src/validator/remp_catchain.rs +++ b/src/validator/remp_catchain.rs @@ -37,6 +37,7 @@ use catchain::{ use ton_api::{ IntoBoxed, ton::ton_node::RempCatchainRecordV2 }; +use ton_api::ton::ton_node::RempCatchainRecordV2::{TonNode_RempCatchainMessageDigestV2, TonNode_RempCatchainMessageHeaderV2}; use ton_api::ton::ton_node::RmqRecord; use ton_block::ValidatorDescr; use ton_types::{error, fail, KeyId, Result, UInt256}; @@ -61,25 +62,39 @@ pub struct RempCatchainInstanceImpl { pub rmq_catchain_receiver: crossbeam_channel::Receiver, rmq_catchain_sender: crossbeam_channel::Sender, + query_response_receiver: crossbeam_channel::Receiver, + query_response_sender: Arc>, + + pending_messages_queries_receiver: crossbeam_channel::Receiver<(Arc,RempCatchainRecordV2)>, + pending_messages_queries_sender: crossbeam_channel::Sender<(Arc,RempCatchainRecordV2)>, + pub pending_messages_broadcast_receiver: crossbeam_channel::Receiver, - pending_messages_broadcast_sender: crossbeam_channel::Sender + pending_messages_broadcast_sender: crossbeam_channel::Sender, } impl RempCatchainInstanceImpl { fn new(catchain_ptr: CatchainPtr) -> Self { - let (pending_messages_queue_sender, pending_messages_queue_receiver) = - crossbeam_channel::unbounded(); - let (rmq_catchain_sender, rmq_catchain_receiver) = - crossbeam_channel::unbounded(); + let (pending_messages_queue_sender, pending_messages_queue_receiver) = crossbeam_channel::unbounded(); + let (rmq_catchain_sender, rmq_catchain_receiver) = crossbeam_channel::unbounded(); let (remp_messages_sender, remp_messages_receiver) = crossbeam_channel::unbounded(); + let (query_response_sender, query_response_receiver) = crossbeam_channel::unbounded(); + let (pending_messages_queries_sender, pending_messages_queries_receiver) = crossbeam_channel::unbounded(); Self { catchain_ptr, pending_messages_queue_sender, pending_messages_queue_receiver, rmq_catchain_sender, rmq_catchain_receiver, pending_messages_broadcast_sender: remp_messages_sender, - pending_messages_broadcast_receiver: remp_messages_receiver + pending_messages_broadcast_receiver: remp_messages_receiver, + query_response_sender: Arc::new(query_response_sender), + query_response_receiver, + pending_messages_queries_sender, + pending_messages_queries_receiver } } + + fn get_query_response_sender(&self) -> Arc> { + self.query_response_sender.clone() + } } pub struct RempCatchainInstance { @@ -154,6 +169,32 @@ impl RempCatchainInstance { } } + pub fn pending_messages_queries_send(&self, dst: Arc, query: RempCatchainRecordV2) -> Result<()> { + let instance = self.get_instance_impl()?; + match instance.pending_messages_queries_sender.send((dst, query)) { + Ok(()) => Ok(()), + Err(e) => fail!("pending_messages_queries_send error: {}", e) + } + } + + pub fn pending_messages_queries_try_recv(&self) -> Result,RempCatchainRecordV2)>> { + let instance = self.get_instance_impl()?; + match instance.pending_messages_queries_receiver.try_recv() { + Ok(x) => Ok(Some(x)), + Err(crossbeam_channel::TryRecvError::Empty) => Ok(None), + Err(crossbeam_channel::TryRecvError::Disconnected) => fail!("channel disconnected") + } + } + + pub fn query_response_receiver_try_recv(&self) -> Result> { + let instance = self.get_instance_impl()?; + match instance.query_response_receiver.try_recv() { + Ok(x) => Ok(Some(x)), + Err(crossbeam_channel::TryRecvError::Empty) => Ok(None), + Err(crossbeam_channel::TryRecvError::Disconnected) => fail!("query response receiver channel disconnected") + } + } + pub fn rmq_catchain_receiver_len(&self) -> Result { let instance = self.get_instance_impl()?; Ok(instance.rmq_catchain_receiver.len()) @@ -195,10 +236,50 @@ impl RempCatchainInstance { Ok(()) } - pub fn activate_exchange(&self, delay: Duration, max_broadcasts: u32) -> Result<()> { + fn deserialize_and_put_query_response_to_queue(r: Result, query_response_sender: Arc>) -> Result<()> { + let response_record = RmqMessage::deserialize(r?.data())?; + match query_response_sender.send(response_record) { + Ok(()) => Ok(()), + Err(e) => fail!("query_response_sender: send error {}", e) + } + } + + fn poll_pending_queries(&self, max_queries: u32, query_timeout: SystemTime, max_answer_size: u64) -> Result<()> { + for _msg_count in 0..max_queries { + match self.pending_messages_queries_try_recv()? { + None => break, + Some((dst,TonNode_RempCatchainMessageHeaderV2(query))) => { + let query_id = query.message_id.clone(); + let query_payload = CatchainFactory::create_block_payload( + serialize_tl_boxed_object!(&TonNode_RempCatchainMessageHeaderV2(query)) + ); + let query_response_sender = self.get_instance_impl()?.get_query_response_sender(); + self.get_instance_impl()?.catchain_ptr.send_query_via_rldp( + dst.clone(), + "message body request".to_string(), + Box::new (move |r: Result| { + if let Err(e) = Self::deserialize_and_put_query_response_to_queue(r, query_response_sender) { + log::error!(target: "remp", "Cannot deserelialize response to query for message {:x}, error: {}", query_id, e) + } + }), + query_timeout, + query_payload, + max_answer_size + ) + }, + Some((_dst,TonNode_RempCatchainMessageDigestV2(d))) => { + log::error!(target: "remp", "Unexpected pending query {:?}, only MessageHeader record may be sent", d) + } + } + } + Ok(()) + } + + pub fn activate_exchange(&self, delay: Duration, max_broadcasts: u32, max_queries: u32, query_timeout: Duration, max_answer_size: u64) -> Result<()> { let session = &self.get_instance_impl()?.catchain_ptr; session.request_new_block(SystemTime::now() + delay); - self.poll_pending_broadcasts(max_broadcasts) + self.poll_pending_broadcasts(max_broadcasts)?; + self.poll_pending_queries(max_queries, SystemTime::now() + query_timeout, max_answer_size) } } From e53ea81fc6e84cac6837bee3ace356d8b9fecb3a Mon Sep 17 00:00:00 2001 From: Dmitry Shtukenberg Date: Tue, 12 Mar 2024 00:00:38 +0300 Subject: [PATCH 14/40] Fixed infinite loop bug --- src/validator/reliable_message_queue.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/validator/reliable_message_queue.rs b/src/validator/reliable_message_queue.rs index 3601b062..41087de3 100644 --- a/src/validator/reliable_message_queue.rs +++ b/src/validator/reliable_message_queue.rs @@ -538,6 +538,7 @@ impl MessageQueue { }, Ok(None) => { log::trace!(target: "remp", "RMQ {}: no more query responses", self); + break 'responses_loop; }, Ok(Some(rmq_record)) => { let rmq_message = Arc::new(RmqMessage::from_rmq_record(&rmq_record)?); From b0fa099d4aec342cd0e877ada0e84e52e96f373e Mon Sep 17 00:00:00 2001 From: Dmitry Shtukenberg Date: Tue, 12 Mar 2024 21:55:03 +0300 Subject: [PATCH 15/40] Fixed bug - ignoring message if have body --- src/validator/message_cache.rs | 27 ++++++++++++++++++++++++++- src/validator/remp_manager.rs | 9 ++++++--- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/src/validator/message_cache.rs b/src/validator/message_cache.rs index 85114cb0..5555ba8c 100644 --- a/src/validator/message_cache.rs +++ b/src/validator/message_cache.rs @@ -421,6 +421,23 @@ impl MessageCacheSession { self.message_headers.contains_key(msg_id) } + fn is_message_fully_known(&self, msg_id: &UInt256) -> Result { + if self.message_headers.contains_key(msg_id) && + self.message_origins.contains_key(msg_id) && + self.messages.contains_key(msg_id) + { + if self.message_status.contains_key(msg_id) { + return Ok(true); + } + else { + fail!("Inconsistent message info in cache: {}", self.message_info(msg_id)) + } + } + else { + return Ok(false); + } + } + fn starts_before_block(&self, blk: &BlockIdExt) -> bool { for inf in &self.inf_shards { if inf.shard().intersect_with(blk.shard()) { @@ -640,6 +657,7 @@ impl MessageCache { None } + #[allow(dead_code)] pub fn get_message(&self, message_id: &UInt256) -> Result>> { let msg = self.get_session_for_message(message_id).map( |session| session.messages.get(message_id).map( @@ -698,13 +716,20 @@ impl MessageCache { } } - fn get_message_info(&self, message_id: &UInt256) -> Result { + pub fn get_message_info(&self, message_id: &UInt256) -> Result { match self.get_session_for_message(message_id) { None => Ok("absent".to_string()), Some(s) => Ok(s.message_info(message_id)) } } + pub fn is_message_fully_known(&self, message_id: &UInt256) -> Result { + match self.get_session_for_message(message_id) { + None => Ok(false), + Some(s) => s.is_message_fully_known(message_id) + } + } + fn insert_message(&self, session: Arc, message: Arc, message_header: Arc, message_origin: Option>, status: &RempMessageStatus) -> Result<()> { if message.message_id != message_header.message_id { fail!("Inconsistent message: message {} and message_header {} have different message_id", message, message_header) diff --git a/src/validator/remp_manager.rs b/src/validator/remp_manager.rs index 62240abd..3ad5c8be 100644 --- a/src/validator/remp_manager.rs +++ b/src/validator/remp_manager.rs @@ -647,14 +647,17 @@ impl RempCoreInterface for RempInterfaceQueues { 0 )?; - if self.message_cache.get_message(&message_id)?.is_some() { + if self.message_cache.is_message_fully_known(&message_id)? { log::trace!(target: "remp", - "Point 1. We already know about message {:x}, no forwarding to incoming queue is necessary", + "Point 1. Message {:x} is fully known, no forwarding to incoming queue is necessary", message_id ); } else { - log::trace!(target: "remp", "Point 1. Adding incoming message {} to incoming queue", remp_message); + log::trace!(target: "remp", + "Point 1. Adding incoming message {} to incoming queue, known message info {}", + remp_message, self.message_cache.get_message_info(&message_id)? + ); self.incoming_sender.send(Arc::new(RempMessageWithOrigin { message: remp_message, origin: remp_message_origin }))?; #[cfg(feature = "telemetry")] self.engine.remp_core_telemetry().in_channel_from_fullnode(self.incoming_sender.len()); From 6ab00dcb1b4f01bfe92243516e0136500f2a2abc Mon Sep 17 00:00:00 2001 From: Dmitry Shtukenberg Date: Thu, 14 Mar 2024 14:07:54 +0300 Subject: [PATCH 16/40] Fixed query incorrect reply bug --- src/validator/message_cache.rs | 1 - src/validator/remp_catchain.rs | 34 +++++++++++++++++++++++++++++----- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/src/validator/message_cache.rs b/src/validator/message_cache.rs index 5555ba8c..0844e3e0 100644 --- a/src/validator/message_cache.rs +++ b/src/validator/message_cache.rs @@ -657,7 +657,6 @@ impl MessageCache { None } - #[allow(dead_code)] pub fn get_message(&self, message_id: &UInt256) -> Result>> { let msg = self.get_session_for_message(message_id).map( |session| session.messages.get(message_id).map( diff --git a/src/validator/remp_catchain.rs b/src/validator/remp_catchain.rs index ece8f005..64474c58 100644 --- a/src/validator/remp_catchain.rs +++ b/src/validator/remp_catchain.rs @@ -548,13 +548,27 @@ impl RempCatchain { } } - fn unpack_broadcast(&self, payload: &BlockPayloadPtr) -> Result { + fn unpack_broadcast(&self, payload: BlockPayloadPtr) -> Result { let message = RmqMessage::deserialize(payload.data())?; let rmqrecord = Arc::new(RmqMessage::from_rmq_record(&message)?); self.remp_manager.message_cache.update_message_body(rmqrecord.clone())?; Ok(rmqrecord.message_id.clone()) } + + fn process_query(&self, query_payload: BlockPayloadPtr) -> Result { + let query = RempMessageHeader::deserialize(query_payload.data())?; + let message_id = query.message_id().ok_or_else(|| error!("No message id in query"))?; + if let Some(message_body) = self.remp_manager.message_cache.get_message(message_id)? { + let rmqrecord = message_body.as_rmq_record(); + let response_payload = CatchainFactory::create_block_payload(serialize_tl_boxed_object!(&rmqrecord)); + + Ok(response_payload) + } + else { + fail!("Message body {:x} is not present in message cache", message_id) + } + } } impl fmt::Display for RempCatchain { @@ -631,15 +645,25 @@ impl CatchainListener for RempCatchain { fn process_broadcast(&self, source_id: PublicKeyHash, data: BlockPayloadPtr) { let len = data.data().len(); log::trace!(target: "remp", "MessageQueue {} process broadcast from {}, len {}", self, source_id, len); - match self.unpack_broadcast(&data) { + match self.unpack_broadcast(data) { Err(e) => log::error!(target: "remp", "MessageQueue {}, error processing broadcast from {}, message body will be ignored: `{}`", self, source_id, e), Ok(id) => log::trace!(target: "remp", "MessageQueue {} broadcast message: id {:x}, len {}", self, id, len) } } - fn process_query(&self, source_id: PublicKeyHash, data: BlockPayloadPtr, _callback: ExternalQueryResponseCallback) { - let data = data.data(); - log::trace!(target: "remp", "Processing RMQ {} Query {:?} from {}", self, data.0.as_slice(), source_id); + fn process_query(&self, source_id: PublicKeyHash, data: BlockPayloadPtr, callback: ExternalQueryResponseCallback) { + let raw_data = data.data(); + log::trace!(target: "remp", "Processing RMQ {} Query {:?} from {}", self, raw_data.0.as_slice(), source_id); + + match self.process_query(data) { + Err(e) => { + log::error!(target: "remp", "MessageQueue {}, error processing query from {}, message query will be ignored: `{}`", self, source_id, e); + callback(Err(error!("RMQ {}: error processing query from {}: {}", self, source_id, e))) + }, + Ok(body) => { + callback(Ok(body)) + } + } } fn set_time(&self, _timestamp: SystemTime) { From 95e5ec3d2f7a602654d43bd6f74453911cce4d9a Mon Sep 17 00:00:00 2001 From: Dmitry Shtukenberg Date: Thu, 14 Mar 2024 22:12:24 +0300 Subject: [PATCH 17/40] Changed origin adnl_id to remp catchain source adnl_id --- src/validator/reliable_message_queue.rs | 94 ++++++++++++++++++------- src/validator/remp_catchain.rs | 22 ++++-- 2 files changed, 84 insertions(+), 32 deletions(-) diff --git a/src/validator/reliable_message_queue.rs b/src/validator/reliable_message_queue.rs index 41087de3..8f887f3a 100644 --- a/src/validator/reliable_message_queue.rs +++ b/src/validator/reliable_message_queue.rs @@ -29,7 +29,7 @@ use ton_api::ton::ton_node::{ rempcatchainrecordv2::{RempCatchainMessageHeaderV2, RempCatchainMessageDigestV2} }; use ton_block::{ShardIdent, Message, BlockIdExt, ValidatorDescr}; -use ton_types::{UInt256, Result, fail}; +use ton_types::{UInt256, Result, fail, gen_random_index, error}; use catchain::{PrivateKey, PublicKey}; use crate::{ engine_traits::EngineOperations, @@ -65,6 +65,9 @@ struct MessageQueueImpl { /// Messages, waiting for collator invocation pending_collation_order: BinaryHeap<(Reverse, UInt256)>, + + /// Body sources for the message (actual if current node didn't receive body yet) + body_sources: HashMap> } pub struct MessageQueue { @@ -76,22 +79,38 @@ pub struct MessageQueue { } impl MessageQueueImpl { - pub fn add_to_collation_queue(&mut self, message_id: &UInt256, timestamp: u32, add_if_absent: bool) -> Result<(bool,usize)> { - match self.pending_collation_set.get_mut(message_id) { - Some(waiting_collation) if *waiting_collation => Ok((false, self.pending_collation_set.len())), + fn insert_body_source(&mut self, message_id: &UInt256, body_source: u32) { + match self.body_sources.remove(message_id) { + None => self.body_sources.insert(message_id.clone(), vec!(body_source)), + Some(mut vec) => { + if !vec.contains(&body_source) { + vec.push(body_source); + } + self.body_sources.insert(message_id.clone(), vec) + } + }; + } + + pub fn add_to_collation_queue(&mut self, message_id: &UInt256, timestamp: u32, remp_node_sender: Option, add_if_absent: bool) -> Result<(bool,usize)> { + let added = match self.pending_collation_set.get_mut(message_id) { + Some(waiting_collation) if *waiting_collation => false, Some(waiting_collation) => { self.pending_collation_order.push((Reverse(timestamp), message_id.clone())); *waiting_collation = true; - Ok((true, self.pending_collation_set.len())) + true }, None if add_if_absent => { self.pending_collation_set.insert(message_id.clone(), true); self.pending_collation_order.push((Reverse(timestamp), message_id.clone())); // Max heap --- need earliest message - Ok((true, self.pending_collation_set.len())) + true } None => fail!("Message {} is not present in collation set, cannot return it to collation queue", message_id) + }; + if let Some(ns) = remp_node_sender { + self.insert_body_source(message_id, ns); } + Ok((added, self.pending_collation_set.len())) } pub fn take_first_for_collation(&mut self) -> Result> { @@ -107,6 +126,10 @@ impl MessageQueueImpl { } } + pub fn get_body_sources(&self, message_id: &UInt256) -> Option<&Vec> { + self.body_sources.get(message_id) + } + pub fn list_pending_for_forwarding(&mut self) -> Result> { return Ok(self.pending_collation_set.keys().cloned().collect()) } @@ -125,6 +148,7 @@ impl MessageQueue { status: MessageQueueStatus::Created, pending_collation_set: HashMap::new(), pending_collation_order: BinaryHeap::new(), + body_sources: HashMap::new(), }, format!("<>", remp_catchain_instance), #[cfg(feature = "telemetry")] @@ -290,10 +314,10 @@ impl MessageQueue { } } - async fn add_pending_collation(&self, message_id: &UInt256, remp_message_origin: Arc, status_to_send: Option) -> Result<()> { + async fn add_pending_collation(&self, message_id: &UInt256, remp_message_origin: Arc, remp_node_sender: u32, status_to_send: Option) -> Result<()> { let (added_to_queue, _len) = self.queues.execute_sync( |catchain| catchain.add_to_collation_queue( - message_id, remp_message_origin.timestamp, true + message_id, remp_message_origin.timestamp, Some(remp_node_sender), true ) ).await?; @@ -365,15 +389,15 @@ impl MessageQueue { self.catchain_instance.is_session_active() } - async fn process_pending_remp_catchain_message(&self, catchain_record: &RempCatchainMessageHeaderV2) -> Result<()> { + async fn process_pending_remp_catchain_message(&self, catchain_record: &RempCatchainMessageHeaderV2, remp_node_sender: u32) -> Result<()> { let remp_message_header = Arc::new(RempMessageHeader::from_remp_catchain(catchain_record)?); let remp_message_origin = Arc::new(RempMessageOrigin::from_remp_catchain(catchain_record)?); let message_master_seqno = catchain_record.masterchain_seqno as u32; let forwarded = self.catchain_info.get_master_cc_seqno() > message_master_seqno; log::trace!(target: "remp", - "Point 4. RMQ {}: inserting pending message {}, {} from RMQ into message_cache, forwarded {}, message_master_cc {}", - self, remp_message_header, remp_message_origin, forwarded, message_master_seqno + "Point 4. RMQ {}: inserting pending message {}, {} from RMQ node {} into message_cache, forwarded {}, message_master_cc {}", + self, remp_message_header, remp_node_sender, remp_message_origin, forwarded, message_master_seqno ); let status_if_new = if let Some((_msg, status)) = self.is_queue_overloaded().await { @@ -453,7 +477,7 @@ impl MessageQueue { Some(x) => format!(" (old status {})", x) } ); - self.add_pending_collation(&remp_message_header.message_id, remp_message_origin, Some(new_status)).await?; + self.add_pending_collation(&remp_message_header.message_id, remp_message_origin, remp_node_sender, Some(new_status)).await?; #[cfg(feature = "telemetry")] self.engine.remp_core_telemetry().add_to_cache_attempt(true); } @@ -490,10 +514,10 @@ impl MessageQueue { Ok(()) } - async fn process_pending_remp_catchain_record(&self, remp_catchain_record: &RempCatchainRecordV2) -> Result<()> { + async fn process_pending_remp_catchain_record(&self, remp_catchain_record: &RempCatchainRecordV2, relayer: u32) -> Result<()> { match remp_catchain_record { RempCatchainRecordV2::TonNode_RempCatchainMessageHeaderV2(msg) => - self.process_pending_remp_catchain_message(msg).await, + self.process_pending_remp_catchain_message(msg,relayer).await, RempCatchainRecordV2::TonNode_RempCatchainMessageDigestV2(digest) => self.process_pending_remp_catchain_digest(digest).await } @@ -525,7 +549,7 @@ impl MessageQueue { log::trace!(target: "remp", "RMQ {}: No more messages in rmq_queue", self); break 'queue_loop; }, - Ok(Some(record)) => self.process_pending_remp_catchain_record(&record).await? + Ok(Some((record,relayer))) => self.process_pending_remp_catchain_record(&record,relayer).await? } } @@ -565,15 +589,35 @@ impl MessageQueue { continue } Ok(Some((None, h, o, s, cc))) => { - log::trace!( - target: "remp", - "Point 5. RMQ {}: message {:x} with status {:?} from {} found in pending_collation queue, but has no body yet; requesting body again", - self, msgid, s, o - ); - self.catchain_instance.pending_messages_queries_send( - o.source_key.clone(), - o.as_remp_catchain_record(&msgid, &h.message_uid, cc) - )?; + // -1 for absent index seems to be the easiest way to get around borrow/scope checker + let dst_idx = self.queues.execute_sync(|x| + match x.get_body_sources(&msgid) { + None => -1, + Some(v) if v.len() == 0 => -1, + Some(v) => v.get(gen_random_index(v.len() as u16) as usize).map(|nn| *nn as i32).unwrap_or(-1) + } + ).await; + + if dst_idx >= 0 { + log::trace!( + target: "remp", + "Point 5. RMQ {}: message {:x} with status {:?} from {} found in pending_collation queue, but has no body yet; requesting body again from node {}", + self, msgid, s, o, dst_idx + ); + + let dst_adnl_id = self.catchain_instance.get_adnl_id(dst_idx as usize).ok_or_else( + || error!("RMQ {}: cannot find adnl id for idx {}", self, dst_idx) + )?; + self.catchain_instance.pending_messages_queries_send( + dst_adnl_id, + o.as_remp_catchain_record(&msgid, &h.message_uid, cc) + )?; + } + else { + log::error!(target: "remp", "Point 5. RMQ {}: message {:x} with status {:?} from {} has no body and no body source; skipping", + self, msgid, s, o + ) + } continue }, Ok(None) => { @@ -685,7 +729,7 @@ impl MessageQueue { pub async fn return_to_collation_queue(&self, message_id: &UInt256) -> Result<()> { if let Some(origin) = self.remp_manager.message_cache.get_message_origin(message_id)? { self.queues.execute_sync( - |catchain| catchain.add_to_collation_queue(message_id, origin.timestamp, false) + |catchain| catchain.add_to_collation_queue(message_id, origin.timestamp, None, false) ).await?; Ok(()) } diff --git a/src/validator/remp_catchain.rs b/src/validator/remp_catchain.rs index 64474c58..b3930725 100644 --- a/src/validator/remp_catchain.rs +++ b/src/validator/remp_catchain.rs @@ -59,8 +59,8 @@ pub struct RempCatchainInstanceImpl { pending_messages_queue_receiver: crossbeam_channel::Receiver, pub pending_messages_queue_sender: crossbeam_channel::Sender, - pub rmq_catchain_receiver: crossbeam_channel::Receiver, - rmq_catchain_sender: crossbeam_channel::Sender, + pub rmq_catchain_receiver: crossbeam_channel::Receiver<(RempCatchainRecordV2,u32)>, + rmq_catchain_sender: crossbeam_channel::Sender<(RempCatchainRecordV2,u32)>, query_response_receiver: crossbeam_channel::Receiver, query_response_sender: Arc>, @@ -128,6 +128,10 @@ impl RempCatchainInstance { self.instance_impl.load().map(|inst| inst.catchain_ptr.clone()) } + pub fn get_adnl_id(&self, node_idx: usize) -> Option> { + self.info.get_adnl_id(node_idx) + } + pub fn pending_messages_queue_send(&self, msg: RempCatchainRecordV2) -> Result<()> { let instance = self.get_instance_impl()?; match instance.pending_messages_queue_sender.send(msg) { @@ -200,18 +204,18 @@ impl RempCatchainInstance { Ok(instance.rmq_catchain_receiver.len()) } - pub fn rmq_catchain_try_recv(&self) -> Result> { + pub fn rmq_catchain_try_recv(&self) -> Result> { let instance = self.get_instance_impl()?; match instance.rmq_catchain_receiver.try_recv() { - Ok(x) => Ok(Some(x)), + Ok((x,relayer)) => Ok(Some((x,relayer))), Err(crossbeam_channel::TryRecvError::Empty) => Ok(None), Err(crossbeam_channel::TryRecvError::Disconnected) => fail!("channel disconnected") } } - pub fn rmq_catchain_send(&self, msg: RempCatchainRecordV2) -> Result<()> { + pub fn rmq_catchain_send(&self, msg: RempCatchainRecordV2, msg_remp_source_idx: u32) -> Result<()> { let instance = self.get_instance_impl()?; - match instance.rmq_catchain_sender.send(msg) { + match instance.rmq_catchain_sender.send((msg, msg_remp_source_idx)) { Ok(()) => Ok(()), Err(e) => fail!("rmq_cathcain_sender: send error {}", e) } @@ -408,6 +412,10 @@ impl RempCatchainInfo { general_eq } + + pub fn get_adnl_id(&self, index: usize) -> Option> { + self.nodes.get(index).map(|n| n.adnl_id.clone()) + } } impl fmt::Display for RempCatchainInfo { @@ -521,7 +529,7 @@ impl RempCatchain { "Point 4. Message received from RMQ {}: {:?}, decoded {:?}, put to rmq_catchain queue", self, msg.signature.0, record //catchain.received_messages.len() ); - if let Err(e) = self.instance.rmq_catchain_send(record.clone()) { + if let Err(e) = self.instance.rmq_catchain_send(record.clone(), source_idx) { log::error!( target: "remp", "Point 4. Cannot put message {:?} from RMQ {} to queue: {}", record, self, e From 391c2c5111956300c9059afc1a2ed1baffe8fed2 Mon Sep 17 00:00:00 2001 From: Dmitry Shtukenberg Date: Fri, 15 Mar 2024 11:54:24 +0300 Subject: [PATCH 18/40] One more bugfix: adding local message body without broadcasts to local node --- src/validator/reliable_message_queue.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/validator/reliable_message_queue.rs b/src/validator/reliable_message_queue.rs index 8f887f3a..b5c3b3ed 100644 --- a/src/validator/reliable_message_queue.rs +++ b/src/validator/reliable_message_queue.rs @@ -289,6 +289,7 @@ impl MessageQueue { msg.as_remp_catchain_record(self.catchain_info.get_master_cc_seqno()), msg.as_rmq_record() )?; + self.remp_manager.message_cache.update_message_body(Arc::new(msg.message.clone()))?; #[cfg(feature = "telemetry")] self.engine.remp_core_telemetry().in_channel_to_catchain( From a2567aa2a846f6928e00105f27bc0bf877551a92 Mon Sep 17 00:00:00 2001 From: Dmitry Shtukenberg Date: Thu, 21 Mar 2024 15:14:38 +0300 Subject: [PATCH 19/40] Changed protocols, repaired tests --- src/validator/message_cache.rs | 50 ++++++++++++++++------ src/validator/reliable_message_queue.rs | 16 +++---- src/validator/remp_catchain.rs | 52 +++++++++++------------ src/validator/tests/test_message_cache.rs | 33 ++++++++------ 4 files changed, 90 insertions(+), 61 deletions(-) diff --git a/src/validator/message_cache.rs b/src/validator/message_cache.rs index 0844e3e0..b20d71a7 100644 --- a/src/validator/message_cache.rs +++ b/src/validator/message_cache.rs @@ -48,10 +48,7 @@ use ton_api::{ } }; -use ton_block::{ - Deserializable, Message, Serializable, MsgAddressInt, MsgAddrStd, - ExternalInboundMessageHeader, BlockIdExt, UnixTime32 -}; +use ton_block::{Deserializable, Message, Serializable, MsgAddressInt, MsgAddrStd, ExternalInboundMessageHeader, BlockIdExt, UnixTime32, GetRepresentationHash}; use ton_types::{error, fail, KeyId, SliceData, Result, UInt256}; @@ -70,7 +67,7 @@ impl RmqMessage { pub fn new(message: Arc, message_id: UInt256, message_uid: UInt256) -> Result { return Ok(RmqMessage { message, message_id, message_uid }) } - +/* pub fn as_rmq_record(&self) -> ton_api::ton::ton_node::RmqRecord { ton_api::ton::ton_node::rmqrecord::RmqMessage { message: self.message.write_to_bytes().unwrap().into(), @@ -80,12 +77,18 @@ impl RmqMessage { masterchain_seqno: 0 }.into_boxed() } - - pub fn deserialize(raw: &ton_api::ton::bytes) -> Result { - let rmq_record: ton_api::ton::ton_node::RmqRecord = catchain::utils::deserialize_tl_boxed_object(&raw)?; - Ok(rmq_record) +*/ + pub fn as_remp_message_body(&self) -> ton_api::ton::ton_node::RempMessageBody { + ton_api::ton::ton_node::rempmessagebody::RempMessageBody { + message: self.message.write_to_bytes().unwrap().into() + }.into_boxed() } + pub fn deserialize_message_body(raw: &ton_api::ton::bytes) -> Result { + let message_body: ton_api::ton::ton_node::RempMessageBody = catchain::utils::deserialize_tl_boxed_object(&raw)?; + Ok(message_body) + } +/* pub fn from_rmq_record(record: &ton_api::ton::ton_node::RmqRecord) -> Result { match record { ton_api::ton::ton_node::RmqRecord::TonNode_RmqMessage(rec) => { @@ -99,11 +102,18 @@ impl RmqMessage { _ => fail!("message_cache::from_rmq_record: not an RmqMessage is given: {:?}", record) } } +*/ + pub fn from_message_body(body: &ton_api::ton::ton_node::RempMessageBody) -> Result { + let message = Arc::new(Message::construct_from_bytes(body.message())?); + let message_id = message.hash()?; + let message_uid = get_message_uid(&message); + Self::new (message, message_id, message_uid) + } #[allow(dead_code)] pub fn make_test_message(body: &SliceData) -> Result { let address = UInt256::rand(); - let msg = ton_block::Message::with_ext_in_header_and_body(ExternalInboundMessageHeader { + let msg = Message::with_ext_in_header_and_body(ExternalInboundMessageHeader { src: Default::default(), dst: MsgAddressInt::AddrStd(MsgAddrStd { anycast: None, @@ -194,6 +204,22 @@ impl RempMessageHeader { pub fn from_remp_catchain(record: &RempCatchainMessageHeaderV2) -> Result { Ok(Self::new(&record.message_id, &record.message_uid)) } + + pub fn serialize_query(query: &ton_api::ton::ton_node::RempMessageQuery) -> Result { + let query_serialized = serialize_tl_boxed_object!(query); + return Ok(query_serialized) + } + + pub fn deserialize_query(raw: &ton_api::ton::bytes) -> Result { + let query = catchain::utils::deserialize_tl_boxed_object(raw)?; + Ok(query) + } + + pub fn as_remp_message_query(&self) -> ton_api::ton::ton_node::RempMessageQuery { + ton_api::ton::ton_node::rempmessagequery::RempMessageQuery { + message_id: Default::default(), + }.into_boxed() + } } impl Display for RempMessageHeader { @@ -296,8 +322,8 @@ impl RempMessageWithOrigin { return self.origin.as_remp_catchain_record(&self.message.message_id, &self.message.message_uid, master_cc) } - pub fn as_rmq_record(&self) -> ton_api::ton::ton_node::RmqRecord { - return self.message.as_rmq_record() + pub fn as_remp_message_body(&self) -> ton_api::ton::ton_node::RempMessageBody { + return self.message.as_remp_message_body() } } diff --git a/src/validator/reliable_message_queue.rs b/src/validator/reliable_message_queue.rs index b5c3b3ed..385ba0ee 100644 --- a/src/validator/reliable_message_queue.rs +++ b/src/validator/reliable_message_queue.rs @@ -287,7 +287,7 @@ impl MessageQueue { log::trace!(target: "remp", "Point 3. Pushing to RMQ {}; message {}, {} + broadcast", self, msg, origin_with_idx); self.catchain_instance.pending_messages_broadcast_send( msg.as_remp_catchain_record(self.catchain_info.get_master_cc_seqno()), - msg.as_rmq_record() + msg.as_remp_message_body() )?; self.remp_manager.message_cache.update_message_body(Arc::new(msg.message.clone()))?; @@ -565,8 +565,8 @@ impl MessageQueue { log::trace!(target: "remp", "RMQ {}: no more query responses", self); break 'responses_loop; }, - Ok(Some(rmq_record)) => { - let rmq_message = Arc::new(RmqMessage::from_rmq_record(&rmq_record)?); + Ok(Some(message_body)) => { + let rmq_message = Arc::new(RmqMessage::from_message_body(&message_body)?); self.remp_manager.message_cache.update_message_body(rmq_message.clone())?; } } @@ -589,7 +589,7 @@ impl MessageQueue { ); continue } - Ok(Some((None, h, o, s, cc))) => { + Ok(Some((None, h, o, s, _cc))) => { // -1 for absent index seems to be the easiest way to get around borrow/scope checker let dst_idx = self.queues.execute_sync(|x| match x.get_body_sources(&msgid) { @@ -611,7 +611,7 @@ impl MessageQueue { )?; self.catchain_instance.pending_messages_queries_send( dst_adnl_id, - o.as_remp_catchain_record(&msgid, &h.message_uid, cc) + h.as_remp_message_query() )?; } else { @@ -1081,8 +1081,8 @@ impl RmqQueueManager { }; // Forwarding: - // 1. Rejected messages -- as plain id (message id, message uid) - // 2. Non-final messages -- (full RmqRecord) + // 1. Rejected messages -- MessageDigestV2 (id, uid) + // 2. Non-final messages -- Message (id, uid, origin) + MessageBody as broadcast // All other messages are not forwarded if is_finally_rejected(&message_status) { @@ -1122,7 +1122,7 @@ impl RmqQueueManager { for new in next_queues.iter() { if let Err(x) = new.catchain_instance.pending_messages_broadcast_send( origin.as_remp_catchain_record(&message.message_id, &message.message_uid, message_cc), - message.as_rmq_record() + message.as_remp_message_body() ) { log::error!(target: "remp", diff --git a/src/validator/remp_catchain.rs b/src/validator/remp_catchain.rs index 0040e6e2..08f0304b 100644 --- a/src/validator/remp_catchain.rs +++ b/src/validator/remp_catchain.rs @@ -37,8 +37,7 @@ use catchain::{ use ton_api::{ IntoBoxed, ton::ton_node::RempCatchainRecordV2 }; -use ton_api::ton::ton_node::RempCatchainRecordV2::{TonNode_RempCatchainMessageDigestV2, TonNode_RempCatchainMessageHeaderV2}; -use ton_api::ton::ton_node::RmqRecord; +use ton_api::ton::ton_node::RempMessageQuery; use ton_block::ValidatorDescr; use ton_types::{error, fail, KeyId, Result, UInt256}; @@ -62,14 +61,14 @@ pub struct RempCatchainInstanceImpl { pub rmq_catchain_receiver: crossbeam_channel::Receiver<(RempCatchainRecordV2,u32)>, rmq_catchain_sender: crossbeam_channel::Sender<(RempCatchainRecordV2,u32)>, - query_response_receiver: crossbeam_channel::Receiver, - query_response_sender: Arc>, + query_response_receiver: crossbeam_channel::Receiver, + query_response_sender: Arc>, - pending_messages_queries_receiver: crossbeam_channel::Receiver<(Arc,RempCatchainRecordV2)>, - pending_messages_queries_sender: crossbeam_channel::Sender<(Arc,RempCatchainRecordV2)>, + pending_messages_queries_receiver: crossbeam_channel::Receiver<(Arc,RempMessageQuery)>, + pending_messages_queries_sender: crossbeam_channel::Sender<(Arc,RempMessageQuery)>, - pub pending_messages_broadcast_receiver: crossbeam_channel::Receiver, - pending_messages_broadcast_sender: crossbeam_channel::Sender, + pub pending_messages_broadcast_receiver: crossbeam_channel::Receiver, + pending_messages_broadcast_sender: crossbeam_channel::Sender, } impl RempCatchainInstanceImpl { @@ -92,7 +91,7 @@ impl RempCatchainInstanceImpl { } } - fn get_query_response_sender(&self) -> Arc> { + fn get_query_response_sender(&self) -> Arc> { self.query_response_sender.clone() } } @@ -140,7 +139,7 @@ impl RempCatchainInstance { } } - pub fn pending_messages_broadcast_send(&self, msg: RempCatchainRecordV2, msg_body: RmqRecord) -> Result<()> { + pub fn pending_messages_broadcast_send(&self, msg: RempCatchainRecordV2, msg_body: ton_api::ton::ton_node::RempMessageBody) -> Result<()> { let instance = self.get_instance_impl()?; match (instance.pending_messages_queue_sender.send(msg), instance.pending_messages_broadcast_sender.send(msg_body)) { (Ok(()), Ok(())) => Ok(()), @@ -164,7 +163,7 @@ impl RempCatchainInstance { } } - pub fn pending_messages_broadcast_try_recv(&self) -> Result> { + pub fn pending_messages_broadcast_try_recv(&self) -> Result> { let instance = self.get_instance_impl()?; match instance.pending_messages_broadcast_receiver.try_recv() { Ok(x) => Ok(Some(x)), @@ -173,7 +172,7 @@ impl RempCatchainInstance { } } - pub fn pending_messages_queries_send(&self, dst: Arc, query: RempCatchainRecordV2) -> Result<()> { + pub fn pending_messages_queries_send(&self, dst: Arc, query: RempMessageQuery) -> Result<()> { let instance = self.get_instance_impl()?; match instance.pending_messages_queries_sender.send((dst, query)) { Ok(()) => Ok(()), @@ -181,7 +180,7 @@ impl RempCatchainInstance { } } - pub fn pending_messages_queries_try_recv(&self) -> Result,RempCatchainRecordV2)>> { + pub fn pending_messages_queries_try_recv(&self) -> Result,RempMessageQuery)>> { let instance = self.get_instance_impl()?; match instance.pending_messages_queries_receiver.try_recv() { Ok(x) => Ok(Some(x)), @@ -190,7 +189,7 @@ impl RempCatchainInstance { } } - pub fn query_response_receiver_try_recv(&self) -> Result> { + pub fn query_response_receiver_try_recv(&self) -> Result> { let instance = self.get_instance_impl()?; match instance.query_response_receiver.try_recv() { Ok(x) => Ok(Some(x)), @@ -240,8 +239,8 @@ impl RempCatchainInstance { Ok(()) } - fn deserialize_and_put_query_response_to_queue(r: Result, query_response_sender: Arc>) -> Result<()> { - let response_record = RmqMessage::deserialize(r?.data())?; + fn deserialize_and_put_query_response_to_queue(r: Result, query_response_sender: Arc>) -> Result<()> { + let response_record = RmqMessage::deserialize_message_body(r?.data())?; match query_response_sender.send(response_record) { Ok(()) => Ok(()), Err(e) => fail!("query_response_sender: send error {}", e) @@ -252,10 +251,10 @@ impl RempCatchainInstance { for _msg_count in 0..max_queries { match self.pending_messages_queries_try_recv()? { None => break, - Some((dst,TonNode_RempCatchainMessageHeaderV2(query))) => { - let query_id = query.message_id.clone(); + Some((dst,query)) => { + let query_id = query.message_id().clone(); let query_payload = CatchainFactory::create_block_payload( - serialize_tl_boxed_object!(&TonNode_RempCatchainMessageHeaderV2(query)) + RempMessageHeader::serialize_query(&query)? ); let query_response_sender = self.get_instance_impl()?.get_query_response_sender(); self.get_instance_impl()?.catchain_ptr.send_query_via_rldp( @@ -270,9 +269,6 @@ impl RempCatchainInstance { query_payload, max_answer_size ) - }, - Some((_dst,TonNode_RempCatchainMessageDigestV2(d))) => { - log::error!(target: "remp", "Unexpected pending query {:?}, only MessageHeader record may be sent", d) } } } @@ -557,19 +553,19 @@ impl RempCatchain { } fn unpack_broadcast(&self, payload: BlockPayloadPtr) -> Result { - let message = RmqMessage::deserialize(payload.data())?; - let rmqrecord = Arc::new(RmqMessage::from_rmq_record(&message)?); + let message = RmqMessage::deserialize_message_body(payload.data())?; + let rmqrecord = Arc::new(RmqMessage::from_message_body(&message)?); self.remp_manager.message_cache.update_message_body(rmqrecord.clone())?; Ok(rmqrecord.message_id.clone()) } fn process_query(&self, query_payload: BlockPayloadPtr) -> Result { - let query = RempMessageHeader::deserialize(query_payload.data())?; - let message_id = query.message_id().ok_or_else(|| error!("No message id in query"))?; + let query = RempMessageHeader::deserialize_query(query_payload.data())?; + let message_id = query.message_id(); if let Some(message_body) = self.remp_manager.message_cache.get_message(message_id)? { - let rmqrecord = message_body.as_rmq_record(); - let response_payload = CatchainFactory::create_block_payload(serialize_tl_boxed_object!(&rmqrecord)); + let remp_body_response = message_body.as_remp_message_body(); + let response_payload = CatchainFactory::create_block_payload(serialize_tl_boxed_object!(&remp_body_response)); Ok(response_payload) } diff --git a/src/validator/tests/test_message_cache.rs b/src/validator/tests/test_message_cache.rs index 099cc1ff..783ecf30 100644 --- a/src/validator/tests/test_message_cache.rs +++ b/src/validator/tests/test_message_cache.rs @@ -22,7 +22,7 @@ use ton_block::{BlockIdExt, ShardIdent}; use ton_types::{Result, SliceData, error, UInt256}; use crate::engine_traits::RempDuplicateStatus; use crate::ext_messages::get_level_and_level_change; -use crate::validator::message_cache::{MessageCache, RmqMessage}; +use crate::validator::message_cache::{MessageCache, RmqMessage, RempMessageOrigin}; use crate::validator::reliable_message_queue::MessageQueue; //use crate::test_helper::init_test_log; @@ -170,10 +170,11 @@ fn do_test_message_cache_add_remove(check_uids: bool) -> Result<()> { tb.cache.add_external_message_status( &msg.message_id, &msg.message_uid, Some(msg.clone()), + Some(Arc::new(RempMessageOrigin::create_empty()?)), RempMessageStatus::TonNode_RempNew, |_old,new| new.clone(), master_seq as u32 - ).await?; + )?; let mut should_be_duplicate = false; if check_uids { @@ -216,7 +217,7 @@ fn do_test_message_cache_add_remove(check_uids: bool) -> Result<()> { let lwb = tb.cache.update_master_cc_ranges(ms3, Duration::from_secs(1))?; tb.cache.gc_old_messages(*lwb.start()).await; assert_eq!(tb.cache.gc_old_messages(*lwb.start()).await.total, 0); - assert_eq!(tb.cache.all_messages_count(), 0); + assert_eq!(tb.cache.all_messages_count(), (0,0,0)); Ok(()) }) @@ -243,18 +244,20 @@ fn do_test_message_cache_uids_same_block() -> Result<()> { tb.cache.add_external_message_status( &msg2.message_id, &msg2.message_uid, Some(msg2.clone()), + Some(Arc::new(RempMessageOrigin::create_empty()?)), RempMessageStatus::TonNode_RempNew, |_old,new| new.clone(), seq - ).await?; + )?; tb.cache.add_external_message_status( &msg1.message_id, &msg1.message_uid, Some(msg1.clone()), + Some(Arc::new(RempMessageOrigin::create_empty()?)), RempMessageStatus::TonNode_RempNew, |_old,new| new.clone(), seq - ).await?; + )?; let dup1 = tb.cache.check_message_duplicates(&msg1.message_id)?; let dup2 = tb.cache.check_message_duplicates(&msg2.message_id)?; @@ -303,10 +306,12 @@ fn do_message_cache_simple_random_test() -> Result<()> { if !tb.simple_cache.uid_has_final_status(&msg.message_uid) { tb.cache.add_external_message_status( &msg.message_id, &msg.message_uid, - Some(msg.clone()), status.clone(), + Some(msg.clone()), + Some(Arc::new(RempMessageOrigin::create_empty()?)), + status.clone(), |_old,new| new.clone(), seq - ).await?; + )?; tb.simple_cache.add_external_message_status( &msg.message_id, &msg.message_uid, seq as i32, &status @@ -346,7 +351,7 @@ fn do_message_cache_simple_random_test() -> Result<()> { tb.cache.gc_old_messages(*lwb.start()).await; tb.simple_cache.remove_old_messages(max_seqs as i32 + 1); - assert_eq!(tb.cache.all_messages_count(), tb.simple_cache.all_messages_count()); + assert_eq!(tb.cache.all_messages_count().0, tb.simple_cache.all_messages_count()); Ok(()) }) } @@ -381,23 +386,25 @@ pub fn test_message_cache_cc_ranges_and_gc() -> Result<()> { tb.cache.add_external_message_status( &msg.message_id, &msg.message_uid, - Some(msg.clone()), RempMessageStatus::TonNode_RempNew, + Some(msg.clone()), + Some(Arc::new(RempMessageOrigin::create_empty()?)), + RempMessageStatus::TonNode_RempNew, |_old,new| new.clone(), 1 - ).await?; + )?; tb.cache.try_set_master_cc_start_time(2 as u32,2.into(), vec!())?; let range = tb.cache.update_master_cc_ranges(2, Duration::from_secs(1))?; - assert!(tb.cache.get_message_with_status_cc(&msg.message_id)?.is_some()); + assert!(tb.cache.get_message_with_origin_status_cc(&msg.message_id)?.is_some()); tb.cache.gc_old_messages(*range.start()).await; tb.cache.try_set_master_cc_start_time(3 as u32,3.into(), vec!())?; let range = tb.cache.update_master_cc_ranges(3, Duration::from_secs(1))?; - assert!(tb.cache.get_message_with_status_cc(&msg.message_id)?.is_some()); + assert!(tb.cache.get_message_with_origin_status_cc(&msg.message_id)?.is_some()); tb.cache.gc_old_messages(*range.start()).await; - assert!(tb.cache.get_message_with_status_cc(&msg.message_id)?.is_none()); + assert!(tb.cache.get_message_with_origin_status_cc(&msg.message_id)?.is_none()); Ok(()) }) From cf70ea166495077bda7fd20f7b5011a5783d87f0 Mon Sep 17 00:00:00 2001 From: Dmitry Shtukenberg Date: Tue, 26 Mar 2024 17:47:59 +0300 Subject: [PATCH 20/40] Advanced node version --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 52bd2cd2..76a884dd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ build = 'common/build/build.rs' edition = '2021' name = 'ton_node' -version = '0.56.5' +version = '0.56.7' [workspace] members = [ 'storage' ] From 2d72c175200c672eabee7829a66be44279d3b090 Mon Sep 17 00:00:00 2001 From: Dmitry Shtukenberg Date: Tue, 26 Mar 2024 18:54:34 +0300 Subject: [PATCH 21/40] Made remp config field nullable --- src/config.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/config.rs b/src/config.rs index 94a001a1..5c44b405 100644 --- a/src/config.rs +++ b/src/config.rs @@ -164,8 +164,7 @@ pub struct TonNodeConfig { port: Option, #[serde(skip)] file_name: String, - #[serde(default = "RempConfig::default")] - remp: RempConfig, + remp: Option, #[serde(default)] restore_db: bool, #[serde(default)] @@ -590,8 +589,11 @@ impl TonNodeConfig { pub fn extensions(&self) -> &NodeExtensions { &self.extensions } - pub fn remp_config(&self) -> &RempConfig { - &self.remp + pub fn remp_config(&self) -> RempConfig { + match &self.remp { + Some(x) => x.clone(), + None => RempConfig::default() + } } pub fn restore_db(&self) -> bool { self.restore_db From fb7b81d3c435a5d3bf54e2c095f3299959bdb701 Mon Sep 17 00:00:00 2001 From: Dmitry Shtukenberg Date: Wed, 27 Mar 2024 00:48:47 +0300 Subject: [PATCH 22/40] Added broadcast skipping if message is already known before Point 0 --- src/validator/message_cache.rs | 64 ++++++++++++++++--------- src/validator/reliable_message_queue.rs | 21 +++++--- src/validator/remp_catchain.rs | 16 +++++-- 3 files changed, 67 insertions(+), 34 deletions(-) diff --git a/src/validator/message_cache.rs b/src/validator/message_cache.rs index b20d71a7..b883d7aa 100644 --- a/src/validator/message_cache.rs +++ b/src/validator/message_cache.rs @@ -393,22 +393,24 @@ impl MessageCacheSession { Ok(()) } - fn insert_message(&self, msg: Arc, msg_hdr: Arc, msg_origin: Option>) -> Result<()> { + fn insert_message(&self, msg: Arc, msg_hdr: Arc, msg_origin: Option>) -> Result { if msg.message_uid != msg_hdr.message_uid || msg.message_id != msg_hdr.message_id { fail!("Message with id {:x} and uid {:x} and its header {} have different uids or ids", msg.message_id, msg.message_uid, msg_hdr); } self.insert_message_header(&msg.message_id, msg_hdr, msg_origin)?; match self.messages.insert(msg.message_id.clone(), msg.clone()) { - None => Ok(()), - Some(prev) if prev == msg => Ok(()), + None => Ok(true), + Some(prev) if prev == msg => Ok(false), Some(p) => fail!("Different messages for same id {:x}, replacing {} with {}", msg.message_id, p, msg ) } } - fn update_missing_fields(&self, message_id: &UInt256, message: Option>, origin: Option>) -> Result<()> { + /// Returns body_updated flag + fn update_missing_fields(&self, message_id: &UInt256, message: Option>, origin: Option>) -> Result { + let mut body_updated = false; if let Some(m1) = &message { if let Some(m2) = self.messages.get(message_id) { if m1 != m2.value() { @@ -420,9 +422,16 @@ impl MessageCacheSession { if m1 != &m2 { fail!("Parallel updating of message body: new body '{}', another body '{}'", m1, m2) } + log::warn!(target: "remp", + "Update_missing_fields: body for message {:x} inserted as new, although it is present already in cache, considering non-updated", + message_id + ) + } + else { + body_updated = true } } - } + }; // If origin is different from the stored one, that's not a big deal. if let Some(o1) = &origin { @@ -434,13 +443,13 @@ impl MessageCacheSession { else { if let Some(o2) = self.message_origins.insert(message_id.clone(), o1.clone()) { if o1 != &o2 { - fail!("Different message origin: new origin '{}', current cached origin '{}'", o1, o2); + log::trace!("Different message origin: new origin '{}', current cached origin '{}'", o1, o2); } } } } - Ok(()) + Ok(body_updated) } fn is_message_present(&self, msg_id: &UInt256) -> bool { @@ -755,7 +764,7 @@ impl MessageCache { } } - fn insert_message(&self, session: Arc, message: Arc, message_header: Arc, message_origin: Option>, status: &RempMessageStatus) -> Result<()> { + fn insert_message(&self, session: Arc, message: Arc, message_header: Arc, message_origin: Option>, status: &RempMessageStatus) -> Result { if message.message_id != message_header.message_id { fail!("Inconsistent message: message {} and message_header {} have different message_id", message, message_header) } @@ -767,8 +776,7 @@ impl MessageCache { } session.message_status.insert(message_id.clone(), status.clone()); - session.insert_message(message, message_header, message_origin)?; - Ok(()) + session.insert_message(message, message_header, message_origin) } fn insert_message_header(&self, session: Arc, message_header: Arc, message_origin: Option>, status: &RempMessageStatus) -> Result<()> { @@ -786,12 +794,12 @@ impl MessageCache { /// If we know something about message -- that's more important than anything we discover from RMQ /// If we do not know anything -- TODO: if all reject, then 'Rejected'. Otherwise 'New' /// Actual -- get it as granted ("imprinting") - /// Returns old status and new (added) status + /// Returns old status, new (added) status, and body_updated flag pub fn add_external_message_status(&self, message_id: &UInt256, message_uid: &UInt256, message: Option>, message_origin: Option>, status_if_new: RempMessageStatus, status_updater: F, master_cc: u32 - ) -> Result<(Option,RempMessageStatus)> + ) -> Result<(Option,RempMessageStatus,bool)> where F: FnOnce(&RempMessageStatus, &RempMessageStatus) -> RempMessageStatus { match self.get_session_for_message(message_id) { @@ -807,27 +815,37 @@ impl MessageCache { message_uid ); - match message { - None => - self.insert_message_header( session, header, message_origin, &status_if_new)?, + let body_updated = match message { + None => { + self.insert_message_header( session, header, message_origin, &status_if_new)?; + false + }, Some(message) => self.insert_message(session, message, header, message_origin.clone(), &status_if_new)? }; - Ok((None, status_if_new)) + Ok((None, status_if_new, body_updated)) }, Some(session) => { - if let Err(e) = session.update_missing_fields(&message_id, message, message_origin) { - log::error!(target: "remp", "Different cache contents for external message {}: {}", message_id, e); - } + let body_updated = session + .update_missing_fields(&message_id, message, message_origin) + .unwrap_or_else(|e| { + log::error!(target: "remp", "Different cache contents for external message {}: {}", message_id, e); + false + }); + let (old_status, final_status) = session.alter_message_status(&message_id, |old| status_updater(old,&status_if_new))?; - Ok((Some(old_status), final_status)) + + Ok((Some(old_status), final_status, body_updated)) }, } } - pub fn update_message_body(&self, data: Arc) -> Result<()> { - let (old_status,new_status) = self.add_external_message_status( + /// Updates message body for the message + /// `data` --- Message body with ids + /// `return` --- Whether the body added or not + pub fn update_message_body(&self, data: Arc) -> Result { + let (old_status, new_status, body_updated) = self.add_external_message_status( &data.message_id, &data.message_uid, Some(data.clone()), @@ -842,7 +860,7 @@ impl MessageCache { new_status, info ); - Ok(()) + Ok(body_updated) } /// Checks whether message msg_id is accepted by collator; diff --git a/src/validator/reliable_message_queue.rs b/src/validator/reliable_message_queue.rs index 385ba0ee..eb2057a7 100644 --- a/src/validator/reliable_message_queue.rs +++ b/src/validator/reliable_message_queue.rs @@ -284,12 +284,21 @@ impl MessageQueue { } let origin_with_idx = Arc::new(msg.origin.new_with_updated_source_idx(self.catchain_info.local_idx as u32)); - log::trace!(target: "remp", "Point 3. Pushing to RMQ {}; message {}, {} + broadcast", self, msg, origin_with_idx); + let body_updated = self.remp_manager.message_cache.update_message_body(Arc::new(msg.message.clone()))?; + log::trace!(target: "remp", "Point 3. Pushing to RMQ {}; message {}, {}{}", + self, msg, origin_with_idx, + (if body_updated { " + broadcast" } else { "" }).to_string() + ); + + let msg_body = if body_updated { + Some(msg.as_remp_message_body()) + } + else { None }; + self.catchain_instance.pending_messages_broadcast_send( msg.as_remp_catchain_record(self.catchain_info.get_master_cc_seqno()), - msg.as_remp_message_body() + msg_body )?; - self.remp_manager.message_cache.update_message_body(Arc::new(msg.message.clone()))?; #[cfg(feature = "telemetry")] self.engine.remp_core_telemetry().in_channel_to_catchain( @@ -461,7 +470,7 @@ impl MessageQueue { self, remp_message_header, e ); }, - Ok((Some(_),new_status)) if Self::is_final_status(&new_status) => { + Ok((Some(_),new_status,_body_updated)) if Self::is_final_status(&new_status) => { log::trace!(target: "remp", "Point 4. RMQ {}. Message {:x} master_cc_seqno {} from validator {} has final status {}, skipping", self, remp_message_header.message_id, message_master_seqno, remp_message_origin.source_idx, new_status @@ -469,7 +478,7 @@ impl MessageQueue { #[cfg(feature = "telemetry")] self.engine.remp_core_telemetry().add_to_cache_attempt(false); } - Ok((old_status,new_status)) => { + Ok((old_status,new_status,_body_updated)) => { log::trace!(target: "remp", "Point 4. RMQ {}. Message {:x} master_cc_seqno {} from validator {} has non-final status {}{}, will be collated", self, remp_message_header.message_id, message_master_seqno, remp_message_origin.source_idx, new_status, @@ -1122,7 +1131,7 @@ impl RmqQueueManager { for new in next_queues.iter() { if let Err(x) = new.catchain_instance.pending_messages_broadcast_send( origin.as_remp_catchain_record(&message.message_id, &message.message_uid, message_cc), - message.as_remp_message_body() + Some(message.as_remp_message_body()) ) { log::error!(target: "remp", diff --git a/src/validator/remp_catchain.rs b/src/validator/remp_catchain.rs index 08f0304b..a2533c45 100644 --- a/src/validator/remp_catchain.rs +++ b/src/validator/remp_catchain.rs @@ -139,13 +139,19 @@ impl RempCatchainInstance { } } - pub fn pending_messages_broadcast_send(&self, msg: RempCatchainRecordV2, msg_body: ton_api::ton::ton_node::RempMessageBody) -> Result<()> { + pub fn pending_messages_broadcast_send(&self, msg: RempCatchainRecordV2, msg_body: Option) -> Result<()> { let instance = self.get_instance_impl()?; - match (instance.pending_messages_queue_sender.send(msg), instance.pending_messages_broadcast_sender.send(msg_body)) { - (Ok(()), Ok(())) => Ok(()), - (Err(e), _) => fail!("pending_messages_queue_sender: send error {}", e), - (_, Err(e)) => fail!("pending_messages_broadcast_sender: send error {}", e) + if let Err(e) = instance.pending_messages_queue_sender.send(msg) { + fail!("pending_messages_queue_sender: send error {}", e) } + + if let Some(body) = msg_body { + if let Err(e) = instance.pending_messages_broadcast_sender.send(body) { + fail!("pending_messages_broadcast_sender: send error {}", e) + } + } + + Ok(()) } #[cfg(feature = "telemetry")] From d979bcf148c8f077e0e845fe8fff5701e51daa1b Mon Sep 17 00:00:00 2001 From: Dmitry Shtukenberg Date: Wed, 27 Mar 2024 00:55:22 +0300 Subject: [PATCH 23/40] Removed senseless consistency checks --- src/validator/message_cache.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/validator/message_cache.rs b/src/validator/message_cache.rs index b883d7aa..8e17f5ba 100644 --- a/src/validator/message_cache.rs +++ b/src/validator/message_cache.rs @@ -771,9 +771,9 @@ impl MessageCache { let message_id = message.message_id.clone(); - if session.is_message_present(&message_id) { - fail!("Inconsistent message cache contents: message {} present in cache, although should not", message_id) - } + //if session.is_message_present(&message_id) { + // fail!("Inconsistent message cache contents: message {} present in cache, although should not", message_id) + //} session.message_status.insert(message_id.clone(), status.clone()); session.insert_message(message, message_header, message_origin) @@ -781,9 +781,9 @@ impl MessageCache { fn insert_message_header(&self, session: Arc, message_header: Arc, message_origin: Option>, status: &RempMessageStatus) -> Result<()> { let message_id = message_header.message_id.clone(); - if session.is_message_present(&message_id) { - fail!("Inconsistent message cache contents: message header {:x} present in cache, although should not", message_id) - } + //if session.is_message_present(&message_id) { + // fail!("Inconsistent message cache contents: message header {:x} present in cache, although should not", message_id) + //} session.message_status.insert(message_id.clone(), status.clone()); session.insert_message_header(&message_id, message_header, message_origin)?; From a6e43299ffee8a1b0ede7ccdbbb37aafa7c3b047 Mon Sep 17 00:00:00 2001 From: Dmitry Shtukenberg Date: Tue, 9 Apr 2024 15:23:36 +0300 Subject: [PATCH 24/40] Updated tests, made engine/remp references Weak in remp_service --- src/engine_operations.rs | 1 - src/validator/message_cache.rs | 30 +- src/validator/remp_catchain.rs | 6 +- src/validator/remp_service.rs | 32 +- src/validator/tests/test_rmq_messages.rs | 613 ++++++++++++++++++++++- 5 files changed, 639 insertions(+), 43 deletions(-) diff --git a/src/engine_operations.rs b/src/engine_operations.rs index 197eff59..93acb357 100644 --- a/src/engine_operations.rs +++ b/src/engine_operations.rs @@ -995,7 +995,6 @@ impl EngineOperations for Engine { async fn check_remp_duplicate(&self, message_id: &UInt256) -> Result { self.remp_service() .ok_or_else(|| error!("Can't get message status because remp service was not set"))? - .remp_core_interface()? .check_remp_duplicate(message_id) } diff --git a/src/validator/message_cache.rs b/src/validator/message_cache.rs index 8e17f5ba..4ccce9bd 100644 --- a/src/validator/message_cache.rs +++ b/src/validator/message_cache.rs @@ -51,6 +51,7 @@ use ton_api::{ use ton_block::{Deserializable, Message, Serializable, MsgAddressInt, MsgAddrStd, ExternalInboundMessageHeader, BlockIdExt, UnixTime32, GetRepresentationHash}; use ton_types::{error, fail, KeyId, SliceData, Result, UInt256}; +use crate::ext_messages::create_ext_message; #[cfg(test)] #[path = "tests/test_message_cache.rs"] @@ -84,30 +85,25 @@ impl RmqMessage { }.into_boxed() } + pub fn serialize_message_body(body: &ton_api::ton::ton_node::RempMessageBody) -> ton_api::ton::bytes { + serialize_tl_boxed_object!(body) + } + pub fn deserialize_message_body(raw: &ton_api::ton::bytes) -> Result { let message_body: ton_api::ton::ton_node::RempMessageBody = catchain::utils::deserialize_tl_boxed_object(&raw)?; Ok(message_body) } -/* - pub fn from_rmq_record(record: &ton_api::ton::ton_node::RmqRecord) -> Result { - match record { - ton_api::ton::ton_node::RmqRecord::TonNode_RmqMessage(rec) => { - let message_body = Arc::new(Message::construct_from_bytes(&rec.message)?); - Ok(Self { - message: message_body.clone(), - message_id: rec.message_id.clone().into(), - message_uid: get_message_uid(&message_body) - }) - }, - _ => fail!("message_cache::from_rmq_record: not an RmqMessage is given: {:?}", record) - } + + pub fn from_raw_message(raw_msg: &ton_api::ton::bytes) -> Result { + let (message_id, message) = create_ext_message(raw_msg)?; + let message_uid = get_message_uid(&message); + Self::new (Arc::new(message), message_id, message_uid) } -*/ + pub fn from_message_body(body: &ton_api::ton::ton_node::RempMessageBody) -> Result { - let message = Arc::new(Message::construct_from_bytes(body.message())?); - let message_id = message.hash()?; + let (message_id, message) = create_ext_message(body.message())?; let message_uid = get_message_uid(&message); - Self::new (message, message_id, message_uid) + Self::new (Arc::new(message), message_id, message_uid) } #[allow(dead_code)] diff --git a/src/validator/remp_catchain.rs b/src/validator/remp_catchain.rs index a2533c45..7094a9ae 100644 --- a/src/validator/remp_catchain.rs +++ b/src/validator/remp_catchain.rs @@ -35,9 +35,8 @@ use catchain::{ PublicKey, PublicKeyHash }; use ton_api::{ - IntoBoxed, ton::ton_node::RempCatchainRecordV2 + IntoBoxed, ton::ton_node::{RempCatchainRecordV2, RempMessageQuery} }; -use ton_api::ton::ton_node::RempMessageQuery; use ton_block::ValidatorDescr; use ton_types::{error, fail, KeyId, Result, UInt256}; @@ -571,7 +570,8 @@ impl RempCatchain { let message_id = query.message_id(); if let Some(message_body) = self.remp_manager.message_cache.get_message(message_id)? { let remp_body_response = message_body.as_remp_message_body(); - let response_payload = CatchainFactory::create_block_payload(serialize_tl_boxed_object!(&remp_body_response)); + let remp_body_response_raw = RmqMessage::serialize_message_body(&remp_body_response); + let response_payload = CatchainFactory::create_block_payload(remp_body_response_raw); Ok(response_payload) } diff --git a/src/validator/remp_service.rs b/src/validator/remp_service.rs index cd7f343b..57283eef 100644 --- a/src/validator/remp_service.rs +++ b/src/validator/remp_service.rs @@ -3,14 +3,15 @@ use crate::{ network::remp::RempMessagesSubscriber, }; -use std::{ops::Deref, sync::Arc}; +use std::{sync::Arc, sync::Weak}; use ton_api::ton::ton_node::RempMessage; -use ton_types::{error, fail, KeyId, Result}; +use ton_types::{error, fail, KeyId, Result, UInt256}; +use crate::engine_traits::RempDuplicateStatus; #[derive(Default)] pub struct RempService { - engine: tokio::sync::OnceCell>, - remp_core: tokio::sync::OnceCell>, + engine: tokio::sync::OnceCell>, + remp_core: tokio::sync::OnceCell>, } impl RempService { @@ -19,21 +20,30 @@ impl RempService { } pub fn set_engine(&self, engine: Arc) -> Result<()> { - self.engine.set(engine).map_err(|_| error!("Attempt to set engine twice")) + self.engine.set(Arc::downgrade(&engine)).map_err(|_| error!("Attempt to set engine twice")) } pub fn set_remp_core_interface(&self, remp_core: Arc) -> Result<()> { - self.remp_core.set(remp_core).map_err(|_| error!("Attempt to set remp_core twice")) + self.remp_core.set(Arc::downgrade(&remp_core)).map_err(|_| error!("Attempt to set remp_core twice")) } - pub fn remp_core_interface(&self) -> Result<&dyn RempCoreInterface> { - self.remp_core.get().ok_or_else(|| error!("remp_core was not set")).map(|rci| rci.deref()) + fn get_core_interface(&self) -> Result> { + self.remp_core.get() + .ok_or_else(|| error!("remp_core was not set"))? + .upgrade().ok_or_else(|| error!("remp_core weak reference is null")) + } + + pub fn check_remp_duplicate(&self, id: &UInt256) -> Result { + self.get_core_interface()?.check_remp_duplicate(id) } async fn new_remp_message(&self, message: RempMessage, source: &Arc) -> Result<()> { // TODO send error receipt in case of any error - let engine = self.engine.get().ok_or_else(|| error!("engine was not set"))?; - let remp_core = self.remp_core_interface()?; + let engine = self.engine + .get().ok_or_else(|| error!("engine was not set"))? + .upgrade().ok_or_else(|| error!("engine weak reference is null"))?; + + let remp_core = self.get_core_interface()?; #[cfg(feature = "telemetry")] engine.remp_core_telemetry().message_from_fullnode(); @@ -42,6 +52,8 @@ impl RempService { fail!("Can't process REMP message because validator is out of sync"); } + log::trace!(target: "remp", "Message: {:?}", message.message()); + // deserialise message let id = message.id().clone(); let (real_id, message) = create_ext_message(&message.message())?; diff --git a/src/validator/tests/test_rmq_messages.rs b/src/validator/tests/test_rmq_messages.rs index 3e4398d0..524a8eee 100644 --- a/src/validator/tests/test_rmq_messages.rs +++ b/src/validator/tests/test_rmq_messages.rs @@ -1,13 +1,602 @@ -/* -* Copyright (C) 2019-2021 TON Labs. All Rights Reserved. -* -* Licensed under the SOFTWARE EVALUATION License (the "License"); you may not use -* this file except in compliance with the License. -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific TON DEV software governing permissions and -* limitations under the License. -*/ +use std::{collections::HashSet, str::FromStr, sync::Arc, time::Duration}; +use std::ops::RangeInclusive; +use ton_api::ton::ton_node::{RempMessageBody, RempMessageLevel, RempMessageLevel::TonNode_RempMasterchain, RempMessageStatus, rempmessagestatus::{RempAccepted, RempIgnored}}; + +use ton_block::{ + Message, Serializable, Deserializable, ExternalInboundMessageHeader, + MsgAddressInt, Grams, ShardIdent, ValidatorDescr, SigPubKey, BlockIdExt, + MsgAddressExt::AddrNone, UnixTime32 +}; +use ton_types::{error, fail, Result, SliceData, UInt256}; + +use crate::{ + config::RempConfig, + engine_traits::{EngineOperations, RempCoreInterface, RempDuplicateStatus}, + validator::{ + reliable_message_queue::{MessageQueue, RmqMessage}, + remp_block_parser::{BlockProcessor, RempMasterBlockIndexingProcessor}, + remp_manager::{RempInterfaceQueues, RempManager, RempSessionStats}, + sessions_computing::GeneralSessionInfo, + validator_utils::{ + get_message_uid, sigpubkey_to_publickey, ValidatorListHash + } + } +}; +#[cfg(feature = "telemetry")] +use crate::validator::telemetry::RempCoreTelemetry; + +use catchain::PublicKey; +use crate::ext_messages::create_ext_message; +use crate::validator::message_cache::{RempMessageOrigin, RempMessageWithOrigin}; +use crate::validator::remp_catchain::RempCatchainInfo; + +#[test] +fn test_rmq_message_serialize() -> Result<()> { + let pre_message = Message::with_ext_in_header_and_body( + ExternalInboundMessageHeader { + src: AddrNone, + dst: MsgAddressInt::from_str( + "-1:7777777777777777777777777777777777777777777777777777777777777777" + ).unwrap(), + import_fee: Grams::zero() + }, + SliceData::from_string( + "e32fb2df7d15a1266c0b4e949607b67f69f2edfa8dc401bec2df9e985e69eecf8e5fb3903cfc3d19d9910077f0e83c4516cf544c9ca65d33a42c71f6b74ed281d54fe4d47282105f230ce104f0bf52cc3adfdb4d5a538f86e33acf55ca68d1480000005f43e9ca6118634e4fdb079a4f00000000603_" + ).unwrap() + ); + let message = Arc::new(pre_message); + let message_uid = get_message_uid(&message); + + let rmq_message_test0: Arc = Arc::new(RmqMessage { + message, + message_id: UInt256::from_str("66e5dd1695ab795c04d474cf304b802413488d229639c2f4a980a28be3fbecb1").unwrap(), + message_uid, + //source_key: KeyId::from_data([249, 183, 41, 53, 7, 177, 255, 161, 106, 8, 18, 35, 65, 57, 50, 226, 122, 180, 239, 174, 198, 50, 125, 69, 218, 207, 153, 216, 215, 79, 253, 155]), + //source_idx: 0, + //timestamp: 0, + }); + + // Testing proper message field serialization + let data: ton_api::ton::bytes = rmq_message_test0.message.write_to_bytes().unwrap().into(); + let msg_buffer = Message::construct_from_bytes(&data).unwrap(); + + assert_eq!(*rmq_message_test0.message, msg_buffer); + + // Testing body serialization + let test0_serialized = RmqMessage::serialize_message_body(&rmq_message_test0.as_remp_message_body()); + println!("Serialized message data: {:?}", data); + println!("Serialized remp_message_body: {:?}", test0_serialized); + let msg = RmqMessage::deserialize_message_body(&test0_serialized).unwrap(); + println!("Deserialzed message data: {:?}", msg.message()); + let rmq_message = RmqMessage::from_message_body(&msg /*RmqMessage::deserialize_message_body(&msg)?*/)?; + let mut message_without_params = Message::default(); + message_without_params.set_header(rmq_message.message.header().clone()); + message_without_params.set_body(rmq_message.message.body().unwrap()); + assert_eq!(*rmq_message_test0.message, message_without_params); + assert_eq!(rmq_message_test0.message_id, rmq_message.message_id, + "Message ids differ: {:x} /= {:x}", rmq_message_test0.message_id, rmq_message.message_id); + assert_eq!(rmq_message_test0.message_uid, rmq_message.message_uid, + "Message uids differ: {:x} /= {:x}", rmq_message_test0.message_uid, rmq_message.message_uid); + + Ok(()) +} + +#[test] +fn test_rmq_message_id() -> Result<()> { + //let message_data = SliceData::from_string( + // "b5ee9c720101030100c300014589feeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee0\ + // c0101ebbb6f5976b0533dc287fea336a34e909c9d7f18018af9afd3ad0fe36ee058000f2ab5d13c159aaffb04917f\ + // 7be06ebbb58f45b14de97fe7df5edaf78847abf483554fe4d47282105f230ce104f0bf52cc3adfdb4d5a538f86e33\ + // acf55ca68d14800000063aa878d5b1983b83c1b079a4f000000006030020043d06435a826dde79131e36acc2ba1f8\ + // d92476518ecd71be9398816ee95b666cae4540_")?; + + + let message_data = SliceData::from_string(format!("{}{}{}{}", + "b5ee9c720101030100c300014589feeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee0", + "c0101ebbb6f5976b0533dc287fea336a34e909c9d7f18018af9afd3ad0fe36ee058000f2ab5d13c159aaffb04917f", + "7be06ebbb58f45b14de97fe7df5edaf78847abf483554fe4d47282105f230ce104f0bf52cc3adfdb4d5a538f86e33", + "acf55ca68d14800000063aa878d5b1983b83c1b079a4f000000006030_" //020043d06435a826dde79131e36acc2ba1f8","d92476518ecd71be9398816ee95b666cae4540" + ).as_str())?; + + let message_id = UInt256::from_str("e8da5b5c46b97ec31330a874347beae2bcac9ab2b3dbc6c3cd4deec9994e3c33")?; + + let message_bytes = message_data.write_to_bytes()?; + let rmq_message = RmqMessage::from_raw_message(&message_bytes)?; + assert_eq!(message_id, rmq_message.message_id); + Ok(()) +} + +struct RmqTestEngine { + #[cfg(feature = "telemetry")] + remp_core_telemetry: RempCoreTelemetry, + collator_queue: lockfree::queue::Queue<(UInt256, Arc)> +} + +impl EngineOperations for RmqTestEngine { + fn new_remp_message(&self, id: UInt256, message: Arc) -> Result<()> { + println!("New message received for collation: {:x}", id); + self.collator_queue.push((id, message)); + Ok(()) + } + + #[cfg(feature = "telemetry")] + fn remp_core_telemetry(&self) -> &RempCoreTelemetry { + &self.remp_core_telemetry + } +} + +impl RmqTestEngine { + pub fn new() -> Self { + Self { + #[cfg(feature = "telemetry")] + remp_core_telemetry : RempCoreTelemetry::new(10), + collator_queue: lockfree::queue::Queue::new() + } + } +} + +struct RmqTestbench { + engine: Arc, + remp_manager: Arc, + remp_interface_queues: RempInterfaceQueues, + params: Arc, + local_key: PublicKey, + curr_validators: Vec, + next_validators: Vec, + node_list_id: UInt256, + rp_guarantee: Duration, + message_queue: MessageQueue +} + +impl RmqTestbench { + fn create_runtime() -> Result { + let runtime = match tokio::runtime::Runtime::new() { + Ok(r) => r, + Err(e) => fail!(e) + }; + return Ok(runtime); + } + + async fn new(runtime_handle: &tokio::runtime::Handle, masterchain_seqno: u32, rp_guarantee: Duration) -> Result { + let engine = Arc::new(RmqTestEngine::new()); + + let remp_config = RempConfig::create_empty(); + let (remp_manager_value, remp_interface_queues) = RempManager::create_with_options( + engine.clone(), remp_config.clone(), Arc::new(runtime_handle.clone()) + ); + let remp_manager = Arc::new(remp_manager_value); + let local_validator = ValidatorDescr::with_params ( + SigPubKey::from_bytes(UInt256::rand().as_slice())?, + 1, None, None + ); + let local_key = sigpubkey_to_publickey(&local_validator.public_key); + let curr_validators = vec!(local_validator.clone()); + let next_validators = vec!(local_validator.clone()); + let params = Arc::new(GeneralSessionInfo { + shard: ShardIdent::with_tagged_prefix(0,0xc000_0000_0000_0000)?, + opts_hash: Default::default(), + catchain_seqno: 2, + key_seqno: 1, + max_vertical_seqno: 0 + }); + let node_list_id = ValidatorListHash::rand(); + + for cc in 1..=masterchain_seqno { + remp_manager.create_master_cc_session(cc, 0.into(), vec!())?; + } + let masterchain_range = remp_manager.advance_master_cc(masterchain_seqno, rp_guarantee.clone())?; + + let queue_info = Arc::new(RempCatchainInfo::create( + params.clone(), &masterchain_range, + &curr_validators, &next_validators, + &local_key, node_list_id.clone(), + )?); + let message_queue = MessageQueue::create( + engine.clone(), remp_manager.clone(), queue_info + )?; + + Ok(RmqTestbench { + engine, + remp_manager, + remp_interface_queues, + params, + local_key, + curr_validators, + next_validators, + node_list_id, + rp_guarantee, + message_queue + }) + } + + async fn send_pending_message(&self, msg: &RempMessageWithOrigin, masterchain_seqno: u32) -> Result<()> { + self.message_queue.process_pending_remp_catchain_record( + &msg.as_remp_catchain_record(masterchain_seqno), + 0 + ).await + } + + fn replace_message_queue(&mut self, masterchain_range: &RangeInclusive) -> Result<()> { + let info = Arc::new(RempCatchainInfo::create( + self.params.clone(), masterchain_range, + &self.curr_validators, &self.next_validators, + &self.local_key, self.node_list_id.clone())?); + + self.message_queue = MessageQueue::create( + self.engine.clone(), self.remp_manager.clone(), info + )?; + Ok(()) + } + + async fn advance_master_cc(&mut self, masterchain_seqno: u32, mc_time: UnixTime32) -> Result { + self.remp_manager.create_master_cc_session(masterchain_seqno, mc_time, vec!())?; + let new_range = self.remp_manager.advance_master_cc(masterchain_seqno, self.rp_guarantee)?; + self.replace_message_queue(&new_range)?; + Ok(self.remp_manager.gc_old_messages(*new_range.start()).await) + } +} + +fn random_message() -> Result { + RmqMessage::make_test_message(&SliceData::from(UInt256::rand())) +} + +fn make_test_message_with_origin(body: &SliceData) -> Result { + let m = RmqMessage::make_test_message(body)?; + let o = RempMessageOrigin::create_empty()?; + Ok(RempMessageWithOrigin { message: m, origin: o }) +} + +fn make_test_random_message_with_origin() -> Result { + make_test_message_with_origin(&SliceData::from(UInt256::rand())) +} + +#[test] +fn remp_simple_forwarding_test() -> Result<()> { + // -- seq=0/seq=1,forwarding + // 5k -- new/(k=0/1rejected_l/2rejected_r/3rejected_lr) + // 5k+1 -- ignored/() + // 5k+2 -- duplicate/() + // 5k+3 -- accepted/() + // 5k+4 -- rejected/() + + //let mut msgs = Vec::new(); + //init_test_log(); + let runtime = RmqTestbench::create_runtime()?; + let runtime_handle = runtime.handle().clone(); + + runtime.block_on(async move { + let testbench = RmqTestbench::new(&runtime_handle, 2, Duration::from_secs(10)).await?; + + let blk1 = BlockIdExt::with_params( + testbench.params.shard.clone(), + 5129, + UInt256::rand(), + UInt256::rand() + ); + + let m1 = make_test_random_message_with_origin()?; + let m2 = make_test_random_message_with_origin()?; + + testbench.remp_manager.message_cache.add_external_message_status( + m1.get_message_id(), + &m1.message.message_uid, + Some(Arc::new(m1.message.clone())), + Some(Arc::new(m1.origin.clone())), + RempMessageStatus::TonNode_RempAccepted(RempAccepted { + level: RempMessageLevel::TonNode_RempShardchain, + block_id: blk1.clone(), + master_id: Default::default() + }), + |_old,new| new.clone(), + 1 + )?; + + //testbench.remp_manager.options + println!("{:?}", testbench.remp_manager.message_cache.get_message_with_origin_status_cc(m1.get_message_id())); + + testbench.send_pending_message(&m1, 1).await?; + testbench.send_pending_message(&m2, 1).await?; + + let ign = RempIgnored { level: RempMessageLevel::TonNode_RempMasterchain, block_id: blk1 }; + assert_eq!(testbench.remp_manager.message_cache.get_message_status(m1.get_message_id())?.unwrap(), RempMessageStatus::TonNode_RempIgnored(ign)); + assert_eq!(testbench.remp_manager.message_cache.get_message_status(m2.get_message_id())?.unwrap(), MessageQueue::forwarded_ignored_status()); + + println!("Remp interface queues: {} repsonses", testbench.remp_interface_queues.response_receiver.len()); + Ok(()) + }) +} + +#[test] +fn remp_simple_collation_equal_uids_test() -> Result<()> { + //init_test_log(); + let runtime = RmqTestbench::create_runtime()?; + let runtime_handle = runtime.handle().clone(); + + runtime.block_on(async move { + let testbench = RmqTestbench::new(&runtime_handle, 2, Duration::from_secs(10)).await?; + + let body1 = SliceData::from(UInt256::rand()); + let body2 = SliceData::from(UInt256::rand()); + + let mut msgs = Vec::new(); + for _cnt in 0..5 { + msgs.push(make_test_message_with_origin(&body1)?); + msgs.push(make_test_message_with_origin(&body2)?); + } + + let (_min_id,uid_for_min_id) = msgs.iter().map(|a| (a.get_message_id(), &a.message.message_uid)).min().unwrap(); + let msg_to_collate = msgs.iter().filter(|x| &x.message.message_uid != uid_for_min_id).map(|x| x.get_message_id()).min().unwrap(); + let (acc_id,acc_uid) = msgs.iter().filter(|x| &x.message.message_uid == uid_for_min_id).map(|x| (x.get_message_id(),&x.message.message_uid)).max().unwrap(); + + let mut must_be_collated = HashSet::::new(); + must_be_collated.insert(msg_to_collate.clone()); + let must_be_rejected : Vec = msgs.iter() + .filter(|a| !must_be_collated.contains(a.get_message_id()) && a.get_message_id() != acc_id) + .map(|a| a.clone()) + .collect(); + + for m in msgs.iter() { + let pc = if must_be_collated.contains(m.get_message_id()) { "C" } else { "" }.to_string(); + let pa = if m.get_message_id() == acc_id { "A" } else { "" }.to_string(); + let pr = if must_be_rejected.contains(&m) { "R" } else { "" }.to_string(); + println!("Pending msg: {:x} {}{}{}", m.get_message_id(), pc, pa, pr); + testbench.send_pending_message(m, testbench.message_queue.catchain_info.get_master_cc_seqno()).await?; + } + + let accepted = RempAccepted { + level: RempMessageLevel::TonNode_RempMasterchain, + block_id: Default::default(), + master_id: Default::default() + }; + testbench.remp_manager.message_cache.add_external_message_status( + &acc_id, &acc_uid, None, None, + RempMessageStatus::TonNode_RempAccepted(accepted), + |_o,n| n.clone(), + 2 + )?; + + testbench.message_queue.collect_messages_for_collation().await?; + + for (id, _msg) in testbench.engine.collator_queue.pop_iter() { + println!("collated: {:x}", id); + assert!(must_be_collated.remove(&id)); + } + assert!(must_be_collated.is_empty()); + + for id in must_be_rejected.iter() { + let status = testbench.remp_manager.message_cache.get_message_status(id.get_message_id())?; + if let Some(RempMessageStatus::TonNode_RempRejected(rejected)) = &status { + assert_eq!(rejected.level, RempMessageLevel::TonNode_RempQueue); + println!("rejected: {}, '{:?}'", id, status) + } + else { + panic!("Expected rejected status, found {:?}", status); + } + } + + Ok(()) + }) +} + +#[test] +fn remp_simple_expiration_test() -> Result<()> { + //init_test_log(); + let runtime = RmqTestbench::create_runtime()?; + let runtime_handle = runtime.handle().clone(); + + runtime.block_on(async move { + let mut testbench = RmqTestbench::new(&runtime_handle, 2, Duration::from_secs(10)).await?; + + let mut bodies = Vec::new(); + for _ in 0..5 { + bodies.push(SliceData::from(UInt256::rand())); + } + + for b in bodies.iter() { + let m = make_test_message_with_origin(b)?; + println!("Pending msg 2: {:x}", m.get_message_id()); + testbench.send_pending_message(&m, testbench.message_queue.catchain_info.get_master_cc_seqno()).await?; + } + println!("Collected old messages 3: {}", testbench.advance_master_cc(3, 30.into()).await?); + println!("Collected old messages 4: {}", testbench.advance_master_cc(4, 40.into()).await?); + println!("Collected old messages 5: {}", testbench.advance_master_cc(5, 50.into()).await?); + + let mut msgs = Vec::new(); + for b in bodies.iter() { + let m = make_test_message_with_origin(b)?; + println!("Pending msg 5: {:x}", m.get_message_id()); + testbench.send_pending_message(&m, testbench.message_queue.catchain_info.get_master_cc_seqno()).await?; + msgs.push(m); + } + + for m in msgs.iter() { + assert_eq!( + testbench.remp_interface_queues.check_remp_duplicate(m.get_message_id())?, + RempDuplicateStatus::Fresh(m.message.message_uid.clone()) + ); + testbench.remp_manager.message_cache.add_external_message_status( + m.get_message_id(), + &m.message.message_uid, + Some(Arc::new(m.message.clone())), + Some(Arc::new(m.origin.clone())), + RempMessageStatus::TonNode_RempAccepted(RempAccepted { + level: TonNode_RempMasterchain, + block_id: Default::default(), + master_id: Default::default() + }), + |_o,n| n.clone(), + 5 + )?; + } + msgs.clear(); + + println!("Collected old messages 6: {}", testbench.advance_master_cc(6, 60.into()).await?); + + for b in bodies.iter() { + let m = make_test_message_with_origin(b)?; + println!("Pending msg 6: {:x}", m.get_message_id()); + testbench.send_pending_message(&m, testbench.message_queue.catchain_info.get_master_cc_seqno()).await?; + msgs.push(m); + } + + for m in msgs.iter() { + match testbench.remp_interface_queues.check_remp_duplicate(&m.get_message_id())? { + RempDuplicateStatus::Absent => panic!("Message {} must present", m), + RempDuplicateStatus::Fresh(uid) => panic!("Message {} must not be fresh with uid {:x}", m, uid), + RempDuplicateStatus::Duplicate(_, uid, _) => assert_eq!(m.message.message_uid, uid) + } + } + Ok(()) + }) +} + +#[test] +fn remp_simple_status_updating_test() -> Result<()> { + //init_test_log(); + let runtime = RmqTestbench::create_runtime()?; + let runtime_handle = runtime.handle().clone(); + + runtime.block_on(async move { + let mut testbench = RmqTestbench::new(&runtime_handle, 1, Duration::from_secs(10)).await?; + let blk1 = BlockIdExt::with_params( + testbench.params.shard.clone(), + 5129, + UInt256::rand(), + UInt256::rand() + ); + + let common_body = SliceData::from(UInt256::rand()); + + let m1 = make_test_message_with_origin(&common_body)?; + let m2 = make_test_message_with_origin(&common_body)?; + + let proc = RempMasterBlockIndexingProcessor::new( + blk1.clone(), blk1.clone(), + testbench.remp_manager.message_cache.clone(), + 1 + ); + proc.process_message(m1.get_message_id(), &m1.message.message_uid).await; + + testbench.advance_master_cc(2, 10.into()).await?; + + testbench.remp_manager.message_cache.add_external_message_status( + m2.get_message_id(), + &m2.message.message_uid, + Some(Arc::new(m2.message.clone())), + Some(Arc::new(m2.origin.clone())), + RempMessageStatus::TonNode_RempNew, + |_old,new| new.clone(), + 2 + )?; + + assert_eq!( + testbench.remp_interface_queues.check_remp_duplicate(m2.get_message_id())?, + RempDuplicateStatus::Duplicate( + blk1.clone(), m1.message.message_uid.clone(), m1.get_message_id().clone() + ) + ); + + testbench.advance_master_cc(3, 20.into()).await?; + + assert_eq!( + testbench.remp_interface_queues.check_remp_duplicate(m2.get_message_id())?, + RempDuplicateStatus::Fresh(m2.message.message_uid.clone()) + ); + + Ok(()) + }) +} + +async fn push_random_msgs(testbench: &RmqTestbench, count: usize) -> Result>> { + let blk1 = BlockIdExt::with_params( + testbench.params.shard.clone(), + 5129, + UInt256::rand(), + UInt256::rand() + ); + + let mut msgs = Vec::new(); + for _ in 0..count { + msgs.push(Arc::new(random_message()?)); + } + + let proc = RempMasterBlockIndexingProcessor::new( + blk1.clone(), blk1.clone(), + testbench.remp_manager.message_cache.clone(), + testbench.message_queue.catchain_info.get_master_cc_seqno() + ); + + for m in msgs.iter() { + proc.process_message(&m.message_id, &m.message_uid).await; + } + + Ok(msgs) +} + +#[test] +fn remp_simple_gc_range_test() -> Result<()> { + //init_test_log(); + let runtime = RmqTestbench::create_runtime()?; + let runtime_handle = runtime.handle().clone(); + + runtime.block_on(async move { + let mut testbench = RmqTestbench::new(&runtime_handle, 1, Duration::from_secs(10)).await?; + + let msgs = push_random_msgs(&testbench, 20).await?; + + let stats = testbench.advance_master_cc(2, 5.into()).await?; + assert_eq!(stats.total, 0); + + let msgs2 = push_random_msgs(&testbench, 24).await?; + + let stats = testbench.advance_master_cc(3, 7.into()).await?; + assert_eq!(stats.total, 0); + let stats = testbench.advance_master_cc(4, 9.into()).await?; + assert_eq!(stats.total, 0); + let stats = testbench.advance_master_cc(5, 14.into()).await?; + assert_eq!(stats.total, 0); + let stats = testbench.advance_master_cc(6, 15.into()).await?; + assert_eq!(stats.has_only_header, msgs.len()); + let stats = testbench.advance_master_cc(7, 17.into()).await?; + assert_eq!(stats.has_only_header, msgs2.len()); + let stats = testbench.advance_master_cc(8, 18.into()).await?; + assert_eq!(stats.total, 0); + + let msgs3 = push_random_msgs(&testbench, 17).await?; + let stats = testbench.advance_master_cc(9, 100.into()).await?; + assert_eq!(stats.total, 0); + let _msgs4 = push_random_msgs(&testbench, 417).await?; + let stats = testbench.advance_master_cc(10, 218.into()).await?; + assert_eq!(stats.has_only_header, msgs3.len()); + let stats = testbench.advance_master_cc(10, 218.into()).await?; + assert_eq!(stats.total, 0); + + Ok(()) + }) +} + +#[test] +fn remp_simple_advance_special_cases() -> Result<()> { + //init_test_log(); + let runtime = RmqTestbench::create_runtime()?; + let runtime_handle = runtime.handle().clone(); + + runtime.block_on(async move { + let mut testbench = RmqTestbench::new(&runtime_handle, 1, Duration::from_secs(10)).await?; + let msgs = push_random_msgs(&testbench, 20).await?; + let stat = testbench.advance_master_cc(2, 20.into()).await?; + assert_eq!(stat.total, 0); + let msgs2 = push_random_msgs(&testbench, 20).await?; + let stat = testbench.advance_master_cc(3, 30.into()).await?; + assert_eq!(stat.total, msgs.len()); + assert!(testbench.advance_master_cc(3, 40.into()).await.is_err()); + assert!(testbench.advance_master_cc(3, 29.into()).await.is_err()); + let stat = testbench.advance_master_cc(3, 30.into()).await?; + assert_eq!(stat.total, 0); + let stat = testbench.advance_master_cc(4, 40.into()).await?; + assert_eq!(stat.total, msgs2.len()); + assert!(testbench.advance_master_cc(5, 39.into()).await.is_err()); + Ok(()) + }) +} From 83892c9fb25d106f0baafc9adce5ac57df458960 Mon Sep 17 00:00:00 2001 From: Dmitry Shtukenberg Date: Tue, 9 Apr 2024 21:27:41 +0300 Subject: [PATCH 25/40] Fixed remp tests --- src/engine_traits.rs | 2 +- src/validator/message_cache.rs | 35 ++++++------- src/validator/remp_manager.rs | 41 +++++++-------- src/validator/remp_service.rs | 29 +++-------- src/validator/tests/test_rmq_messages.rs | 64 +++++++++--------------- 5 files changed, 67 insertions(+), 104 deletions(-) diff --git a/src/engine_traits.rs b/src/engine_traits.rs index 1181718e..6f1cc38f 100644 --- a/src/engine_traits.rs +++ b/src/engine_traits.rs @@ -955,6 +955,6 @@ pub enum RempDuplicateStatus { #[async_trait::async_trait] pub trait RempCoreInterface: Sync + Send { - async fn process_incoming_message(&self, message_id: UInt256, message: Message, source: Arc) -> Result<()>; + async fn process_incoming_message(&self, message: &RempMessage, source: Arc) -> Result<()>; fn check_remp_duplicate(&self, message_id: &UInt256) -> Result; } diff --git a/src/validator/message_cache.rs b/src/validator/message_cache.rs index 4ccce9bd..7fb2ae58 100644 --- a/src/validator/message_cache.rs +++ b/src/validator/message_cache.rs @@ -65,20 +65,21 @@ pub struct RmqMessage { } impl RmqMessage { - pub fn new(message: Arc, message_id: UInt256, message_uid: UInt256) -> Result { + pub fn new(message: Arc) -> Result { + let message_id = message.hash()?; + Self::new_with_id (message, message_id) + } + + fn new_with_id(message: Arc, message_id: UInt256) -> Result { + let message_uid = get_message_uid(&message); return Ok(RmqMessage { message, message_id, message_uid }) } -/* - pub fn as_rmq_record(&self) -> ton_api::ton::ton_node::RmqRecord { - ton_api::ton::ton_node::rmqrecord::RmqMessage { - message: self.message.write_to_bytes().unwrap().into(), - message_id: self.message_id.clone().into(), - source_key_id: UInt256::default(), - source_idx: 0, - masterchain_seqno: 0 - }.into_boxed() + + pub fn from_raw_message(raw_msg: &ton_api::ton::bytes) -> Result { + let (message_id, message) = create_ext_message(raw_msg)?; + Self::new_with_id (Arc::new(message), message_id) } -*/ + pub fn as_remp_message_body(&self) -> ton_api::ton::ton_node::RempMessageBody { ton_api::ton::ton_node::rempmessagebody::RempMessageBody { message: self.message.write_to_bytes().unwrap().into() @@ -94,16 +95,9 @@ impl RmqMessage { Ok(message_body) } - pub fn from_raw_message(raw_msg: &ton_api::ton::bytes) -> Result { - let (message_id, message) = create_ext_message(raw_msg)?; - let message_uid = get_message_uid(&message); - Self::new (Arc::new(message), message_id, message_uid) - } - pub fn from_message_body(body: &ton_api::ton::ton_node::RempMessageBody) -> Result { let (message_id, message) = create_ext_message(body.message())?; - let message_uid = get_message_uid(&message); - Self::new (Arc::new(message), message_id, message_uid) + Self::new_with_id (Arc::new(message), message_id) } #[allow(dead_code)] @@ -132,9 +126,8 @@ impl RmqMessage { msg, msg_cell.data(), msg_cell.repr_hash().to_hex_string() ); - let (msg_id, msg_uid, msg) = (msg_cell.repr_hash(), get_message_uid(&msg), msg); - RmqMessage::new (Arc::new(msg), msg_id, msg_uid) + RmqMessage::new (Arc::new(msg)) } } diff --git a/src/validator/remp_manager.rs b/src/validator/remp_manager.rs index 3ad5c8be..cd6c52b5 100644 --- a/src/validator/remp_manager.rs +++ b/src/validator/remp_manager.rs @@ -22,7 +22,7 @@ use std::cmp::{max, Reverse}; use std::collections::BinaryHeap; use ton_block::{BlockIdExt, CatchainConfig, Message, ShardIdent, UnixTime32}; -use ton_api::ton::ton_node::RempMessageStatus; +use ton_api::ton::ton_node::{RempMessageStatus}; use ton_types::{error, fail, KeyId, Result, SliceData, UInt256}; use crate::{ @@ -31,7 +31,7 @@ use crate::{ validator::{ message_cache::{RmqMessage, RempMessageOrigin, RempMessageWithOrigin, MessageCache}, mutex_wrapper::MutexWrapper, remp_catchain::RempCatchainStore, - validator_utils::{get_message_uid, get_shard_by_message} + validator_utils::get_shard_by_message } }; @@ -632,33 +632,34 @@ impl RempInterfaceQueues { #[async_trait::async_trait] impl RempCoreInterface for RempInterfaceQueues { - async fn process_incoming_message(&self, message_id: UInt256, message: Message, source: Arc) -> Result<()> { - let arc_message = Arc::new(message.clone()); - + async fn process_incoming_message(&self, message: &ton_api::ton::ton_node::RempMessage, source: Arc) -> Result<()> { // build message - let remp_message = RmqMessage::new ( - arc_message, - message_id.clone(), - get_message_uid(&message), - )?; - - let remp_message_origin = RempMessageOrigin::new ( - source, - 0 - )?; - - if self.message_cache.is_message_fully_known(&message_id)? { + let remp_message = RmqMessage::from_raw_message(message.message())?; + if message.id() != &remp_message.message_id { + fail!("Message {:x} has invalid message id {:x}, message will be ignored", + remp_message.message_id, message.id() + ); + } + if self.message_cache.is_message_fully_known(&remp_message.message_id)? { log::trace!(target: "remp", "Point 1. Message {:x} is fully known, no forwarding to incoming queue is necessary", - message_id + remp_message.message_id ); } else { + let remp_message_origin = RempMessageOrigin::new ( + source, + 0 + )?; + log::trace!(target: "remp", "Point 1. Adding incoming message {} to incoming queue, known message info {}", - remp_message, self.message_cache.get_message_info(&message_id)? + remp_message, self.message_cache.get_message_info(&remp_message.message_id)? ); - self.incoming_sender.send(Arc::new(RempMessageWithOrigin { message: remp_message, origin: remp_message_origin }))?; + self.incoming_sender.send(Arc::new(RempMessageWithOrigin { + message: remp_message, + origin: remp_message_origin + }))?; #[cfg(feature = "telemetry")] self.engine.remp_core_telemetry().in_channel_from_fullnode(self.incoming_sender.len()); } diff --git a/src/validator/remp_service.rs b/src/validator/remp_service.rs index 57283eef..710629d7 100644 --- a/src/validator/remp_service.rs +++ b/src/validator/remp_service.rs @@ -1,12 +1,10 @@ use crate::{ - engine_traits::{EngineOperations, RempCoreInterface}, ext_messages::create_ext_message, - network::remp::RempMessagesSubscriber, + engine_traits::{EngineOperations, RempCoreInterface, RempDuplicateStatus}, + network::remp::RempMessagesSubscriber }; use std::{sync::Arc, sync::Weak}; -use ton_api::ton::ton_node::RempMessage; use ton_types::{error, fail, KeyId, Result, UInt256}; -use crate::engine_traits::RempDuplicateStatus; #[derive(Default)] pub struct RempService { @@ -37,7 +35,7 @@ impl RempService { self.get_core_interface()?.check_remp_duplicate(id) } - async fn new_remp_message(&self, message: RempMessage, source: &Arc) -> Result<()> { + async fn new_remp_message(&self, message: &ton_api::ton::ton_node::RempMessage, source: &Arc) -> Result<()> { // TODO send error receipt in case of any error let engine = self.engine .get().ok_or_else(|| error!("engine was not set"))? @@ -52,34 +50,19 @@ impl RempService { fail!("Can't process REMP message because validator is out of sync"); } - log::trace!(target: "remp", "Message: {:?}", message.message()); - - // deserialise message - let id = message.id().clone(); - let (real_id, message) = create_ext_message(&message.message())?; - if real_id != id { - fail!("Given message id {:x} is not equal calculated one {:x}", id, real_id); - } - - log::trace!(target: "remp", "Point 0. Incoming REMP message {:x} received from {}: {:?}", - id, source, message - ); - // push into remp catchain - remp_core.process_incoming_message(id, message, source.clone()).await?; - - Ok(()) + remp_core.process_incoming_message(message, source.clone()).await } } #[async_trait::async_trait] impl RempMessagesSubscriber for RempService { - async fn new_remp_message(&self, message: RempMessage, source: &Arc) -> Result<()> { + async fn new_remp_message(&self, message: ton_api::ton::ton_node::RempMessage, source: &Arc) -> Result<()> { // TODO send error receipt in case of any error let id = message.id().clone(); log::trace!(target: "remp", "Point 0. Processing incoming REMP message {:x}", id); - match self.new_remp_message(message, source).await { + match self.new_remp_message(&message, source).await { Ok(_) => log::trace!(target: "remp", "Point 0. Processed incoming REMP message {:x}", id), Err(e) => log::error!(target: "remp", "Point 0. Error processing incoming REMP message {:x}: {}", id, e) } diff --git a/src/validator/tests/test_rmq_messages.rs b/src/validator/tests/test_rmq_messages.rs index 524a8eee..e9c6dc79 100644 --- a/src/validator/tests/test_rmq_messages.rs +++ b/src/validator/tests/test_rmq_messages.rs @@ -1,13 +1,9 @@ use std::{collections::HashSet, str::FromStr, sync::Arc, time::Duration}; use std::ops::RangeInclusive; -use ton_api::ton::ton_node::{RempMessageBody, RempMessageLevel, RempMessageLevel::TonNode_RempMasterchain, RempMessageStatus, rempmessagestatus::{RempAccepted, RempIgnored}}; +use ton_api::ton::ton_node::{RempMessageLevel, RempMessageLevel::TonNode_RempMasterchain, RempMessageStatus, rempmessagestatus::{RempAccepted, RempIgnored}}; -use ton_block::{ - Message, Serializable, Deserializable, ExternalInboundMessageHeader, - MsgAddressInt, Grams, ShardIdent, ValidatorDescr, SigPubKey, BlockIdExt, - MsgAddressExt::AddrNone, UnixTime32 -}; +use ton_block::{Message, Serializable, Deserializable, ExternalInboundMessageHeader, MsgAddressInt, Grams, ShardIdent, ValidatorDescr, SigPubKey, BlockIdExt, MsgAddressExt::AddrNone, UnixTime32, GetRepresentationHash}; use ton_types::{error, fail, Result, SliceData, UInt256}; use crate::{ @@ -27,7 +23,6 @@ use crate::{ use crate::validator::telemetry::RempCoreTelemetry; use catchain::PublicKey; -use crate::ext_messages::create_ext_message; use crate::validator::message_cache::{RempMessageOrigin, RempMessageWithOrigin}; use crate::validator::remp_catchain::RempCatchainInfo; @@ -46,30 +41,22 @@ fn test_rmq_message_serialize() -> Result<()> { ).unwrap() ); let message = Arc::new(pre_message); - let message_uid = get_message_uid(&message); - - let rmq_message_test0: Arc = Arc::new(RmqMessage { - message, - message_id: UInt256::from_str("66e5dd1695ab795c04d474cf304b802413488d229639c2f4a980a28be3fbecb1").unwrap(), - message_uid, - //source_key: KeyId::from_data([249, 183, 41, 53, 7, 177, 255, 161, 106, 8, 18, 35, 65, 57, 50, 226, 122, 180, 239, 174, 198, 50, 125, 69, 218, 207, 153, 216, 215, 79, 253, 155]), - //source_idx: 0, - //timestamp: 0, - }); - - // Testing proper message field serialization + let rmq_message_test0: Arc = Arc::new(RmqMessage::new(message)?); + + // Testing proper message serialization let data: ton_api::ton::bytes = rmq_message_test0.message.write_to_bytes().unwrap().into(); let msg_buffer = Message::construct_from_bytes(&data).unwrap(); - assert_eq!(*rmq_message_test0.message, msg_buffer); - // Testing body serialization + // Testing RempMessageBody structure serialization/deserailization let test0_serialized = RmqMessage::serialize_message_body(&rmq_message_test0.as_remp_message_body()); println!("Serialized message data: {:?}", data); println!("Serialized remp_message_body: {:?}", test0_serialized); let msg = RmqMessage::deserialize_message_body(&test0_serialized).unwrap(); println!("Deserialzed message data: {:?}", msg.message()); - let rmq_message = RmqMessage::from_message_body(&msg /*RmqMessage::deserialize_message_body(&msg)?*/)?; + let rmq_message = RmqMessage::from_message_body(&msg)?; + + // Testing message construction from header and body let mut message_without_params = Message::default(); message_without_params.set_header(rmq_message.message.header().clone()); message_without_params.set_body(rmq_message.message.body().unwrap()); @@ -84,26 +71,25 @@ fn test_rmq_message_serialize() -> Result<()> { #[test] fn test_rmq_message_id() -> Result<()> { - //let message_data = SliceData::from_string( - // "b5ee9c720101030100c300014589feeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee0\ - // c0101ebbb6f5976b0533dc287fea336a34e909c9d7f18018af9afd3ad0fe36ee058000f2ab5d13c159aaffb04917f\ - // 7be06ebbb58f45b14de97fe7df5edaf78847abf483554fe4d47282105f230ce104f0bf52cc3adfdb4d5a538f86e33\ - // acf55ca68d14800000063aa878d5b1983b83c1b079a4f000000006030020043d06435a826dde79131e36acc2ba1f8\ - // d92476518ecd71be9398816ee95b666cae4540_")?; - - - let message_data = SliceData::from_string(format!("{}{}{}{}", - "b5ee9c720101030100c300014589feeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee0", - "c0101ebbb6f5976b0533dc287fea336a34e909c9d7f18018af9afd3ad0fe36ee058000f2ab5d13c159aaffb04917f", - "7be06ebbb58f45b14de97fe7df5edaf78847abf483554fe4d47282105f230ce104f0bf52cc3adfdb4d5a538f86e33", - "acf55ca68d14800000063aa878d5b1983b83c1b079a4f000000006030_" //020043d06435a826dde79131e36acc2ba1f8","d92476518ecd71be9398816ee95b666cae4540" - ).as_str())?; - - let message_id = UInt256::from_str("e8da5b5c46b97ec31330a874347beae2bcac9ab2b3dbc6c3cd4deec9994e3c33")?; + let message_id = UInt256::from_str("c80d5e0e0ada0f4c74ecb03c97947e09204c4d989ac720e1754aeec6f01a4acf")?; + let message_uid = UInt256::from_str("e4089ffa779a932656bd91a9e5b7323f714fb1782c052e8755fb56312be02c83")?; + let message_bytes = [ + 181, 238, 156, 114, 1, 1, 4, 1, 0, 209, 0, 1, 69, 137, 254, 238, 238, 238, 238, 238, 238, 238, 238, 238, 238, + 238, 238, 238, 238, 238, 238, 238, 238, 238, 238, 238, 238, 238, 238, 238, 238, 238, 238, 238, 238, 238, 238, + 12, 1, 1, 225, 138, 38, 33, 171, 175, 201, 185, 142, 85, 15, 44, 87, 151, 128, 86, 56, 56, 18, 59, 118, 138, + 154, 35, 57, 76, 24, 106, 28, 34, 58, 59, 105, 58, 38, 145, 18, 99, 38, 183, 252, 21, 167, 111, 232, 24, 140, + 227, 54, 240, 143, 185, 206, 53, 134, 220, 228, 17, 238, 112, 57, 5, 129, 96, 1, 242, 26, 212, 19, 110, 243, 200, + 152, 241, 181, 102, 21, 208, 252, 108, 146, 59, 40, 199, 102, 184, 223, 73, 204, 64, 183, 116, 173, 179, 54, + 87, 34, 128, 0, 0, 99, 176, 207, 135, 51, 25, 133, 83, 224, 68, 199, 96, 179, 96, 2, 1, 99, 128, 9, 75, 218, + 166, 71, 170, 33, 189, 182, 153, 177, 109, 126, 100, 87, 19, 110, 125, 93, 129, 102, 179, 156, 141, 198, 170, + 248, 140, 147, 62, 90, 201, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 239, 171, 102, 80, 210, 228, 3, 0, 0 + ].to_vec(); - let message_bytes = message_data.write_to_bytes()?; let rmq_message = RmqMessage::from_raw_message(&message_bytes)?; assert_eq!(message_id, rmq_message.message_id); + assert_eq!(message_uid, rmq_message.message_uid); + assert_eq!(message_id, rmq_message.message.hash()?); + assert_eq!(message_uid, get_message_uid(&rmq_message.message)); Ok(()) } From 977b29d3318826869dc4fa896d29de3477935d50 Mon Sep 17 00:00:00 2001 From: Dmitry Shtukenberg Date: Wed, 10 Apr 2024 15:55:52 +0300 Subject: [PATCH 26/40] Updated remp_simple tests: messages/message bodies are now added separately --- src/validator/remp_manager.rs | 2 +- src/validator/tests/test_rmq_messages.rs | 14 +++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/validator/remp_manager.rs b/src/validator/remp_manager.rs index cd6c52b5..b3ee3238 100644 --- a/src/validator/remp_manager.rs +++ b/src/validator/remp_manager.rs @@ -636,7 +636,7 @@ impl RempCoreInterface for RempInterfaceQueues { // build message let remp_message = RmqMessage::from_raw_message(message.message())?; if message.id() != &remp_message.message_id { - fail!("Message {:x} has invalid message id {:x}, message will be ignored", + fail!("Message with computed id {:x} has different id {:x} in RempMessage struct, message will be ignored", remp_message.message_id, message.id() ); } diff --git a/src/validator/tests/test_rmq_messages.rs b/src/validator/tests/test_rmq_messages.rs index e9c6dc79..f81da049 100644 --- a/src/validator/tests/test_rmq_messages.rs +++ b/src/validator/tests/test_rmq_messages.rs @@ -196,11 +196,12 @@ impl RmqTestbench { }) } - async fn send_pending_message(&self, msg: &RempMessageWithOrigin, masterchain_seqno: u32) -> Result<()> { + async fn send_pending_message(&self, msg: &RempMessageWithOrigin, masterchain_seqno: u32) -> Result { self.message_queue.process_pending_remp_catchain_record( &msg.as_remp_catchain_record(masterchain_seqno), 0 - ).await + ).await?; + self.remp_manager.message_cache.update_message_body(Arc::new(msg.message.clone())) } fn replace_message_queue(&mut self, masterchain_range: &RangeInclusive) -> Result<()> { @@ -281,8 +282,8 @@ fn remp_simple_forwarding_test() -> Result<()> { //testbench.remp_manager.options println!("{:?}", testbench.remp_manager.message_cache.get_message_with_origin_status_cc(m1.get_message_id())); - testbench.send_pending_message(&m1, 1).await?; - testbench.send_pending_message(&m2, 1).await?; + assert_eq!(testbench.send_pending_message(&m1, 1).await?, false); // false: we've added m1, but not m2 + assert_eq!(testbench.send_pending_message(&m2, 1).await?, true); let ign = RempIgnored { level: RempMessageLevel::TonNode_RempMasterchain, block_id: blk1 }; assert_eq!(testbench.remp_manager.message_cache.get_message_status(m1.get_message_id())?.unwrap(), RempMessageStatus::TonNode_RempIgnored(ign)); @@ -327,7 +328,9 @@ fn remp_simple_collation_equal_uids_test() -> Result<()> { let pa = if m.get_message_id() == acc_id { "A" } else { "" }.to_string(); let pr = if must_be_rejected.contains(&m) { "R" } else { "" }.to_string(); println!("Pending msg: {:x} {}{}{}", m.get_message_id(), pc, pa, pr); - testbench.send_pending_message(m, testbench.message_queue.catchain_info.get_master_cc_seqno()).await?; + + // All msg ids are different, therefore body must be added + assert!(testbench.send_pending_message(m, testbench.message_queue.catchain_info.get_master_cc_seqno()).await?); } let accepted = RempAccepted { @@ -342,6 +345,7 @@ fn remp_simple_collation_equal_uids_test() -> Result<()> { 2 )?; + println!("Collecting messages for collation"); testbench.message_queue.collect_messages_for_collation().await?; for (id, _msg) in testbench.engine.collator_queue.pop_iter() { From 9d1309e6979502d77b77969f246633e0713d73dc Mon Sep 17 00:00:00 2001 From: Dmitry Shtukenberg Date: Thu, 11 Apr 2024 23:51:42 +0300 Subject: [PATCH 27/40] Modified polling of new messages --- src/validator/reliable_message_queue.rs | 69 ++++++++----------------- src/validator/remp_catchain.rs | 18 ++++--- 2 files changed, 32 insertions(+), 55 deletions(-) diff --git a/src/validator/reliable_message_queue.rs b/src/validator/reliable_message_queue.rs index eb2057a7..1d2456e7 100644 --- a/src/validator/reliable_message_queue.rs +++ b/src/validator/reliable_message_queue.rs @@ -305,14 +305,9 @@ impl MessageQueue { &self.catchain_info.general_session_info.shard, self.catchain_instance.pending_messages_queue_len()?); if self.catchain_instance.is_session_active() { - log::trace!(target: "remp", "Point 3. Activating RMQ {} processing", self); - self.catchain_instance.activate_exchange( - RMQ_REQUEST_NEW_BLOCK_INTERVAL, - RMQ_MAXIMAL_BROADCASTS_IN_PACK, - RMQ_MAXIMAL_QUERIES_IN_PACK, - RMQ_MESSAGE_QUERY_TIMEOUT, - MAX_EXTERNAL_MESSAGE_SIZE as u64 + 1000 - )?; + log::trace!(target: "remp", "Activating RMQ {} processing", self); + self.activate_exchange()?; + self.poll_outbound_queues()?; // Temporary status "New" --- message is not registered yet self.send_response_to_fullnode(msg.get_message_id(), origin_with_idx, RempMessageStatus::TonNode_RempNew); @@ -533,10 +528,24 @@ impl MessageQueue { } } + /// Check pending queues, activate catchain exchange + fn poll_outbound_queues(&self) -> Result<()> { + self.catchain_instance.poll_outbound_queues( + RMQ_MAXIMAL_BROADCASTS_IN_PACK, + RMQ_MAXIMAL_QUERIES_IN_PACK, + RMQ_MESSAGE_QUERY_TIMEOUT, + MAX_EXTERNAL_MESSAGE_SIZE as u64 + 1000 + ) + } + + fn activate_exchange(&self) -> Result<()> { + self.catchain_instance.activate_exchange(RMQ_REQUEST_NEW_BLOCK_INTERVAL) + } + /// Check received messages queue and put all received messages into /// hash map. Check status of all old messages in the hash map. pub async fn poll(&self) -> Result<()> { - log::debug!(target: "remp", "Point 4. RMQ {}: polling; total {} messages in cache, {} messages in rmq_queue", + log::debug!(target: "remp", "RMQ {}: polling; total {} messages in cache, {} messages in rmq_queue", self, self.received_messages_count().await, self.catchain_instance.rmq_catchain_receiver_len()? @@ -581,7 +590,8 @@ impl MessageQueue { } } - Ok(()) + self.poll_outbound_queues()?; + self.activate_exchange() } /// Prepare messages for collation - to be called just before collator invocation. @@ -1095,23 +1105,6 @@ impl RmqQueueManager { // All other messages are not forwarded if is_finally_rejected(&message_status) { -/* - if let Some(digest) = rejected_message_digests.get_mut (&message_cc) { - digest.messages.0.push(ton_api::ton::ton_node::rempcatchainmessageids::RempCatchainMessageIds { - id: message.message_id.clone(), - uid: message.message_uid.clone() - }); - } - else { - let mut digest = ton_api::ton::ton_node::rempcatchainrecord::RempCatchainMessageDigest::default(); - digest.masterchain_seqno = message_cc as i32; - digest.messages.0.push(ton_api::ton::ton_node::rempcatchainmessageids::RempCatchainMessageIds { - id: message.message_id.clone(), - uid: message.message_uid.clone() - }); - rejected_message_digests.insert(message_cc, digest); - } -*/ let mut digest = RempCatchainMessageDigestV2::default(); digest.masterchain_seqno = message_cc as i32; digest.messages.0.push(ton_api::ton::ton_node::rempcatchainmessageids::RempCatchainMessageIds { @@ -1146,21 +1139,6 @@ impl RmqQueueManager { } for digest in rejected_message_digests.into_iter() { -/* - if !digest.messages.0.is_empty() { - let digest_len = digest.messages.0.len(); - let msg = ton_api::ton::ton_node::RempCatchainRecord::TonNode_RempCatchainMessageDigest(digest); - for new in next_queues.iter() { - if let Err(x) = new.catchain_instance.pending_messages_queue_send(msg.clone()) { - log::error!(target: "remp", - "Point 5a. RMQ {}: message digest (len={}) cannot be put to new queue {}: `{}`", - self, digest_len, new, x - ) - } else { - sent = sent + 1; - } - } - */ let digest_len = digest.messages.0.len(); let msg = ton_api::ton::ton_node::RempCatchainRecordV2::TonNode_RempCatchainMessageDigestV2(digest); for new in next_queues.iter() { @@ -1174,11 +1152,6 @@ impl RmqQueueManager { sent_rejects = sent_rejects + 1; } } - /* - else { - log::error!(target: "remp", "RMQ {}: rejected message digest for {} is empty, but present in cache!", self, master_cc) - } - */ } log::info!(target: "remp", "RMQ {}: forwarding messages to new RMQ, total {}, actually sent {} (with {} rejects of them)", @@ -1284,7 +1257,7 @@ impl RmqQueueManager { } pub async fn poll(&self) { - log::trace!(target: "remp", "Point 2. RMQ {} manager: polling incoming messages", self); + log::trace!(target: "remp", "RMQ {} manager: polling incoming messages", self); if let Some(cur_queue) = &self.cur_queue { if !cur_queue.is_session_active() { log::warn!(target: "remp", "RMQ {} is not active yet, waiting...", self); diff --git a/src/validator/remp_catchain.rs b/src/validator/remp_catchain.rs index 7094a9ae..cb32b42d 100644 --- a/src/validator/remp_catchain.rs +++ b/src/validator/remp_catchain.rs @@ -159,7 +159,7 @@ impl RempCatchainInstance { Ok(instance.pending_messages_queue_sender.len()) } - pub fn pending_messages_queue_try_recv(&self) -> Result> { + fn pending_messages_queue_try_recv(&self) -> Result> { let instance = self.get_instance_impl()?; match instance.pending_messages_queue_receiver.try_recv() { Ok(x) => Ok(Some(x)), @@ -168,7 +168,7 @@ impl RempCatchainInstance { } } - pub fn pending_messages_broadcast_try_recv(&self) -> Result> { + fn pending_messages_broadcast_try_recv(&self) -> Result> { let instance = self.get_instance_impl()?; match instance.pending_messages_broadcast_receiver.try_recv() { Ok(x) => Ok(Some(x)), @@ -185,7 +185,7 @@ impl RempCatchainInstance { } } - pub fn pending_messages_queries_try_recv(&self) -> Result,RempMessageQuery)>> { + fn pending_messages_queries_try_recv(&self) -> Result,RempMessageQuery)>> { let instance = self.get_instance_impl()?; match instance.pending_messages_queries_receiver.try_recv() { Ok(x) => Ok(Some(x)), @@ -217,7 +217,7 @@ impl RempCatchainInstance { } } - pub fn rmq_catchain_send(&self, msg: RempCatchainRecordV2, msg_remp_source_idx: u32) -> Result<()> { + fn rmq_catchain_send(&self, msg: RempCatchainRecordV2, msg_remp_source_idx: u32) -> Result<()> { let instance = self.get_instance_impl()?; match instance.rmq_catchain_sender.send((msg, msg_remp_source_idx)) { Ok(()) => Ok(()), @@ -280,12 +280,16 @@ impl RempCatchainInstance { Ok(()) } - pub fn activate_exchange(&self, delay: Duration, max_broadcasts: u32, max_queries: u32, query_timeout: Duration, max_answer_size: u64) -> Result<()> { - let session = &self.get_instance_impl()?.catchain_ptr; - session.request_new_block(SystemTime::now() + delay); + pub fn poll_outbound_queues(&self, max_broadcasts: u32, max_queries: u32, query_timeout: Duration, max_answer_size: u64) -> Result<()> { self.poll_pending_broadcasts(max_broadcasts)?; self.poll_pending_queries(max_queries, SystemTime::now() + query_timeout, max_answer_size) } + + pub fn activate_exchange(&self, delay: Duration) -> Result<()> { + let session = &self.get_instance_impl()?.catchain_ptr; + session.request_new_block(SystemTime::now() + delay); + Ok(()) + } } impl fmt::Display for RempCatchainInstance { From 94d212fe1d28fc1ad60797e1c6d7ae78a6db3c65 Mon Sep 17 00:00:00 2001 From: Dmitry Shtukenberg Date: Tue, 16 Apr 2024 21:41:52 +0300 Subject: [PATCH 28/40] Fixed message cache bug: final to non-final status should be impossible --- src/validator/message_cache.rs | 148 ++++++++++++++++++------ src/validator/reliable_message_queue.rs | 11 +- 2 files changed, 117 insertions(+), 42 deletions(-) diff --git a/src/validator/message_cache.rs b/src/validator/message_cache.rs index 7fb2ae58..89f1513d 100644 --- a/src/validator/message_cache.rs +++ b/src/validator/message_cache.rs @@ -341,6 +341,7 @@ pub struct MessageCacheSession { messages: DashMap>, message_events: LockfreeMapSet, //Map>, message_status: DashMap, + message_finally_accepted: DashMap, blocks_processed: DashSet } @@ -450,7 +451,7 @@ impl MessageCacheSession { self.message_origins.contains_key(msg_id) && self.messages.contains_key(msg_id) { - if self.message_status.contains_key(msg_id) { + if self.get_message_status(msg_id)?.is_some() { return Ok(true); } else { @@ -477,6 +478,60 @@ impl MessageCacheSession { (self.message_headers.len(), self.messages.len(), self.message_origins.len()) } + fn alter_message_status(&self, message_id: &UInt256, status_updater: F) + -> Result<(RempMessageStatus,RempMessageStatus)> + where F: FnOnce(&RempMessageStatus) -> RempMessageStatus + { + match &mut self.message_status.get_mut(message_id) { + None => { + fail!("Changing status: no message {:x} in message cache session {}", message_id, self) + }, + Some(status) => { + let old_status = status.value().clone(); + let new_status = status_updater(&old_status); + if is_finally_accepted(&new_status) { + if let Some(prev) = self.set_finally_accepted_status(message_id, new_status.clone())? { + Ok((prev, new_status)) + } + else { + Ok((status.value().clone(), new_status)) + } + } + else { + *status.value_mut() = new_status.clone(); + Ok((old_status, status.value().clone())) + } + } + } + } + + fn update_message_status(&self, message_id: &UInt256, new_status: RempMessageStatus) -> Result<()> { + if is_finally_accepted (&new_status) { + log::trace!(target: "remp", "Setting finally accepting status for {:x}: {}", message_id, new_status); + self.set_finally_accepted_status(message_id, new_status)?; + } + else { + let old_status = self.message_status.insert(message_id.clone(), new_status.clone()); + log::trace!(target: "remp", "Changing message status for {:x}: {:?} => {}", message_id, old_status, new_status); + + if let Some(actual_status) = self.get_message_status(&message_id)? { + if is_finally_accepted(&actual_status) { + log::error!(target: "remp", "Changing status for {:x} to {}, although it has final status {}", message_id, new_status, actual_status); + } + } + } + Ok(()) + } + + fn set_finally_accepted_status(&self, message_id: &UInt256, new_status: RempMessageStatus) -> Result> { + if !is_finally_accepted(&new_status) { + fail!("Set finally accepted status for {:x}: status {} is not final.", message_id, new_status) + } + let old = self.message_finally_accepted.insert(message_id.clone(), new_status); + Ok(old) + } + +/* fn update_message_status(&self, message_id: &UInt256, new_status: RempMessageStatus) -> Result { match self.message_status.insert(message_id.clone(), new_status.clone()) { None => fail!("Changing status to {}: no message {:x} in message cache session {}", new_status, message_id, self), @@ -486,18 +541,22 @@ impl MessageCacheSession { } } } - - fn alter_message_status(&self, message_id: &UInt256, status_updater: F) - -> Result<(RempMessageStatus,RempMessageStatus)> - where F: FnOnce(&RempMessageStatus) -> RempMessageStatus - { - match &mut self.message_status.get_mut(message_id) { - None => fail!("Changing status: no message {:x} in message cache session {}", message_id, self), - Some(status) => { - let old_status = status.value().clone(); - *status.value_mut() = status_updater(&old_status); - Ok((old_status, status.value().clone())) + */ + fn get_message_status(&self, message_id: &UInt256) -> Result> { + if let Some(status) = self.message_finally_accepted.get(message_id) { + if !is_finally_accepted(status.value()) { + fail!("Only finally accepted status may be stored in message_finally_accepted, for id {:x} found {}", message_id, status.value()) + } + Ok(Some(status.value().clone())) + } + else if let Some(status) = self.message_status.get(message_id) { + if is_finally_accepted(status.value()) { + fail!("No finally accepted status may be stored in message_status, for id {:x} found {}", message_id, status.value()) } + Ok(Some(status.value().clone())) + } + else { + Ok(None) } } @@ -518,10 +577,14 @@ impl MessageCacheSession { .get(message_id).map(|x| format!("uid: {:x}", x.value().message_uid)) .unwrap_or_else(|| "*error: no header in cache*".to_owned()); - let status = self.message_status - .get(message_id) - .map(|x| format!("{:?}", x.value())) - .unwrap_or_else(|| "*error: no status in cache*".to_owned()); + let status1 = self.message_status.get(message_id).map(|x| x.value().clone()); + let status2 = self.message_finally_accepted.get(message_id).map(|x| x.value().clone()); + let status = match (status1, status2) { + (Some(x), None) => format!("{:?}/**", x), + (None, Some(x)) => format!("**/{:?}", x), + (Some(x), Some(y)) => format!("{:?}/{:?}", x, y), + (None, None) => "*error: no status in cache*".to_owned() + }; let has_additional_info = self.messages.contains_key(message_id) || self.message_origins.contains_key(message_id) @@ -564,16 +627,25 @@ impl MessageCacheSession { log::debug!(target: "remp", "Removing old message: {}", self.message_info(&id)); - match (self.messages.get(&id), self.message_status.get(&id)) { + let message_status = match self.get_message_status(&id) { + Err(e) => { + log::error!(target: "remp", "Record for message {:?} is incorrect: err: {}", id, e); + stats.incorrect += 1; + continue; + } + Ok(s) => s + }; + + match (self.messages.get(&id), message_status) { (Some(_m),Some(status)) => { - if is_finally_accepted(status.value()) { stats.accepted_in_session += 1 } - else if is_finally_rejected(status.value()) { stats.rejected_in_session += 1 } + if is_finally_accepted(&status) { stats.accepted_in_session += 1 } + else if is_finally_rejected(&status) { stats.rejected_in_session += 1 } }, (None,Some(_status)) => stats.has_only_header += 1, (m, h) => { log::error!(target: "remp", "Record for message {:?} is in incorrect state: msg = {:?}, status = {:?}", - id, m.map(|x| x.value().clone()), h.map(|x| x.value().clone()) + id, m.map(|x| x.value().clone()), h.map(|x| x.clone()) ); stats.incorrect += 1 } @@ -592,6 +664,7 @@ impl MessageCacheSession { message_events: LockfreeMapSet::default(), messages: DashMap::default(), message_status: DashMap::default(), + message_finally_accepted: DashMap::default(), inf_shards: HashSet::from_iter(inf_shards.into_iter()), blocks_processed: DashSet::default(), } @@ -636,21 +709,13 @@ impl MessageCache { (result, with_bodies, with_origins) } - /// Returns new message status, if it worths reporting (final statuses do not need to be reported) - pub fn update_message_status(&self, message_id: &UInt256, new_status: RempMessageStatus) -> Result> { + /// Updates message status: changes status to the given value + pub fn update_message_status(&self, message_id: &UInt256, new_status: RempMessageStatus) -> Result<()> { let session = self.get_session_for_message(message_id).ok_or_else( || error!("Cannot find message {:x} to change its status to {:?}", message_id, new_status) )?; - if let RempMessageStatus::TonNode_RempAccepted(acc_new) = &new_status { - if acc_new.level == RempMessageLevel::TonNode_RempMasterchain { - session.update_message_status(message_id, new_status.clone())?; - return Ok(None) - } - } - - session.update_message_status(message_id, new_status.clone())?; - Ok(Some(new_status)) + session.update_message_status(message_id, new_status) } fn get_session_for_message(&self, message_id: &UInt256) -> Option> { @@ -693,10 +758,7 @@ impl MessageCache { pub fn get_message_status(&self, message_id: &UInt256) -> Result> { match self.get_session_for_message(message_id) { None => Ok(None), - Some(s) => - Ok(Some(s.message_status.get(message_id).ok_or_else(|| error!("No status for message {:x}, {}", - message_id, s - ))?.value().clone())) + Some(s) => s.get_message_status(message_id) } } @@ -720,7 +782,7 @@ impl MessageCache { session.message_headers.get(message_id).ok_or_else(|| error!("Message {:x} has no header", message_id))?.value().clone(), session.messages.get(message_id).map(|m| m.value().clone()), session.message_origins.get(message_id).map(|m| m.value().clone()), - session.message_status.get(message_id).map(|m| m.value().clone()) + session.get_message_status(message_id)? ); match (msg, origin, status) { @@ -764,7 +826,12 @@ impl MessageCache { // fail!("Inconsistent message cache contents: message {} present in cache, although should not", message_id) //} - session.message_status.insert(message_id.clone(), status.clone()); + if is_finally_accepted(status) { + session.message_finally_accepted.insert(message_id.clone(), status.clone()); + } + else { + session.message_status.insert(message_id.clone(), status.clone()); + } session.insert_message(message, message_header, message_origin) } @@ -774,7 +841,12 @@ impl MessageCache { // fail!("Inconsistent message cache contents: message header {:x} present in cache, although should not", message_id) //} - session.message_status.insert(message_id.clone(), status.clone()); + if is_finally_accepted(status) { + session.message_finally_accepted.insert(message_id.clone(), status.clone()); + } + else { + session.message_status.insert(message_id.clone(), status.clone()); + } session.insert_message_header(&message_id, message_header, message_origin)?; Ok(()) } diff --git a/src/validator/reliable_message_queue.rs b/src/validator/reliable_message_queue.rs index 1d2456e7..b846d99f 100644 --- a/src/validator/reliable_message_queue.rs +++ b/src/validator/reliable_message_queue.rs @@ -204,14 +204,17 @@ impl MessageQueue { } pub fn update_status_send_response(&self, msgid: &UInt256, origin: Arc, new_status: RempMessageStatus) { - match self.remp_manager.message_cache.update_message_status(&msgid, new_status.clone()) { - Ok(Some(final_status)) => self.send_response_to_fullnode(msgid, origin, final_status), - Ok(None) => (), // Send nothing, no status update is requested - Err(e) => log::error!(target: "remp", + if let Err(e) = self.remp_manager.message_cache.update_message_status(&msgid, new_status.clone()) { + log::error!(target: "remp", "RMQ {}: Cannot update status for {:x}, new status {}, error {}", self, msgid, new_status, e ) } + else { + if !is_finally_accepted(&new_status) { + self.send_response_to_fullnode(msgid, origin, new_status) + } + } } pub async fn update_status_send_response_by_id(&self, msgid: &UInt256, new_status: RempMessageStatus) -> Result> { From cf236c2e10fcef713f957c317f68017679faad95 Mon Sep 17 00:00:00 2001 From: Sumrachek Date: Tue, 23 Apr 2024 10:10:16 +0400 Subject: [PATCH 29/40] Load cells from storing cells cache by id --- CHANGELOG.md | 4 ++++ Cargo.toml | 2 +- storage/src/dynamic_boc_rc_db.rs | 28 +++++++++++++++++----------- storage/src/types/storage_cell.rs | 8 ++++---- 4 files changed, 26 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e50a7fa9..a75e0843 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ All notable changes to this project will be documented in this file. +## Version 0.58.4 + +Load cells from storing cells cache by id + ## Version 0.58.3 - Minor fixes related to renaming diff --git a/Cargo.toml b/Cargo.toml index 7e1f2d72..f3977928 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ build = 'common/build/build.rs' edition = '2021' name = 'ever-node' -version = '0.58.3' +version = '0.58.4' [workspace] members = [ 'storage' ] diff --git a/storage/src/dynamic_boc_rc_db.rs b/storage/src/dynamic_boc_rc_db.rs index c1edd70e..fb368cd1 100644 --- a/storage/src/dynamic_boc_rc_db.rs +++ b/storage/src/dynamic_boc_rc_db.rs @@ -301,9 +301,7 @@ impl DynamicBocDb { // Is thread-safe pub fn load_boc(self: &Arc, root_cell_id: &UInt256, use_cache: bool) -> Result { - let storage_cell = self.load_cell(root_cell_id, use_cache)?; - - Ok(Cell::with_cell_impl_arc(storage_cell)) + self.load_cell(root_cell_id, use_cache) } pub fn check_and_update_cells(&mut self) -> Result<()> { @@ -454,7 +452,7 @@ impl DynamicBocDb { self: &Arc, cell_id: &UInt256, use_cache: bool, - ) -> Result> { + ) -> Result { let deserialize = |value: &[u8]| { StorageCell::deserialize(self, value, use_cache, false).or_else( @@ -469,7 +467,7 @@ impl DynamicBocDb { target: TARGET, "DynamicBocDb::load_cell from cache id {cell_id:x}" ); - return Ok(Arc::new(cell)) + return Ok(Cell::with_cell_impl(cell)) }, Err(e) => log::trace!( target: TARGET, @@ -481,6 +479,15 @@ impl DynamicBocDb { let storage_cell_data = match self.db.get(cell_id) { Ok(data) => data, Err(e) => { + + if let Some(guard) = self.storing_cells.get(cell_id) { + log::error!( + target: TARGET, + "DynamicBocDb::load_cell from storing_cells by id {cell_id:x}", + ); + return Ok(guard.val().clone()); + } + log::error!("FATAL!"); log::error!("FATAL! Can't load cell {:x} from db, error: {:?}", cell_id, e); log::error!("FATAL!"); @@ -519,7 +526,7 @@ impl DynamicBocDb { "DynamicBocDb::load_cell from DB id {cell_id:x} use_cache {use_cache}" ); - Ok(storage_cell) + Ok(Cell::with_cell_impl_arc(storage_cell)) } pub(crate) fn allocated(&self) -> &StorageAlloc { @@ -819,7 +826,7 @@ impl DoneCellsStorage for DoneCellsStorageAdapter { fn get(&self, index: u32) -> Result { let id = UInt256::from_slice(self.index.get(&index.into())?.as_ref()).into(); - Ok(Cell::with_cell_impl_arc(self.boc_db.clone().load_cell(&id, false)?)) + Ok(self.boc_db.clone().load_cell(&id, false)?) } fn cleanup(&mut self) -> Result<()> { @@ -849,8 +856,7 @@ impl CellByHashStorageAdapter { impl CellByHashStorage for CellByHashStorageAdapter { fn get_cell_by_hash(&self, hash: &UInt256) -> Result { - let cell = Cell::with_cell_impl_arc(self.boc_db.clone().load_cell(&hash, self.use_cache)?); - Ok(cell) + self.boc_db.clone().load_cell(&hash, self.use_cache) } } @@ -920,7 +926,7 @@ impl OrderedCellsStorage for OrderedCellsStorageAdapter { fn get_cell_by_index(&self, index: u32) -> Result { let id = UInt256::from_slice(self.index1.get(&index.into())?.as_ref()).into(); - let cell = Cell::with_cell_impl_arc(self.boc_db.clone().load_cell(&id, false)?); + let cell = self.boc_db.clone().load_cell(&id, false)?; let slowdown = self.slowdown(); if index % 1000 == 0 { @@ -1011,4 +1017,4 @@ impl RawCellsCache { Ok(value) } -} +} \ No newline at end of file diff --git a/storage/src/types/storage_cell.rs b/storage/src/types/storage_cell.rs index 66707572..81a24e48 100644 --- a/storage/src/types/storage_cell.rs +++ b/storage/src/types/storage_cell.rs @@ -198,17 +198,17 @@ impl StorageCell { }; let boc_db = self.boc_db.upgrade().ok_or_else(|| error!("BocDb is dropped"))?; - let storage_cell = boc_db.load_cell( + let cell = boc_db.load_cell( &hash, self.use_cache )?; if self.use_cache { self.references.write()[index].cell = - Some(Arc::downgrade(&storage_cell) as Weak); + Some(Arc::downgrade(cell.cell_impl()) as Weak); } - Ok(storage_cell) + Ok(cell.cell_impl().clone()) } } @@ -287,4 +287,4 @@ impl PartialEq for StorageCell { fn eq(&self, other: &Self) -> bool { self.cell_data.raw_hash(MAX_LEVEL) == other.cell_data.raw_hash(MAX_LEVEL) } -} +} \ No newline at end of file From 5fa7882ffd62ee0041ac056408a5198ce4f67234 Mon Sep 17 00:00:00 2001 From: Dmitry Shtukenberg Date: Thu, 2 May 2024 14:07:20 +0300 Subject: [PATCH 30/40] First version with collator queue interface --- src/engine_traits.rs | 7 +- src/validator/collator.rs | 2 + src/validator/fabric.rs | 15 +- src/validator/reliable_message_queue.rs | 307 ++++++++++++++++++++---- src/validator/remp_service.rs | 4 +- src/validator/validator_group.rs | 22 +- 6 files changed, 291 insertions(+), 66 deletions(-) diff --git a/src/engine_traits.rs b/src/engine_traits.rs index 8bfe8dd2..83d4a4bc 100644 --- a/src/engine_traits.rs +++ b/src/engine_traits.rs @@ -43,7 +43,7 @@ use ever_block::{ Deserializable, KeyId, KeyOption, MASTERCHAIN_ID, Message, OutMsgQueue, Result, SHARD_FULL, ShardAccount, ShardIdent, UInt256 }; -use std::{collections::HashSet, sync::{Arc, atomic::AtomicU64}}; +use std::{collections::HashSet, sync::{Arc, atomic::AtomicU64}, time::SystemTime}; use storage::{StorageAlloc, block_handle_db::BlockHandle}; #[cfg(feature = "telemetry")] use storage::StorageTelemetry; @@ -971,3 +971,8 @@ pub trait RempCoreInterface: Sync + Send { async fn process_incoming_message(&self, message: &RempMessage, source: Arc) -> Result<()>; fn check_remp_duplicate(&self, message_id: &UInt256) -> Result; } + +#[async_trait::async_trait] +pub trait RempQueueCollatorInterface : Send + Sync { + async fn get_next_message_for_collation(&self, master_block_id: &BlockIdExt, generation_deadline: SystemTime) -> Result>>; +} diff --git a/src/validator/collator.rs b/src/validator/collator.rs index f484b182..952688b1 100644 --- a/src/validator/collator.rs +++ b/src/validator/collator.rs @@ -65,6 +65,7 @@ use ever_executor::{ TransactionExecutor, }; use ever_block::{error, fail, AccountId, Cell, HashmapType, Result, UInt256, UsageTree, SliceData}; +use crate::engine_traits::RempQueueCollatorInterface; use crate::validator::validator_utils::is_remp_enabled; @@ -1017,6 +1018,7 @@ impl Collator { created_by: UInt256, engine: Arc, rand_seed: Option, + remp_collator_interface: Option>, collator_settings: CollatorSettings ) -> Result { diff --git a/src/validator/fabric.rs b/src/validator/fabric.rs index f79b805f..898e6f2c 100644 --- a/src/validator/fabric.rs +++ b/src/validator/fabric.rs @@ -16,15 +16,16 @@ use std::{ time::SystemTime, }; use super::validator_utils::{ - validator_query_candidate_to_validator_block_candidate, pairvec_to_cryptopair_vec, - get_first_block_seqno_after_prevs + get_first_block_seqno_after_prevs, + pairvec_to_cryptopair_vec, + validator_query_candidate_to_validator_block_candidate, }; use crate::{ - collator_test_bundle::CollatorTestBundle, engine_traits::EngineOperations, - validator::{CollatorSettings, validate_query::ValidateQuery, collator}, - validating_utils::{fmt_next_block_descr_from_next_seqno, fmt_next_block_descr} + collator_test_bundle::CollatorTestBundle, + engine_traits::{EngineOperations, RempQueueCollatorInterface}, + validating_utils::{fmt_next_block_descr_from_next_seqno, fmt_next_block_descr}, + validator::{CollatorSettings, validate_query::ValidateQuery, collator, verification::VerificationManagerPtr} }; -use crate::validator::verification::VerificationManagerPtr; use ever_block::{Block, BlockIdExt, Deserializable, Result, ShardIdent, UInt256, ValidatorSet}; use validator_session::{ValidatorBlockCandidate, BlockPayloadPtr, PublicKeyHash, PublicKey}; @@ -211,6 +212,7 @@ pub async fn run_collate_query ( _min_ts: SystemTime, min_mc_seqno: u32, prev: Vec, + remp_collator_interface: Option>, collator_id: PublicKey, set: ValidatorSet, engine: Arc, @@ -231,6 +233,7 @@ pub async fn run_collate_query ( UInt256::from(collator_id.pub_key()?), engine.clone(), None, + remp_collator_interface, CollatorSettings::default() )?; let collator_result = collator.collate().await; diff --git a/src/validator/reliable_message_queue.rs b/src/validator/reliable_message_queue.rs index ac4c7f27..8966a122 100644 --- a/src/validator/reliable_message_queue.rs +++ b/src/validator/reliable_message_queue.rs @@ -15,9 +15,9 @@ use std::{ cmp::Reverse, collections::{BinaryHeap, HashMap}, fmt, fmt::Formatter, - ops::RangeInclusive, + ops::{Add, RangeInclusive}, sync::Arc, - time::Duration + time::{Duration, SystemTime} }; use dashmap::DashMap; @@ -28,12 +28,12 @@ use ton_api::ton::ton_node::{ RempCatchainRecordV2, rempcatchainrecordv2::{RempCatchainMessageHeaderV2, RempCatchainMessageDigestV2} }; -use ever_block::{ShardIdent, Message, BlockIdExt, ValidatorDescr}; +use ever_block::{ShardIdent, Message, BlockIdExt, ValidatorDescr, UnixTime32}; use ever_block::{UInt256, Result, fail, gen_random_index, error}; use catchain::{PrivateKey, PublicKey}; use crate::{ - engine_traits::EngineOperations, + engine_traits::{EngineOperations, RempDuplicateStatus, RempQueueCollatorInterface}, ext_messages::{get_level_and_level_change, is_finally_accepted, is_finally_rejected, MAX_EXTERNAL_MESSAGE_SIZE}, validator::{ mutex_wrapper::MutexWrapper, @@ -47,7 +47,6 @@ use crate::{ }; use failure::err_msg; use crate::block::BlockIdExtExtention; -use crate::engine_traits::RempDuplicateStatus; #[derive(Debug,PartialEq,Eq,PartialOrd,Ord,Clone)] enum MessageQueueStatus { Created, Starting, Active, Stopping } @@ -153,7 +152,7 @@ impl MessageQueue { }, format!("<>", remp_catchain_instance), #[cfg(feature = "telemetry")] - engine.remp_core_telemetry().rmq_catchain_mutex_metric(&remp_catchain_info.general_session_info.shard), + engine.remp_core_telemetry().rmq_catchain_mutex_metric(&remp_catchain_info.general_session_info.shard), ); log::trace!(target: "remp", "Creating MessageQueue {}", remp_catchain_instance); @@ -167,7 +166,7 @@ impl MessageQueue { }); } - pub async fn create_and_start ( + pub async fn create_and_start( engine: Arc, manager: Arc, info: Arc, local_key: PrivateKey ) -> Result> { let queue = Arc::new(Self::create(engine, manager, info)?); @@ -179,8 +178,7 @@ impl MessageQueue { self.queues.execute_sync(|q| { if q.status != required_status { fail!("RMQ {}: MessageQueue status is {:?}, but required to be {:?}", self, q.status, required_status) - } - else { + } else { q.status = new_status; Ok(()) } @@ -194,8 +192,7 @@ impl MessageQueue { if origin.has_no_source_key() { log::trace!(target: "remp", "RMQ {}: message {:x} was broadcast and has no source key, no response", self, message_id) - } - else if let Err(e) = self.remp_manager.queue_response_to_fullnode( + } else if let Err(e) = self.remp_manager.queue_response_to_fullnode( self.catchain_info.local_key_id.clone(), message_id.clone(), origin.clone(), status.clone() ) { log::error!(target: "remp", "RMQ {}: cannot queue response to fullnode: message id {} from {}, {}, local key {:x}, error `{}`", @@ -210,8 +207,7 @@ impl MessageQueue { "RMQ {}: Cannot update status for {:x}, new status {}, error {}", self, msgid, new_status, e ) - } - else { + } else { if !is_finally_accepted(&new_status) { self.send_response_to_fullnode(msgid, origin, new_status) } @@ -220,11 +216,11 @@ impl MessageQueue { pub async fn update_status_send_response_by_id(&self, msgid: &UInt256, new_status: RempMessageStatus) -> Result> { match &self.remp_manager.message_cache.get_message_with_origin_status_cc(msgid)? { - Some((Some(rm),_header,origin,_,_)) => { + Some((Some(rm), _header, origin, _, _)) => { self.update_status_send_response(msgid, origin.clone(), new_status.clone()); Ok(rm.clone()) }, - Some((None,_,_,_,_)) | None => + Some((None, _, _, _, _)) | None => fail!( "RMQ {}: cannot find message {:x} body in RMQ messages hash! (new status {})", self, msgid, new_status @@ -232,10 +228,10 @@ impl MessageQueue { } } - pub async fn start (self: Arc, local_key: PrivateKey) -> Result<()> { + pub async fn start(self: Arc, local_key: PrivateKey) -> Result<()> { self.set_queue_status(MessageQueueStatus::Created, MessageQueueStatus::Starting).await?; log::trace!(target: "remp", "RMQ {}: starting", self); - + let catchain_instance_res = self.remp_manager.catchain_store.start_catchain( self.engine.clone(), self.remp_manager.clone(), self.catchain_info.clone(), local_key ).await; @@ -259,10 +255,16 @@ impl MessageQueue { loop { let (do_stop, do_break) = self.queues.execute_sync(|q| { match q.status { - MessageQueueStatus::Created => { q.status = MessageQueueStatus::Stopping; Ok((false, true)) }, + MessageQueueStatus::Created => { + q.status = MessageQueueStatus::Stopping; + Ok((false, true)) + }, MessageQueueStatus::Starting => Ok((false, false)), MessageQueueStatus::Stopping => fail!("RMQ {}: cannot stop queue with 'Stopping' status", self), - MessageQueueStatus::Active => { q.status = MessageQueueStatus::Stopping; Ok((true, true)) }, + MessageQueueStatus::Active => { + q.status = MessageQueueStatus::Stopping; + Ok((true, true)) + }, } }).await?; @@ -296,8 +298,7 @@ impl MessageQueue { let msg_body = if body_updated { Some(msg.as_remp_message_body()) - } - else { None }; + } else { None }; self.catchain_instance.pending_messages_broadcast_send( msg.as_remp_catchain_record(self.catchain_info.get_master_cc_seqno()), @@ -316,8 +317,7 @@ impl MessageQueue { // Temporary status "New" --- message is not registered yet self.send_response_to_fullnode(msg.get_message_id(), origin_with_idx, RempMessageStatus::TonNode_RempNew); Ok(()) - } - else { + } else { log::error!(target: "remp", "RMQ {} not started", self); Err(failure::err_msg("RMQ is not started")) } @@ -332,7 +332,7 @@ impl MessageQueue { #[cfg(feature = "telemetry")] self.engine.remp_core_telemetry().pending_collation( - &self.catchain_info.general_session_info.shard, + &self.catchain_info.general_session_info.shard, _len ); @@ -411,8 +411,7 @@ impl MessageQueue { let status_if_new = if let Some((_msg, status)) = self.is_queue_overloaded().await { status - } - else { + } else { if forwarded { Self::forwarded_ignored_status().clone() } else { RempMessageStatus::TonNode_RempNew } }; @@ -432,8 +431,7 @@ impl MessageQueue { |old_status, new_status| { if Self::is_forwarded_status(old_status) { return new_status.clone() - } - else if !Self::is_final_status(old_status) && forwarded { + } else if !Self::is_final_status(old_status) && forwarded { // If the message status is non-negative (that is, not reject, timeout, etc) // And the message is non final, then at least we have not rejected it. // So, we specify non-forwarding Ignored status, giving us @@ -469,7 +467,7 @@ impl MessageQueue { self, remp_message_header, e ); }, - Ok((Some(_),new_status,_body_updated)) if Self::is_final_status(&new_status) => { + Ok((Some(_), new_status, _body_updated)) if Self::is_final_status(&new_status) => { log::trace!(target: "remp", "Point 4. RMQ {}. Message {:x} master_cc_seqno {} from validator {} has final status {}, skipping", self, remp_message_header.message_id, message_master_seqno, remp_message_origin.source_idx, new_status @@ -477,7 +475,7 @@ impl MessageQueue { #[cfg(feature = "telemetry")] self.engine.remp_core_telemetry().add_to_cache_attempt(false); } - Ok((old_status,new_status,_body_updated)) => { + Ok((old_status, new_status, _body_updated)) => { log::trace!(target: "remp", "Point 4. RMQ {}. Message {:x} master_cc_seqno {} from validator {} has non-final status {}{}, will be collated", self, remp_message_header.message_id, message_master_seqno, remp_message_origin.source_idx, new_status, @@ -500,8 +498,7 @@ impl MessageQueue { "Point 4. RMQ {}. Message digest (masterchain_seqno = {}, len = {}) does not fit to RMQ master cc range {}", self, reject_digest.masterchain_seqno, reject_digest.messages.len(), self.catchain_info.master_cc_range_info() ) - } - else { + } else { log::info!(target: "remp", "Point 4. RMQ {}. Message digest (masterchain_seqno = {}, len = {}) received", self, reject_digest.masterchain_seqno, reject_digest.messages.len() @@ -515,7 +512,7 @@ impl MessageQueue { // to grant re-collation for the message, then this knowledge is // more important than some particular reject opinion of some other // validator. - |old,_new| { old.clone() }, + |old, _new| { old.clone() }, reject_digest.masterchain_seqno as u32 )?; } @@ -526,7 +523,7 @@ impl MessageQueue { async fn process_pending_remp_catchain_record(&self, remp_catchain_record: &RempCatchainRecordV2, relayer: u32) -> Result<()> { match remp_catchain_record { RempCatchainRecordV2::TonNode_RempCatchainMessageHeaderV2(msg) => - self.process_pending_remp_catchain_message(msg,relayer).await, + self.process_pending_remp_catchain_message(msg, relayer).await, RempCatchainRecordV2::TonNode_RempCatchainMessageDigestV2(digest) => self.process_pending_remp_catchain_digest(digest).await } @@ -572,7 +569,7 @@ impl MessageQueue { log::trace!(target: "remp", "RMQ {}: No more messages in rmq_queue", self); break 'queue_loop; }, - Ok(Some((record,relayer))) => self.process_pending_remp_catchain_record(&record,relayer).await? + Ok(Some((record, relayer))) => self.process_pending_remp_catchain_record(&record, relayer).await? } } @@ -597,12 +594,185 @@ impl MessageQueue { self.poll_outbound_queues()?; self.activate_exchange() } +} + +impl MessageQueue { + /// Check message whether it is good for collation: + /// * Bool result --- should we reinsert it again into collation queue or not + /// * + async fn check_one_message_for_collation (&self, msg_id: &UInt256) -> Result<(bool, Option<(Arc, Arc)>)> { + let (message, origin, status) = match self.remp_manager.message_cache.get_message_with_origin_status_cc(msg_id)? { + Some((None, h, o, s, _cc)) => { + // -1 for absent index seems to be the easiest way to get around borrow/scope checker + let dst_idx = self.queues.execute_sync(|x| + match x.get_body_sources(&msg_id) { + None => -1, + Some(v) if v.len() == 0 => -1, + Some(v) => v.get(gen_random_index(v.len() as u16) as usize).map(|nn| *nn as i32).unwrap_or(-1) + } + ).await; + + if dst_idx >= 0 { + log::trace!( + target: "remp", + "Point 5. RMQ {}: message {:x} with status {:?} from {} found in pending_collation queue, but has no body yet; requesting body again from node {}", + self, msg_id, s, o, dst_idx + ); + + let dst_adnl_id = self.catchain_instance.get_adnl_id(dst_idx as usize).ok_or_else( + || error!("RMQ {}: cannot find adnl id for idx {}", self, dst_idx) + )?; + self.catchain_instance.pending_messages_queries_send( + dst_adnl_id, + h.as_remp_message_query() + )?; + } + else { + log::error!(target: "remp", "Point 5. RMQ {}: message {:x} with status {:?} from {} has no body and no body source; skipping", + self, msg_id, s, o + ) + } + return Ok((true, None)) + }, + None => { + log::warn!( + target: "remp", + "Point 5. RMQ {}: message {:x} found in pending_collation queue, but has no origin; will not be collated", + self, msg_id + ); + return Ok((true, None)) + }, + Some((Some(m), _h, o, s, _cc)) => (m, o, s) + }; + + match status.clone() { + RempMessageStatus::TonNode_RempNew + | RempMessageStatus::TonNode_RempIgnored(_) => (), + _ => { + log::trace!(target: "remp", "Point 5. RMQ {}: Skipping message {:x}, status does not allow it to collate: {}", + self, msg_id, status + ); + return Ok((false, None)) + } + }; + + match self.remp_manager.message_cache.check_message_duplicates(&message.message_id)? { + RempDuplicateStatus::Absent => fail!("Message {:x} is present in cache, but check_message_duplicates = Absent", &message.message_id), + RempDuplicateStatus::Fresh(_) => + log::trace!(target: "remp", "Point 5. RMQ {}: sending message {:x} to collator queue", self, message.message_id), + + d @ RempDuplicateStatus::Duplicate(_, _, _) => { + let duplicate_info = self.remp_manager.message_cache.duplicate_info(&d); + if !Self::is_final_status(&status) { + log::trace!(target: "remp", "Point 5. RMQ {}: rejecting message {:x}: '{}'", + self, message.message_id, duplicate_info + ); + let rejected = RempRejected { + level: RempMessageLevel::TonNode_RempQueue, + block_id: BlockIdExt::default(), + error: duplicate_info + }; + self.update_status_send_response(msg_id, origin.clone(), RempMessageStatus::TonNode_RempRejected(rejected)); + } + else { + log::error!(target: "remp", "Point 5. RMQ {}: message {:x} must not have a final status {:?}", + self, message.message_id, status + ) + } + return Ok((false, None)) + } + } + + Ok((false, Some((message.message.clone(), origin)))) + } + + async fn put_back_to_collation_queue(&self, msgid: &UInt256, timestamp: u32) -> Result<()> { + let (added, _) = self.queues.execute_sync(|x| + x.add_to_collation_queue(&msgid, timestamp, None, false) + ).await?; + + if added { + fail!("Message {:x} is already waiting for collation while postponing it", msgid); + } + + Ok(()) + } + + pub async fn get_one_message_for_collation(&self, message_deadline: UnixTime32, generation_deadline: SystemTime) -> Result, Arc)>> { + while let Some((msgid, timestamp)) = self.queues.execute_sync(|x| x.take_first_for_collation()).await? { + if let Ok(x) = generation_deadline.elapsed() { + log::trace!(target: "remp", "RMQ {}: Message collation deadline expired {} ms ago", self, x.as_millis()); + return Ok(None) + } + + if message_deadline.as_u32() <= timestamp { + self.put_back_to_collation_queue(&msgid, timestamp).await?; + return Ok(None) + } + + let (message, origin) = match self.check_one_message_for_collation(&msgid).await { + Err(e) => { + log::error!( + target: "remp", + "Point 5. RMQ {}: message {:x} found in pending_collation queue, error retriving it from messages/message_statuses: {}", + self, msgid, e + ); + continue + }, + Ok((leave_in_queue, None)) => { + if leave_in_queue { + self.put_back_to_collation_queue(&msgid, UnixTime32::now().as_u32()).await?; + } + continue + }, + Ok((_, Some(x))) => x + }; + + let new_status = RempMessageStatus::TonNode_RempAccepted (RempAccepted { + level: RempMessageLevel::TonNode_RempQueue, + block_id: BlockIdExt::default(), + master_id: BlockIdExt::default() + }); + self.update_status_send_response(&msgid, origin.clone(), new_status); + + return Ok(Some((msgid,message,origin))); + } + return Ok(None) + } /// Prepare messages for collation - to be called just before collator invocation. pub async fn collect_messages_for_collation (&self) -> Result<()> { log::trace!(target: "remp", "RMQ {}: collecting messages for collation", self); let mut cnt = 0; - while let Some((msgid, _timestamp)) = self.queues.execute_sync(|x| x.take_first_for_collation()).await? { + let message_deadline = UnixTime32::now(); + let deadline = SystemTime::now().add(Duration::from_millis(1000)); + + while let Some((msg_id,message,_origin)) = self.get_one_message_for_collation(message_deadline, deadline).await? { + //self.queues.execute_sync(|x| x.take_first_for_collation()).await? { +/* + let (message, origin) = match self.check_one_message_for_collation(&msgid).await { + Err(e) => { + log::error!( + target: "remp", + "Point 5. RMQ {}: message {:x} found in pending_collation queue, error retriving it from messages/message_statuses: {}", + self, msgid, e + ); + continue + }, + Ok((leave_in_queue, None)) => { + if leave_in_queue { + let (added, _) = self.queues.execute_sync(|x| + x.add_to_collation_queue(&msgid, UnixTime32::now().as_u32(), None, false) + ).await?; + if added { + fail!("Message {:x} is already waiting for collation while postponing it", msgid); + } + } + continue + }, + Ok((_, Some(x))) => x + }; + let (message, origin, status) = match self.remp_manager.message_cache.get_message_with_origin_status_cc(&msgid) { Err(e) => { log::error!( @@ -691,27 +861,16 @@ impl MessageQueue { continue } } - - match self.send_message_to_collator(msgid.clone(), message.message.clone()).await { +*/ + match self.send_message_to_collator(msg_id.clone(), message.clone()).await { Err(e) => { let error = format!("{}", e); log::error!(target: "remp", - "Point 5. RMQ {}: error sending message {:x} to collator: `{}`; message status is unknown till collation end", - self, msgid, &error + "Point 5. RMQ {}: error sending message {:x} to collator: `{}`", + self, msg_id, &error ); - - // No new status: failure inside collator does not say - // anything about the message. Let's wait till collation end. }, - Ok(()) => { - let new_status = RempMessageStatus::TonNode_RempAccepted (RempAccepted { - level: RempMessageLevel::TonNode_RempQueue, - block_id: BlockIdExt::default(), - master_id: BlockIdExt::default() - }); - self.update_status_send_response(&msgid, origin.clone(), new_status); - cnt = cnt + 1; - } + Ok(()) => cnt = cnt + 1 } } log::trace!(target: "remp", "Point 5. RMQ {}: total {} messages for collation", self, cnt); @@ -1371,6 +1530,50 @@ impl fmt::Display for RmqQueueManager { } } +pub struct RempQueueCollatorInterfaceImpl { + queue: Arc, + message_deadline: UnixTime32 +} + +impl RempQueueCollatorInterfaceImpl { + pub fn new(remp_manager: Arc) -> Self { + Self { queue: remp_manager, message_deadline: UnixTime32::now() } + } +} + +impl fmt::Display for RempQueueCollatorInterfaceImpl { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.queue) + } +} + +#[async_trait::async_trait] +impl RempQueueCollatorInterface for RempQueueCollatorInterfaceImpl { + async fn get_next_message_for_collation(&self, master_block_id: &BlockIdExt, generation_deadline: SystemTime) -> Result>> { + let reason = if self.queue.remp_manager.message_cache.is_block_processed(master_block_id)? { + if let Some(q) = &self.queue.cur_queue { + return Ok(q.get_one_message_for_collation(self.message_deadline, generation_deadline) + .await? + .map(|(_id,m,_o)| m) + ); + } + else { + "no current queue is active" + } + } + else { + "block is not processed yet" + }; + + log::warn!(target: "remp", + "RMQ {}: collator request relative to master block {} cannot be processed: {}", + self, master_block_id, reason + ); + + return Ok(None) + } +} + #[cfg(test)] #[path = "tests/test_rmq_messages.rs"] mod tests; diff --git a/src/validator/remp_service.rs b/src/validator/remp_service.rs index 2027a757..c13f0234 100644 --- a/src/validator/remp_service.rs +++ b/src/validator/remp_service.rs @@ -48,7 +48,7 @@ impl RempService { self.get_core_interface()?.check_remp_duplicate(id) } - async fn new_remp_message(&self, message: &ton_api::ton::ton_node::RempMessage, source: &Arc) -> Result<()> { + async fn process_incoming_message(&self, message: &ton_api::ton::ton_node::RempMessage, source: &Arc) -> Result<()> { // TODO send error receipt in case of any error let engine = self.engine .get().ok_or_else(|| error!("engine was not set"))? @@ -75,7 +75,7 @@ impl RempMessagesSubscriber for RempService { let id = message.id().clone(); log::trace!(target: "remp", "Point 0. Processing incoming REMP message {:x}", id); - match self.new_remp_message(&message, source).await { + match self.process_incoming_message(&message, source).await { Ok(_) => log::trace!(target: "remp", "Point 0. Processed incoming REMP message {:x}", id), Err(e) => log::error!(target: "remp", "Point 0. Error processing incoming REMP message {:x}: {}", id, e) } diff --git a/src/validator/validator_group.rs b/src/validator/validator_group.rs index 94ead864..c19906d0 100644 --- a/src/validator/validator_group.rs +++ b/src/validator/validator_group.rs @@ -39,7 +39,7 @@ use validator_session_listener::{ use super::*; use super::fabric::*; use crate::{ - engine_traits::EngineOperations, + engine_traits::{EngineOperations, RempQueueCollatorInterface}, validator::{ catchain_overlay::CatchainOverlayManagerImpl, mutex_wrapper::MutexWrapper, @@ -53,6 +53,7 @@ use crate::{ } }, validating_utils::{fmt_next_block_descr_from_next_seqno, append_rh_to_next_block_descr} }; +use crate::validator::reliable_message_queue::RempQueueCollatorInterfaceImpl; #[cfg(feature = "slashing")] use crate::validator::slashing::SlashingManagerPtr; @@ -582,6 +583,14 @@ impl ValidatorGroup { self.group_impl.execute_sync(|group_impl| group_impl.reliable_queue.clone()).await } + pub async fn get_remp_queue_collator_interface(&self) -> Option> { + let queue_manager = self.get_reliable_message_queue().await; + queue_manager.map(|x| { + let interface: Arc = Arc::new(RempQueueCollatorInterfaceImpl::new(x)); + interface + }) + } + pub async fn info_round(&self, round: u32) -> String { self.group_impl.execute_sync(|group_impl| group_impl.info_round(round)).await } @@ -652,7 +661,7 @@ impl ValidatorGroup { let (_lk_round, prev_block_ids, mm_block_id, min_ts) = self.group_impl.execute_sync(|group_impl| group_impl.update_round (round)).await; - let include_external_messages = match self.check_in_sync(&prev_block_ids).await { + let remp_queue_collator_interface = match self.check_in_sync(&prev_block_ids).await { Err(e) => { log::warn!(target: "validator", "({}): Error checking sync for {}: `{}`", next_block_descr, self.info_round(round).await, e @@ -660,17 +669,19 @@ impl ValidatorGroup { callback(Err(e)); return; } - Ok(external_messages) => external_messages + Ok(false) => None, + Ok(true) => self.get_remp_queue_collator_interface().await }; + // To be removed after moving message queue processing into collator if let Some(rmq) = self.get_reliable_message_queue().await { log::info!( target: "validator", "ValidatorGroup::on_generate_slot: ({}) collecting REMP messages \ for {} for collation: {}", - next_block_descr, self.info_round(round).await, include_external_messages + next_block_descr, self.info_round(round).await, remp_queue_collator_interface.is_some() ); - if include_external_messages { + if remp_queue_collator_interface.is_some() { if let Err(e) = rmq.collect_messages_for_collation().await { log::error!( target: "validator", @@ -690,6 +701,7 @@ impl ValidatorGroup { min_ts, mc.seq_no, prev_block_ids, + remp_queue_collator_interface, self.local_key.clone(), self.validator_set.clone(), self.engine.clone(), From 7d9b749be1371af752d549bd86f8a85a7676a07a Mon Sep 17 00:00:00 2001 From: Dmitry Shtukenberg Date: Thu, 2 May 2024 15:28:42 +0300 Subject: [PATCH 31/40] Changed UnixTime to SystemTime for message queue --- src/tests/test_helper.rs | 5 +-- src/validator/message_cache.rs | 7 +++- src/validator/reliable_message_queue.rs | 43 +++++++++++++----------- src/validator/tests/test_collator.rs | 1 + src/validator/tests/test_rmq_messages.rs | 2 ++ src/validator/validator_group.rs | 6 ++-- 6 files changed, 38 insertions(+), 26 deletions(-) diff --git a/src/tests/test_helper.rs b/src/tests/test_helper.rs index 5f02e442..a0955892 100644 --- a/src/tests/test_helper.rs +++ b/src/tests/test_helper.rs @@ -697,11 +697,12 @@ impl TestEngine { extra.created_by().clone(), self.clone(), Some(extra.rand_seed().clone()), + None, CollatorSettings::default(), )?; - let (block_candidate, new_state) = collator.collate().await?; - + let (block_candidate, new_state) = collator.collate().await?; + if let Some(res_path) = &self.res_path { let new_block = Block::construct_from_bytes(&block_candidate.data)?; diff --git a/src/validator/message_cache.rs b/src/validator/message_cache.rs index 3794aba0..dadcc468 100644 --- a/src/validator/message_cache.rs +++ b/src/validator/message_cache.rs @@ -55,6 +55,7 @@ use ever_block::{ KeyId, Message, MsgAddressInt, MsgAddrStd, Result, Serializable, SliceData, UInt256, UnixTime32 }; +use ever_block_json::unix_time_to_system_time; #[cfg(test)] #[path = "tests/test_message_cache.rs"] @@ -268,7 +269,11 @@ impl RempMessageOrigin { } fn timestamp_now() -> Result { - Ok(SystemTime::now().duration_since(SystemTime::UNIX_EPOCH)?.as_secs() as u32) + Ok(UnixTime32::now().as_u32()) + } + + pub fn system_time(&self) -> Result { + unix_time_to_system_time(UnixTime32::new(self.timestamp).as_u32() as u64) } pub fn create_empty() -> Result { diff --git a/src/validator/reliable_message_queue.rs b/src/validator/reliable_message_queue.rs index 8966a122..3652709a 100644 --- a/src/validator/reliable_message_queue.rs +++ b/src/validator/reliable_message_queue.rs @@ -28,7 +28,7 @@ use ton_api::ton::ton_node::{ RempCatchainRecordV2, rempcatchainrecordv2::{RempCatchainMessageHeaderV2, RempCatchainMessageDigestV2} }; -use ever_block::{ShardIdent, Message, BlockIdExt, ValidatorDescr, UnixTime32}; +use ever_block::{ShardIdent, Message, BlockIdExt, ValidatorDescr}; use ever_block::{UInt256, Result, fail, gen_random_index, error}; use catchain::{PrivateKey, PublicKey}; @@ -64,7 +64,7 @@ struct MessageQueueImpl { pending_collation_set: HashMap, /// Messages, waiting for collator invocation - pending_collation_order: BinaryHeap<(Reverse, UInt256)>, + pending_collation_order: BinaryHeap<(Reverse, UInt256)>, /// Body sources for the message (actual if current node didn't receive body yet) body_sources: HashMap> @@ -91,7 +91,7 @@ impl MessageQueueImpl { }; } - pub fn add_to_collation_queue(&mut self, message_id: &UInt256, timestamp: u32, remp_node_sender: Option, add_if_absent: bool) -> Result<(bool,usize)> { + pub fn add_to_collation_queue(&mut self, message_id: &UInt256, timestamp: SystemTime, remp_node_sender: Option, add_if_absent: bool) -> Result<(bool,usize)> { let added = match self.pending_collation_set.get_mut(message_id) { Some(waiting_collation) if *waiting_collation => false, Some(waiting_collation) => { @@ -113,12 +113,12 @@ impl MessageQueueImpl { Ok((added, self.pending_collation_set.len())) } - pub fn take_first_for_collation(&mut self) -> Result> { - if let Some((timestamp, id)) = self.pending_collation_order.pop() { + pub fn take_first_for_collation(&mut self) -> Result> { + if let Some((Reverse(timestamp), id)) = self.pending_collation_order.pop() { match self.pending_collation_set.insert(id.clone(), false) { None => fail!("Message {:?}: taken from collation queue, but not found in collation set", id), Some(false) => fail!("Message {:?}: already removed from collation queue and given to collator", id), - Some(true) => Ok(Some((id, timestamp.0))) + Some(true) => Ok(Some((id, timestamp))) } } else { @@ -324,9 +324,9 @@ impl MessageQueue { } async fn add_pending_collation(&self, message_id: &UInt256, remp_message_origin: Arc, remp_node_sender: u32, status_to_send: Option) -> Result<()> { - let (added_to_queue, _len) = self.queues.execute_sync( - |catchain| catchain.add_to_collation_queue( - message_id, remp_message_origin.timestamp, Some(remp_node_sender), true + let (added_to_queue, _len) = self.queues.execute_sync(|catchain| + catchain.add_to_collation_queue( + message_id, remp_message_origin.system_time()?, Some(remp_node_sender), true ) ).await?; @@ -686,26 +686,29 @@ impl MessageQueue { Ok((false, Some((message.message.clone(), origin)))) } - async fn put_back_to_collation_queue(&self, msgid: &UInt256, timestamp: u32) -> Result<()> { + async fn put_back_to_collation_queue(&self, msgid: &UInt256, timestamp: SystemTime) -> Result<()> { let (added, _) = self.queues.execute_sync(|x| x.add_to_collation_queue(&msgid, timestamp, None, false) ).await?; - if added { + if !added { fail!("Message {:x} is already waiting for collation while postponing it", msgid); } Ok(()) } - pub async fn get_one_message_for_collation(&self, message_deadline: UnixTime32, generation_deadline: SystemTime) -> Result, Arc)>> { + pub async fn get_one_message_for_collation(&self, message_deadline: SystemTime, generation_deadline: SystemTime) -> Result, Arc)>> { while let Some((msgid, timestamp)) = self.queues.execute_sync(|x| x.take_first_for_collation()).await? { - if let Ok(x) = generation_deadline.elapsed() { - log::trace!(target: "remp", "RMQ {}: Message collation deadline expired {} ms ago", self, x.as_millis()); + if generation_deadline < SystemTime::now() { + log::trace!(target: "remp", + "RMQ {}: Message collation deadline expired at {:?}, stopping external messages collation", + self, generation_deadline + ); return Ok(None) } - if message_deadline.as_u32() <= timestamp { + if message_deadline <= timestamp { self.put_back_to_collation_queue(&msgid, timestamp).await?; return Ok(None) } @@ -721,7 +724,7 @@ impl MessageQueue { }, Ok((leave_in_queue, None)) => { if leave_in_queue { - self.put_back_to_collation_queue(&msgid, UnixTime32::now().as_u32()).await?; + self.put_back_to_collation_queue(&msgid, SystemTime::now()).await?; } continue }, @@ -744,7 +747,7 @@ impl MessageQueue { pub async fn collect_messages_for_collation (&self) -> Result<()> { log::trace!(target: "remp", "RMQ {}: collecting messages for collation", self); let mut cnt = 0; - let message_deadline = UnixTime32::now(); + let message_deadline = SystemTime::now(); let deadline = SystemTime::now().add(Duration::from_millis(1000)); while let Some((msg_id,message,_origin)) = self.get_one_message_for_collation(message_deadline, deadline).await? { @@ -912,7 +915,7 @@ impl MessageQueue { pub async fn return_to_collation_queue(&self, message_id: &UInt256) -> Result<()> { if let Some(origin) = self.remp_manager.message_cache.get_message_origin(message_id)? { self.queues.execute_sync( - |catchain| catchain.add_to_collation_queue(message_id, origin.timestamp, None, false) + |catchain| catchain.add_to_collation_queue(message_id, origin.system_time()?, None, false) ).await?; Ok(()) } @@ -1532,12 +1535,12 @@ impl fmt::Display for RmqQueueManager { pub struct RempQueueCollatorInterfaceImpl { queue: Arc, - message_deadline: UnixTime32 + message_deadline: SystemTime } impl RempQueueCollatorInterfaceImpl { pub fn new(remp_manager: Arc) -> Self { - Self { queue: remp_manager, message_deadline: UnixTime32::now() } + Self { queue: remp_manager, message_deadline: SystemTime::now() } } } diff --git a/src/validator/tests/test_collator.rs b/src/validator/tests/test_collator.rs index 7560ce27..6b710912 100644 --- a/src/validator/tests/test_collator.rs +++ b/src/validator/tests/test_collator.rs @@ -84,6 +84,7 @@ async fn try_collate_by_engine( created_by_opt.unwrap_or_default(), engine.clone(), rand_seed_opt, + None, CollatorSettings::default(), )?; let (block_candidate, new_state) = collator.collate().await?; diff --git a/src/validator/tests/test_rmq_messages.rs b/src/validator/tests/test_rmq_messages.rs index c38a6b28..e47ab0e5 100644 --- a/src/validator/tests/test_rmq_messages.rs +++ b/src/validator/tests/test_rmq_messages.rs @@ -13,6 +13,7 @@ use std::{collections::HashSet, str::FromStr, sync::Arc, time::Duration}; use std::ops::RangeInclusive; +use std::thread::sleep; use ton_api::ton::ton_node::{RempMessageLevel, RempMessageLevel::TonNode_RempMasterchain, RempMessageStatus, rempmessagestatus::{RempAccepted, RempIgnored}}; @@ -363,6 +364,7 @@ fn remp_simple_collation_equal_uids_test() -> Result<()> { )?; println!("Collecting messages for collation"); + sleep(Duration::from_millis(10)); // To overcome SystemTime inconsistency and make tests reproducible. testbench.message_queue.collect_messages_for_collation().await?; for (id, _msg) in testbench.engine.collator_queue.pop_iter() { diff --git a/src/validator/validator_group.rs b/src/validator/validator_group.rs index c19906d0..53e27009 100644 --- a/src/validator/validator_group.rs +++ b/src/validator/validator_group.rs @@ -16,8 +16,9 @@ use std::ops::RangeInclusive; use crossbeam_channel::Receiver; use catchain::utils::get_hash; -use ever_block::{BlockIdExt, ShardIdent, ValidatorSet, ValidatorDescr}; +use ever_block::{BlockIdExt, ShardIdent, ValidatorSet, ValidatorDescr, UnixTime32}; use ever_block::{fail, error, Result, UInt256}; +use ever_block_json::unix_time_to_system_time; use validator_session::{ BlockHash, BlockPayloadPtr, CatchainOverlayManagerPtr, SessionId, SessionPtr, SessionListenerPtr, SessionFactory, @@ -723,8 +724,7 @@ impl ValidatorGroup { let result_message = match &result { Ok(_) => { - let now = std::time::SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH).unwrap_or_default().as_secs(); + let now = UnixTime32::now().as_u32() as u64; self.last_collation_time.fetch_max(now, Ordering::Relaxed); format!("Collation successful") From 7b1dd572b7df21c137e448b0b7dbeaea36a0b10b Mon Sep 17 00:00:00 2001 From: Sumrachek Date: Fri, 3 May 2024 12:59:56 +0400 Subject: [PATCH 32/40] Integrated RempQueueCollatorInterface into collator --- src/collator_test_bundle.rs | 29 +-------- src/engine_operations.rs | 12 ---- src/engine_traits.rs | 21 ++---- src/ext_messages.rs | 5 +- src/validator/collator.rs | 87 ++++++++----------------- src/validator/reliable_message_queue.rs | 15 ++++- src/validator/validator_group.rs | 1 - 7 files changed, 49 insertions(+), 121 deletions(-) diff --git a/src/collator_test_bundle.rs b/src/collator_test_bundle.rs index 83e9488f..24c76ffd 100644 --- a/src/collator_test_bundle.rs +++ b/src/collator_test_bundle.rs @@ -39,7 +39,7 @@ use storage::StorageTelemetry; use ever_block::{ BlockIdExt, Message, ShardIdent, Serializable, MerkleUpdate, Deserializable, ValidatorBaseInfo, BlockSignaturesPure, BlockSignatures, HashmapAugType, - TopBlockDescrSet, GlobalCapabilities, OutMsgQueue, + TopBlockDescrSet, OutMsgQueue, }; use ever_block::{ShardStateUnsplit, TopBlockDescr}; use ever_block::{UInt256, fail, error, Result, CellType, read_boc, read_single_root_boc}; @@ -1208,19 +1208,6 @@ impl CollatorTestBundle { pub fn candidate(&self) -> Option<&BlockCandidate> { self.candidate.as_ref() } pub fn set_notes(&mut self, notes: String) { self.index.notes = notes } - - fn get_messages(&self, remp: bool) -> Result, UInt256)>> { - let remp_enabled = self.states - .get(&self.index.last_mc_state) - .ok_or_else(|| error!("Can't load last ms block to read config"))? - .config_params()?.has_capability(GlobalCapabilities::CapRemp); - - if remp_enabled == remp { - Ok(self.external_messages.clone()) - } else { - Ok(vec!()) - } - } } // Is used instead full node's engine for run tests @@ -1346,20 +1333,6 @@ impl EngineOperations for CollatorTestBundle { &self.allocated } - fn get_remp_messages(&self, _shard: &ShardIdent) -> Result, UInt256)>> { - self.get_messages(true) - } - - fn finalize_remp_messages( - &self, - _block: BlockIdExt, - _accepted: Vec, - _rejected: Vec<(UInt256, String)>, - _ignored: Vec, - ) -> Result<()> { - Ok(()) - } - async fn check_remp_duplicate(&self, _message_id: &UInt256) -> Result { Ok(RempDuplicateStatus::Fresh(UInt256::default())) } diff --git a/src/engine_operations.rs b/src/engine_operations.rs index 9266cd89..c8786c73 100644 --- a/src/engine_operations.rs +++ b/src/engine_operations.rs @@ -990,18 +990,6 @@ impl EngineOperations for Engine { fn new_remp_message(&self, id: UInt256, message: Arc) -> Result<()> { self.remp_messages()?.new_message(id, message) } - fn get_remp_messages(&self, shard: &ShardIdent) -> Result, UInt256)>> { - self.remp_messages()?.get_messages(shard) - } - fn finalize_remp_messages( - &self, - block: BlockIdExt, - accepted: Vec, - rejected: Vec<(UInt256, String)>, - ignored: Vec, - ) -> Result<()> { - self.remp_messages()?.finalize_messages(block, accepted, rejected, ignored) - } fn finalize_remp_messages_as_ignored(&self, block_id: &BlockIdExt) -> Result<()> { self.remp_messages()?.finalize_remp_messages_as_ignored(block_id) diff --git a/src/engine_traits.rs b/src/engine_traits.rs index 83d4a4bc..4d41061b 100644 --- a/src/engine_traits.rs +++ b/src/engine_traits.rs @@ -43,7 +43,7 @@ use ever_block::{ Deserializable, KeyId, KeyOption, MASTERCHAIN_ID, Message, OutMsgQueue, Result, SHARD_FULL, ShardAccount, ShardIdent, UInt256 }; -use std::{collections::HashSet, sync::{Arc, atomic::AtomicU64}, time::SystemTime}; +use std::{collections::HashSet, sync::{Arc, atomic::AtomicU64}}; use storage::{StorageAlloc, block_handle_db::BlockHandle}; #[cfg(feature = "telemetry")] use storage::StorageTelemetry; @@ -628,18 +628,6 @@ pub trait EngineOperations : Sync + Send { fn new_remp_message(&self, id: UInt256, message: Arc) -> Result<()> { unimplemented!() } - fn get_remp_messages(&self, shard: &ShardIdent) -> Result, UInt256)>> { - unimplemented!() - } - fn finalize_remp_messages( - &self, - block: BlockIdExt, - accepted: Vec, - rejected: Vec<(UInt256, String)>, - ignored: Vec, - ) -> Result<()> { - unimplemented!() - } fn finalize_remp_messages_as_ignored(&self, block_id: &BlockIdExt) -> Result<()> { unimplemented!() } @@ -974,5 +962,10 @@ pub trait RempCoreInterface: Sync + Send { #[async_trait::async_trait] pub trait RempQueueCollatorInterface : Send + Sync { - async fn get_next_message_for_collation(&self, master_block_id: &BlockIdExt, generation_deadline: SystemTime) -> Result>>; + async fn init_queue( + &self, + master_block_id: &BlockIdExt, + prev_blocks_ids: &[&BlockIdExt] + ) -> Result<()>; + async fn get_next_message_for_collation(&self) -> Result, UInt256)>>; } diff --git a/src/ext_messages.rs b/src/ext_messages.rs index cdb192e4..30f0f905 100644 --- a/src/ext_messages.rs +++ b/src/ext_messages.rs @@ -460,6 +460,7 @@ pub fn is_finally_accepted(status: &RempMessageStatus) -> bool { } } +// To be deleted pub struct RempMessagesPool { messages: Map>, statuses_queue: lockfree::queue::Queue<(UInt256, Arc, RempMessageStatus)>, @@ -483,7 +484,7 @@ impl RempMessagesPool { // Important! If call get_messages with same shard two times in row (without finalize_messages between) // the messages returned first call will return second time too. - pub fn get_messages(&self, shard: &ShardIdent) -> Result, UInt256)>> { + /*pub fn get_messages(&self, shard: &ShardIdent) -> Result, UInt256)>> { let mut result = vec!(); let mut ids = String::new(); for guard in self.messages.iter() { @@ -506,7 +507,7 @@ impl RempMessagesPool { ); Ok(result) - } + }*/ pub fn finalize_messages( &self, diff --git a/src/validator/collator.rs b/src/validator/collator.rs index 952688b1..be67b992 100644 --- a/src/validator/collator.rs +++ b/src/validator/collator.rs @@ -25,7 +25,7 @@ use crate::{ top_block_descr::{cmp_shard_block_descr, Mode as TbdMode, TopBlockDescrStuff}, }, validating_utils::{ - calc_remp_msg_ordering_hash, check_cur_validator_set, check_this_shard_mc_info, + check_cur_validator_set, check_this_shard_mc_info, may_update_shard_block_info, supported_capabilities, supported_version, UNREGISTERED_CHAIN_MAX_LEN, fmt_next_block_descr_from_next_seqno, }, @@ -216,9 +216,6 @@ struct CollatorData { new_messages: BinaryHeap, // using for priority queue accepted_ext_messages: Vec<(UInt256, i32)>, // message id and wokchain id rejected_ext_messages: Vec<(UInt256, String)>, // message id and reject reason - accepted_remp_messages: Vec, - rejected_remp_messages: Vec<(UInt256, String)>, - ignored_remp_messages: Vec, usage_tree: UsageTree, imported_visited: HashSet, @@ -291,9 +288,6 @@ impl CollatorData { new_messages: Default::default(), accepted_ext_messages: Default::default(), rejected_ext_messages: Default::default(), - accepted_remp_messages: Default::default(), - rejected_remp_messages: Default::default(), - ignored_remp_messages: Default::default(), usage_tree, imported_visited: HashSet::new(), gen_utime, @@ -638,18 +632,6 @@ impl CollatorData { std::mem::take(&mut self.rejected_ext_messages)) } - fn withdraw_remp_msg_statuses(&mut self) -> (Vec, Vec<(UInt256, String)>, Vec) { - (std::mem::take(&mut self.accepted_remp_messages), - std::mem::take(&mut self.rejected_remp_messages), - std::mem::take(&mut self.ignored_remp_messages)) - } - - fn set_remp_msg_statuses(&mut self, accepted: Vec, rejected: Vec<(UInt256, String)>, ignored: Vec) { - self.accepted_remp_messages = accepted; - self.rejected_remp_messages = rejected; - self.ignored_remp_messages = ignored; - } - fn estimate_pruned_count(&self) -> usize { if self.enqueue_count != 0 { let total_count = self.dequeue_count + self.enqueue_count + self.remove_count; @@ -989,6 +971,7 @@ impl ExecutionManager { pub struct Collator { engine: Arc, + remp_collator_interface: Option>, shard: ShardIdent, min_mc_seqno: u32, prev_blocks_ids: Vec, @@ -1091,6 +1074,7 @@ impl Collator { file_hash: UInt256::default(), }, engine, + remp_collator_interface, shard, min_mc_seqno, prev_blocks_ids, @@ -1340,11 +1324,6 @@ impl Collator { ) -> Result> { log::debug!("{}: do_collate", self.collated_block_descr); - let remp_messages = if is_remp_enabled(self.engine.clone(), mc_data.config()) { - Some(self.engine.get_remp_messages(&self.shard)?) - } else { - None - }; self.check_stop_flag()?; // loads out queues from neighbors and out queue of current shard @@ -1419,15 +1398,13 @@ impl Collator { log::debug!("{}: TIME: process_inbound_internal_messages {}ms;", self.collated_block_descr, now.elapsed().as_millis()); - if let Some(remp_messages) = remp_messages { + if is_remp_enabled(self.engine.clone(), mc_data.config()) { // import remp messages (if space&gas left) let now = std::time::Instant::now(); - let total = remp_messages.len(); - let processed = self.process_remp_messages( - prev_data, collator_data, &mut exec_manager, remp_messages - ).await?; - log::debug!("{}: TIME: process_remp_messages {}ms, processed {}, ignored {}", - self.collated_block_descr, now.elapsed().as_millis(), processed, total - processed); + let (accepted, rejected) = self.process_remp_messages( + prev_data, mc_data, collator_data, &mut exec_manager).await?; + log::debug!("{}: TIME: process_remp_messages {}ms, accepted {}, rejected {}", + self.collated_block_descr, now.elapsed().as_millis(), accepted, rejected); } // import inbound external messages (if space&gas left) @@ -2462,58 +2439,51 @@ impl Collator { async fn process_remp_messages( &self, prev_data: &PrevData, + mc_data: &McData, collator_data: &mut CollatorData, exec_manager: &mut ExecutionManager, - mut remp_messages: Vec<(Arc, UInt256)>, - ) -> Result { - log::trace!("{}: process_remp_messages ({}pcs)", self.collated_block_descr, remp_messages.len()); + ) -> Result<(usize, usize)> { + log::trace!("{}: process_remp_messages", self.collated_block_descr); - remp_messages.sort_by_cached_key(|(_, id)| { - calc_remp_msg_ordering_hash(&id, prev_data.pure_states.iter().map(|s| s.block_id())) - }); - log::trace!("{}: process_remp_messages: sorted {} messages", self.collated_block_descr, remp_messages.len()); + let remp_collator_interface = self.remp_collator_interface.as_ref() + .ok_or_else(|| error!("remp_collator_interface is not set"))?; - let mut ignored = vec!(); - let mut ignore = false; - for (msg, id) in remp_messages.drain(..) { - if ignore { - ignored.push(id); - continue; - } + remp_collator_interface.init_queue( + mc_data.state().block_id(), + &prev_data.pure_states.iter().map(|s| s.block_id()).collect::>() + ).await?; + log::trace!("{}: process_remp_messages: inited queue", self.collated_block_descr); + + while let Some((msg, id)) = remp_collator_interface.get_next_message_for_collation().await? { + log::trace!("{}: process_remp_messages: got next message {:x}", self.collated_block_descr, id); let header = msg.ext_in_header().ok_or_else(|| error!("remp message {:x} \ is not external inbound message", id))?; if self.shard.contains_address(&header.dst)? { let pruned_count = collator_data.estimate_pruned_count(); if !collator_data.block_limit_status.fits_normal(REMP_CUTOFF_LIMIT, pruned_count) { log::trace!("{}: block is loaded enough, stop processing remp messages", self.collated_block_descr); - ignored.push(id); - ignore = true; + break; } else if self.check_cutoff_timeout() { log::warn!("{}: TIMEOUT is elapsed, stop processing remp messages", self.collated_block_descr); - ignored.push(id); - ignore = true; + break; } else { let (_, account_id) = header.dst.extract_std_address(true)?; - log::trace!("{}: remp message {:x} sent to execution", self.collated_block_descr, id); - let msg = AsyncMessage::Ext(msg.deref().clone(), id); + let msg = AsyncMessage::Ext(msg.deref().clone(), id.clone()); exec_manager.execute(account_id, msg, prev_data, collator_data).await?; + log::trace!("{}: remp message {:x} sent to execution", self.collated_block_descr, id); } } else { log::warn!( "{}: process_remp_messages: ignored message {:x} for another shard {}", self.collated_block_descr, id, header.dst ); - ignored.push(id); } self.check_stop_flag()?; } exec_manager.wait_transactions(collator_data).await?; let (accepted, rejected) = collator_data.withdraw_ext_msg_statuses(); - let processed = accepted.len() + rejected.len(); - let accepted = accepted.into_iter().map(|(id, _)| id).collect(); - collator_data.set_remp_msg_statuses(accepted, rejected, ignored); - Ok(processed) + Ok((accepted.len(), rejected.len())) } async fn process_new_messages( @@ -2926,11 +2896,6 @@ impl Collator { // } // !!!! DEBUG !!!! - if is_remp_enabled(self.engine.clone(), mc_data.config()) { - let (accepted, rejected, ignored) = collator_data.withdraw_remp_msg_statuses(); - self.engine.finalize_remp_messages(block_id.clone(), accepted, rejected, ignored)?; - } - self.check_stop_flag()?; let collated_data = if !collator_data.shard_top_block_descriptors.is_empty() { diff --git a/src/validator/reliable_message_queue.rs b/src/validator/reliable_message_queue.rs index 3652709a..99bd480e 100644 --- a/src/validator/reliable_message_queue.rs +++ b/src/validator/reliable_message_queue.rs @@ -1552,8 +1552,17 @@ impl fmt::Display for RempQueueCollatorInterfaceImpl { #[async_trait::async_trait] impl RempQueueCollatorInterface for RempQueueCollatorInterfaceImpl { - async fn get_next_message_for_collation(&self, master_block_id: &BlockIdExt, generation_deadline: SystemTime) -> Result>> { - let reason = if self.queue.remp_manager.message_cache.is_block_processed(master_block_id)? { + async fn init_queue( + &self, + master_block_id: &BlockIdExt, + prev_blocks_ids: &[&BlockIdExt] + ) -> Result<()> { + unimplemented!("RempQueueCollatorInterfaceImpl::init_queue") + } + + async fn get_next_message_for_collation(&self) -> Result, UInt256)>> { + unimplemented!("RempQueueCollatorInterfaceImpl::get_next_message_for_collation") + /*let reason = if self.queue.remp_manager.message_cache.is_block_processed(master_block_id)? { if let Some(q) = &self.queue.cur_queue { return Ok(q.get_one_message_for_collation(self.message_deadline, generation_deadline) .await? @@ -1573,7 +1582,7 @@ impl RempQueueCollatorInterface for RempQueueCollatorInterfaceImpl { self, master_block_id, reason ); - return Ok(None) + return Ok(None)*/ } } diff --git a/src/validator/validator_group.rs b/src/validator/validator_group.rs index 53e27009..713cf716 100644 --- a/src/validator/validator_group.rs +++ b/src/validator/validator_group.rs @@ -18,7 +18,6 @@ use crossbeam_channel::Receiver; use catchain::utils::get_hash; use ever_block::{BlockIdExt, ShardIdent, ValidatorSet, ValidatorDescr, UnixTime32}; use ever_block::{fail, error, Result, UInt256}; -use ever_block_json::unix_time_to_system_time; use validator_session::{ BlockHash, BlockPayloadPtr, CatchainOverlayManagerPtr, SessionId, SessionPtr, SessionListenerPtr, SessionFactory, From 5d8c4a126205f564463abf60242db904648809fc Mon Sep 17 00:00:00 2001 From: Dmitry Shtukenberg Date: Tue, 7 May 2024 13:40:34 +0300 Subject: [PATCH 33/40] Changed collation behaviour, now it's Remp responsibility --- src/validator/reliable_message_queue.rs | 129 +++++++++++++++++++----- src/validator/remp_manager.rs | 4 - src/validator/validator_group.rs | 32 ++++-- 3 files changed, 126 insertions(+), 39 deletions(-) diff --git a/src/validator/reliable_message_queue.rs b/src/validator/reliable_message_queue.rs index 99bd480e..8a4bf735 100644 --- a/src/validator/reliable_message_queue.rs +++ b/src/validator/reliable_message_queue.rs @@ -15,10 +15,11 @@ use std::{ cmp::Reverse, collections::{BinaryHeap, HashMap}, fmt, fmt::Formatter, - ops::{Add, RangeInclusive}, + ops::RangeInclusive, sync::Arc, time::{Duration, SystemTime} }; +use std::sync::atomic::{AtomicUsize, Ordering}; use dashmap::DashMap; use ton_api::ton::ton_node::{ @@ -28,7 +29,7 @@ use ton_api::ton::ton_node::{ RempCatchainRecordV2, rempcatchainrecordv2::{RempCatchainMessageHeaderV2, RempCatchainMessageDigestV2} }; -use ever_block::{ShardIdent, Message, BlockIdExt, ValidatorDescr}; +use ever_block::{ShardIdent, Message, BlockIdExt, ValidatorDescr, Sha256}; use ever_block::{UInt256, Result, fail, gen_random_index, error}; use catchain::{PrivateKey, PublicKey}; @@ -698,16 +699,8 @@ impl MessageQueue { Ok(()) } - pub async fn get_one_message_for_collation(&self, message_deadline: SystemTime, generation_deadline: SystemTime) -> Result, Arc)>> { + pub async fn get_one_message_for_collation(&self, message_deadline: SystemTime) -> Result, Arc)>> { while let Some((msgid, timestamp)) = self.queues.execute_sync(|x| x.take_first_for_collation()).await? { - if generation_deadline < SystemTime::now() { - log::trace!(target: "remp", - "RMQ {}: Message collation deadline expired at {:?}, stopping external messages collation", - self, generation_deadline - ); - return Ok(None) - } - if message_deadline <= timestamp { self.put_back_to_collation_queue(&msgid, timestamp).await?; return Ok(None) @@ -743,16 +736,26 @@ impl MessageQueue { return Ok(None) } + pub async fn prepare_messages_for_collation (&self) -> Result, Arc)>> { + let message_deadline = SystemTime::now(); + let mut prepared_for_collation = Vec::new(); + + while let Some(msg_info) = self.get_one_message_for_collation(message_deadline).await? { + prepared_for_collation.push(msg_info); + } + + Ok(prepared_for_collation) + } +/* /// Prepare messages for collation - to be called just before collator invocation. pub async fn collect_messages_for_collation (&self) -> Result<()> { log::trace!(target: "remp", "RMQ {}: collecting messages for collation", self); let mut cnt = 0; let message_deadline = SystemTime::now(); - let deadline = SystemTime::now().add(Duration::from_millis(1000)); - while let Some((msg_id,message,_origin)) = self.get_one_message_for_collation(message_deadline, deadline).await? { + while let Some((msg_id,message,_origin)) = self.get_one_message_for_collation(message_deadline).await? { //self.queues.execute_sync(|x| x.take_first_for_collation()).await? { -/* + let (message, origin) = match self.check_one_message_for_collation(&msgid).await { Err(e) => { log::error!( @@ -864,7 +867,7 @@ impl MessageQueue { continue } } -*/ + match self.send_message_to_collator(msg_id.clone(), message.clone()).await { Err(e) => { let error = format!("{}", e); @@ -879,7 +882,7 @@ impl MessageQueue { log::trace!(target: "remp", "Point 5. RMQ {}: total {} messages for collation", self, cnt); Ok(()) } - +*/ pub async fn all_accepted_by_collator_to_ignored(&self) -> Result> { let mut downgrading = Vec::new(); @@ -1034,12 +1037,6 @@ impl MessageQueue { self.queues.execute_sync(|mq| mq.pending_collation_set.len()).await } - async fn send_message_to_collator(&self, message_id: UInt256, message: Arc) -> Result<()> { - self.remp_manager.collator_receipt_dispatcher.queue.send_message_to_collator( - message_id, message - ).await - } - //pub fn pack_payload } @@ -1361,7 +1358,7 @@ impl RmqQueueManager { fail!("Cannot put message to RMQ {}: queue is not started yet", self) } } - +/* pub async fn collect_messages_for_collation (&self) -> Result<()> { if let Some(queue) = &self.cur_queue { queue.collect_messages_for_collation().await?; @@ -1371,6 +1368,16 @@ impl RmqQueueManager { fail!("Collecting messages for collation: RMQ {} is not started", self) } } +*/ + pub async fn prepare_messages_for_collation (&self) -> Result, Arc)>> { + if let Some(queue) = &self.cur_queue { + let messages = queue.prepare_messages_for_collation().await?; + Ok(messages) + } + else { + fail!("Preparing messages for collation: RMQ {} is not started", self) + } + } pub async fn process_collation_result (&self) -> Result<()> { if let Some(queue) = &self.cur_queue { @@ -1535,12 +1542,46 @@ impl fmt::Display for RmqQueueManager { pub struct RempQueueCollatorInterfaceImpl { queue: Arc, - message_deadline: SystemTime + messages_to_process: lockfree::queue::Queue<(UInt256, Arc, Arc)>, + messages_count: AtomicUsize } impl RempQueueCollatorInterfaceImpl { pub fn new(remp_manager: Arc) -> Self { - Self { queue: remp_manager, message_deadline: SystemTime::now() } + Self { + queue: remp_manager, + messages_to_process: lockfree::queue::Queue::new(), + messages_count: AtomicUsize::new(0) + } + } + + fn compute_ordering_hash(msg_id: &UInt256, prev_blocks_ids: &[&BlockIdExt]) -> UInt256 { + let mut hasher = Sha256::new(); + for prev_id in prev_blocks_ids { + hasher.update(prev_id.root_hash().as_slice()); + } + hasher.update(msg_id.as_slice()); + UInt256::from(hasher.finalize().as_slice()) + } + + /// All messages that were not requested by collator, are returned back to queue by this function. + /// The function returns (total messages, messages returned back) + pub async fn return_prepared_messages_to_queue(&self) -> Result<(usize, usize)> { + let mut cnt = 0; + let cur_queue = match &self.queue.cur_queue { + Some(q) => q, + None => fail!("No current queue {}: cannot return prepared messages to non-initalized/removed queue", self) + }; + for (msg_id,_,_) in self.messages_to_process.pop_iter() { + cur_queue.return_to_collation_queue(&msg_id).await?; + cnt += 1; + } + Ok((self.messages_count.load(Ordering::Relaxed), cnt)) + } + + pub fn into_interface(self: Arc) -> Arc { + let interface: Arc = self.clone(); + interface } } @@ -1552,16 +1593,50 @@ impl fmt::Display for RempQueueCollatorInterfaceImpl { #[async_trait::async_trait] impl RempQueueCollatorInterface for RempQueueCollatorInterfaceImpl { + /// Returns true if the messages were prepared, + /// false if the messages could not be prepared (master block is not processed yet). async fn init_queue( &self, master_block_id: &BlockIdExt, prev_blocks_ids: &[&BlockIdExt] ) -> Result<()> { - unimplemented!("RempQueueCollatorInterfaceImpl::init_queue") + if !self.queue.remp_manager.message_cache.is_block_processed(master_block_id)? { + log::warn!(target: "remp", "Block {} is not ready yet, cannot collect messages relative to it", master_block_id); + return Ok(()); + } + + let prepared_messages = self.queue.prepare_messages_for_collation().await?; + let mut ordered_messages: Vec<(UInt256, UInt256, Arc, Arc)> = prepared_messages.into_iter().map( + |(id,msg,origin)| (Self::compute_ordering_hash(&id, prev_blocks_ids), id, msg, origin) + ).collect(); + ordered_messages.sort_by(|(ordering_id1,_,_,_), (ordering_id2,_,_,_)| ordering_id1.cmp(ordering_id2)); + let cnt = ordered_messages.len(); + + for (_order, id, msg, origin) in ordered_messages.into_iter() { + self.messages_to_process.push((id, msg, origin)); + } + + log::trace!(target: "remp", "Point 5. RMQ {}: total {} messages for collation", self, cnt); + self.messages_count.store(cnt, Ordering::Relaxed); + + Ok(()) } async fn get_next_message_for_collation(&self) -> Result, UInt256)>> { - unimplemented!("RempQueueCollatorInterfaceImpl::get_next_message_for_collation") + let cur_queue = match &self.queue.cur_queue { + Some(q) => q, + None => fail!("No current queue {}: cannot get next message for collation", self) + }; + + Ok(self.messages_to_process.pop().map(|(id,msg,origin)| { + let new_status = RempMessageStatus::TonNode_RempAccepted (RempAccepted { + level: RempMessageLevel::TonNode_RempQueue, + block_id: BlockIdExt::default(), + master_id: BlockIdExt::default() + }); + cur_queue.update_status_send_response(&id, origin.clone(), new_status); + (msg,id) + })) /*let reason = if self.queue.remp_manager.message_cache.is_block_processed(master_block_id)? { if let Some(q) = &self.queue.cur_queue { return Ok(q.get_one_message_for_collation(self.message_deadline, generation_deadline) diff --git a/src/validator/remp_manager.rs b/src/validator/remp_manager.rs index dd04a355..8ae95628 100644 --- a/src/validator/remp_manager.rs +++ b/src/validator/remp_manager.rs @@ -363,10 +363,6 @@ impl CollatorInterfaceWrapper { // message_dispatcher: Mutex::new(HashMap::new()) } } - - pub async fn send_message_to_collator(&self, msg_id: UInt256, msg: Arc) -> Result<()> { - self.engine.new_remp_message(msg_id.clone(), msg) - } } pub struct CollatorResult { diff --git a/src/validator/validator_group.rs b/src/validator/validator_group.rs index 713cf716..9b8eaf36 100644 --- a/src/validator/validator_group.rs +++ b/src/validator/validator_group.rs @@ -39,7 +39,7 @@ use validator_session_listener::{ use super::*; use super::fabric::*; use crate::{ - engine_traits::{EngineOperations, RempQueueCollatorInterface}, + engine_traits::EngineOperations, validator::{ catchain_overlay::CatchainOverlayManagerImpl, mutex_wrapper::MutexWrapper, @@ -583,11 +583,10 @@ impl ValidatorGroup { self.group_impl.execute_sync(|group_impl| group_impl.reliable_queue.clone()).await } - pub async fn get_remp_queue_collator_interface(&self) -> Option> { + pub async fn get_remp_queue_collator_interface(&self) -> Option> { let queue_manager = self.get_reliable_message_queue().await; queue_manager.map(|x| { - let interface: Arc = Arc::new(RempQueueCollatorInterfaceImpl::new(x)); - interface + Arc::new(RempQueueCollatorInterfaceImpl::new(x)) }) } @@ -661,7 +660,7 @@ impl ValidatorGroup { let (_lk_round, prev_block_ids, mm_block_id, min_ts) = self.group_impl.execute_sync(|group_impl| group_impl.update_round (round)).await; - let remp_queue_collator_interface = match self.check_in_sync(&prev_block_ids).await { + let remp_queue_collator_interface_impl = match self.check_in_sync(&prev_block_ids).await { Err(e) => { log::warn!(target: "validator", "({}): Error checking sync for {}: `{}`", next_block_descr, self.info_round(round).await, e @@ -673,15 +672,17 @@ impl ValidatorGroup { Ok(true) => self.get_remp_queue_collator_interface().await }; + /* // To be removed after moving message queue processing into collator if let Some(rmq) = self.get_reliable_message_queue().await { log::info!( target: "validator", "ValidatorGroup::on_generate_slot: ({}) collecting REMP messages \ for {} for collation: {}", - next_block_descr, self.info_round(round).await, remp_queue_collator_interface.is_some() + next_block_descr, self.info_round(round).await, remp_queue_collator_interface_impl.is_some() ); - if remp_queue_collator_interface.is_some() { + if let Some(interface_impl) = remp_queue_collator_interface_impl { + let interface if let Err(e) = rmq.collect_messages_for_collation().await { log::error!( target: "validator", @@ -693,6 +694,7 @@ impl ValidatorGroup { } } } + */ let result = match mm_block_id { Some(mc) => { @@ -701,7 +703,7 @@ impl ValidatorGroup { min_ts, mc.seq_no, prev_block_ids, - remp_queue_collator_interface, + remp_queue_collator_interface_impl.clone().map(|x| x.into_interface()), self.local_key.clone(), self.validator_set.clone(), self.engine.clone(), @@ -713,6 +715,18 @@ impl ValidatorGroup { None => Err(error!("Min masterchain block id missing")), }; + let return_result_message = if let Some(x) = remp_queue_collator_interface_impl { + match x.return_prepared_messages_to_queue().await { + Ok((total, returned)) => format!("total external messages {}, processed {}, returned to queue {}", + total, total as i64 - returned as i64, returned + ), + Err(e) => format!("error returning non-processed external messages to queue `{}`", e) + } + } + else { + format!("no external messages processed") + }; + let candidate = match self.verification_manager.clone() { Some(_) => match &result { Ok(candidate) => Some(candidate.clone()), @@ -755,6 +769,8 @@ impl ValidatorGroup { } }; + let result_message = format!("{}{}", result_message, return_result_message); + if let Some(rmq) = self.get_reliable_message_queue().await { if let Err(e) = rmq.process_collation_result().await { log::error!(target: "validator", "({}): Error processing collation results for {}: `{}`", From f252e1240a34eb4ad903f77fd3571f7304ec28f7 Mon Sep 17 00:00:00 2001 From: Dmitry Shtukenberg Date: Tue, 7 May 2024 14:10:24 +0300 Subject: [PATCH 34/40] Fixed tests --- src/validator/tests/test_rmq_messages.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/validator/tests/test_rmq_messages.rs b/src/validator/tests/test_rmq_messages.rs index e47ab0e5..f4fa1e0a 100644 --- a/src/validator/tests/test_rmq_messages.rs +++ b/src/validator/tests/test_rmq_messages.rs @@ -365,7 +365,10 @@ fn remp_simple_collation_equal_uids_test() -> Result<()> { println!("Collecting messages for collation"); sleep(Duration::from_millis(10)); // To overcome SystemTime inconsistency and make tests reproducible. - testbench.message_queue.collect_messages_for_collation().await?; + let messages = testbench.message_queue.prepare_messages_for_collation().await?; + for (msg_id, msg, _order) in messages.into_iter() { + testbench.engine.new_remp_message(msg_id.clone(), msg)?; + } for (id, _msg) in testbench.engine.collator_queue.pop_iter() { println!("collated: {:x}", id); From 49499029d360d960acf94f2becd27206658263ca Mon Sep 17 00:00:00 2001 From: Dmitry Shtukenberg Date: Tue, 7 May 2024 18:10:35 +0300 Subject: [PATCH 35/40] Just a test --- src/validator/reliable_message_queue.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/validator/reliable_message_queue.rs b/src/validator/reliable_message_queue.rs index 8a4bf735..9d427a01 100644 --- a/src/validator/reliable_message_queue.rs +++ b/src/validator/reliable_message_queue.rs @@ -1609,7 +1609,7 @@ impl RempQueueCollatorInterface for RempQueueCollatorInterfaceImpl { let mut ordered_messages: Vec<(UInt256, UInt256, Arc, Arc)> = prepared_messages.into_iter().map( |(id,msg,origin)| (Self::compute_ordering_hash(&id, prev_blocks_ids), id, msg, origin) ).collect(); - ordered_messages.sort_by(|(ordering_id1,_,_,_), (ordering_id2,_,_,_)| ordering_id1.cmp(ordering_id2)); + ordered_messages.sort_by(|(ordering_id1,_,_,_), (ordering_id2,_,_,_)| ordering_id2.cmp(ordering_id1)); let cnt = ordered_messages.len(); for (_order, id, msg, origin) in ordered_messages.into_iter() { From 3b5965ccef2ec3d9628109f669b9617f125fab11 Mon Sep 17 00:00:00 2001 From: Dmitry Shtukenberg Date: Wed, 8 May 2024 17:03:52 +0300 Subject: [PATCH 36/40] Debug print added --- src/validator/reliable_message_queue.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/validator/reliable_message_queue.rs b/src/validator/reliable_message_queue.rs index 9d427a01..27b7d984 100644 --- a/src/validator/reliable_message_queue.rs +++ b/src/validator/reliable_message_queue.rs @@ -29,7 +29,7 @@ use ton_api::ton::ton_node::{ RempCatchainRecordV2, rempcatchainrecordv2::{RempCatchainMessageHeaderV2, RempCatchainMessageDigestV2} }; -use ever_block::{ShardIdent, Message, BlockIdExt, ValidatorDescr, Sha256}; +use ever_block::{ShardIdent, Message, BlockIdExt, ValidatorDescr, Sha256, Serializable}; use ever_block::{UInt256, Result, fail, gen_random_index, error}; use catchain::{PrivateKey, PublicKey}; @@ -1612,9 +1612,13 @@ impl RempQueueCollatorInterface for RempQueueCollatorInterfaceImpl { ordered_messages.sort_by(|(ordering_id1,_,_,_), (ordering_id2,_,_,_)| ordering_id2.cmp(ordering_id1)); let cnt = ordered_messages.len(); - for (_order, id, msg, origin) in ordered_messages.into_iter() { + log::trace!(target: "remp", "DebugMessageCollationStart"); + for (order, id, msg, origin) in ordered_messages.into_iter() { + let msg_serialized: ton_api::ton::bytes = msg.write_to_bytes().unwrap().into(); + log::trace!(target: "remp", "DebugMessageCollation: {:x} {:x} {:?}", order, id, msg_serialized); self.messages_to_process.push((id, msg, origin)); } + log::trace!(target: "remp", "DebugMessageCollationFinish"); log::trace!(target: "remp", "Point 5. RMQ {}: total {} messages for collation", self, cnt); self.messages_count.store(cnt, Ordering::Relaxed); From 3d1399b0c3ed379370608376650495c2f8e507bc Mon Sep 17 00:00:00 2001 From: Dmitry Shtukenberg Date: Wed, 8 May 2024 17:51:04 +0300 Subject: [PATCH 37/40] Additional debug message --- src/validator/reliable_message_queue.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/validator/reliable_message_queue.rs b/src/validator/reliable_message_queue.rs index 27b7d984..cf2db1d1 100644 --- a/src/validator/reliable_message_queue.rs +++ b/src/validator/reliable_message_queue.rs @@ -1612,7 +1612,9 @@ impl RempQueueCollatorInterface for RempQueueCollatorInterfaceImpl { ordered_messages.sort_by(|(ordering_id1,_,_,_), (ordering_id2,_,_,_)| ordering_id2.cmp(ordering_id1)); let cnt = ordered_messages.len(); - log::trace!(target: "remp", "DebugMessageCollationStart"); + log::trace!(target: "remp", "DebugMessageCollationStart: {}", + prev_blocks_ids.iter().map(|x| format!(" {} ", x)).collect::() + ); for (order, id, msg, origin) in ordered_messages.into_iter() { let msg_serialized: ton_api::ton::bytes = msg.write_to_bytes().unwrap().into(); log::trace!(target: "remp", "DebugMessageCollation: {:x} {:x} {:?}", order, id, msg_serialized); From 6759affd619e761b754fa52116b1cd21dcdd3eff Mon Sep 17 00:00:00 2001 From: Dmitry Shtukenberg Date: Tue, 14 May 2024 19:36:30 +0300 Subject: [PATCH 38/40] Remp network performance test --- src/validator/reliable_message_queue.rs | 18 +++++++++++++++--- src/validator/remp_catchain.rs | 3 +++ tests/test_run_net/log_cfg.yml | 2 +- tests/test_run_net/test_run_net.sh | 2 +- tests/test_run_net/zero_state_blanc_1.json | 4 ++-- 5 files changed, 22 insertions(+), 7 deletions(-) diff --git a/src/validator/reliable_message_queue.rs b/src/validator/reliable_message_queue.rs index cf2db1d1..904e3af4 100644 --- a/src/validator/reliable_message_queue.rs +++ b/src/validator/reliable_message_queue.rs @@ -47,12 +47,13 @@ use crate::{ } }; use failure::err_msg; +use storage::db::traits::DbKey; use crate::block::BlockIdExtExtention; #[derive(Debug,PartialEq,Eq,PartialOrd,Ord,Clone)] enum MessageQueueStatus { Created, Starting, Active, Stopping } const RMQ_STOP_POLLING_INTERVAL: Duration = Duration::from_millis(50); -const RMQ_REQUEST_NEW_BLOCK_INTERVAL: Duration = Duration::from_millis(50); +const RMQ_REQUEST_NEW_BLOCK_INTERVAL: Duration = Duration::from_millis(10); const RMQ_MAXIMAL_BROADCASTS_IN_PACK: u32 = 1000; const RMQ_MAXIMAL_QUERIES_IN_PACK: u32 = 1000; const RMQ_MESSAGE_QUERY_TIMEOUT: Duration = Duration::from_millis(2000); @@ -1611,7 +1612,7 @@ impl RempQueueCollatorInterface for RempQueueCollatorInterfaceImpl { ).collect(); ordered_messages.sort_by(|(ordering_id1,_,_,_), (ordering_id2,_,_,_)| ordering_id2.cmp(ordering_id1)); let cnt = ordered_messages.len(); - +/* log::trace!(target: "remp", "DebugMessageCollationStart: {}", prev_blocks_ids.iter().map(|x| format!(" {} ", x)).collect::() ); @@ -1621,8 +1622,19 @@ impl RempQueueCollatorInterface for RempQueueCollatorInterfaceImpl { self.messages_to_process.push((id, msg, origin)); } log::trace!(target: "remp", "DebugMessageCollationFinish"); +*/ + for (_order, id, _msg, origin) in ordered_messages.into_iter() { + if let Some(x) = &self.queue.cur_queue { + let reject = RempRejected { + level: RempMessageLevel::TonNode_RempCollator, + block_id: BlockIdExt::default(), + error: "Test message rejected".as_string() + }; + x.update_status_send_response(&id, origin, RempMessageStatus::TonNode_RempRejected(reject)) + } + } - log::trace!(target: "remp", "Point 5. RMQ {}: total {} messages for collation", self, cnt); + log::info!(target: "remp", "Point 5. RMQ {}: total {} messages for collation", self, cnt); self.messages_count.store(cnt, Ordering::Relaxed); Ok(()) diff --git a/src/validator/remp_catchain.rs b/src/validator/remp_catchain.rs index d51e2768..4f7b3025 100644 --- a/src/validator/remp_catchain.rs +++ b/src/validator/remp_catchain.rs @@ -534,12 +534,15 @@ impl RempCatchain { "Point 4. Message received from RMQ {}: {:?}, decoded {:?}, put to rmq_catchain queue", self, msg.signature, record //catchain.received_messages.len() ); + /* if let Err(e) = self.instance.rmq_catchain_send(record.clone(), source_idx) { log::error!( target: "remp", "Point 4. Cannot put message {:?} from RMQ {} to queue: {}", record, self, e ) } + + */ }, Err(e) => log::error!(target: "remp", "Cannot deserialize message from RMQ {} {:?}: {}", self, msg.signature, e diff --git a/tests/test_run_net/log_cfg.yml b/tests/test_run_net/log_cfg.yml index 2293d9ac..13c1f09c 100644 --- a/tests/test_run_net/log_cfg.yml +++ b/tests/test_run_net/log_cfg.yml @@ -25,7 +25,7 @@ appenders: limit: 5 gb roller: kind: fixed_window - pattern: '/home/ruser/TON/ever-node/tests/test_high_load/logs/output_NODE_NUM_{}.log' + pattern: '/shared/output_NODE_NUM_{}.log' base: 1 count: 10 diff --git a/tests/test_run_net/test_run_net.sh b/tests/test_run_net/test_run_net.sh index 9fce7bc0..630f2846 100755 --- a/tests/test_run_net/test_run_net.sh +++ b/tests/test_run_net/test_run_net.sh @@ -1,6 +1,6 @@ #!/bin/bash -NODES=6 +NODES=3 WORKCHAINS=false REMP_TEST=false CURRENT_BRANCH="$(git branch --show-current)" diff --git a/tests/test_run_net/zero_state_blanc_1.json b/tests/test_run_net/zero_state_blanc_1.json index 61d536ec..bde11397 100644 --- a/tests/test_run_net/zero_state_blanc_1.json +++ b/tests/test_run_net/zero_state_blanc_1.json @@ -128,7 +128,7 @@ "p16": { "max_validators": 1000, "max_main_validators": 100, - "min_validators": 5 + "min_validators": 3 }, "p17": { "min_stake": "10000000000000", @@ -248,6 +248,6 @@ "utime_since": nowdate, "utime_until": 2000242087, "total": p34_total, - "main": 5, + "main": 3, "total_weight": p34_total_weight, "list": [ From 84df91a2e0cdcb91254fcd4ceb8e28f41eea27ba Mon Sep 17 00:00:00 2001 From: Dmitry Shtukenberg Date: Tue, 14 May 2024 20:53:29 +0300 Subject: [PATCH 39/40] Small optimization --- src/validator/remp_catchain.rs | 10 ++++---- src/validator/remp_manager.rs | 37 +++++++++++++++++------------- src/validator/validator_manager.rs | 10 +++++++- src/validator/validator_utils.rs | 31 +++++++++++++++++-------- 4 files changed, 57 insertions(+), 31 deletions(-) diff --git a/src/validator/remp_catchain.rs b/src/validator/remp_catchain.rs index 4f7b3025..ec58e7ce 100644 --- a/src/validator/remp_catchain.rs +++ b/src/validator/remp_catchain.rs @@ -534,15 +534,14 @@ impl RempCatchain { "Point 4. Message received from RMQ {}: {:?}, decoded {:?}, put to rmq_catchain queue", self, msg.signature, record //catchain.received_messages.len() ); - /* + if let Err(e) = self.instance.rmq_catchain_send(record.clone(), source_idx) { log::error!( target: "remp", "Point 4. Cannot put message {:?} from RMQ {} to queue: {}", record, self, e ) } - - */ + }, Err(e) => log::error!(target: "remp", "Cannot deserialize message from RMQ {} {:?}: {}", self, msg.signature, e @@ -608,8 +607,9 @@ impl CatchainListener for RempCatchain { let mut msg_vect: Vec<::ton_api::ton::validator_session::round::Message> = Vec::new(); let mut msg_ids: Vec = Vec::new(); + let MAX_VECT_COUNT = 8; - if let Ok(Some(msg)) = self.instance.pending_messages_queue_try_recv() { + while let Ok(Some(msg)) = self.instance.pending_messages_queue_try_recv() { let msg_body = ::ton_api::ton::validator_session::round::validator_session::message::message::Commit { round: 0, candidate: Default::default(), @@ -621,6 +621,8 @@ impl CatchainListener for RempCatchain { msg_vect.push(msg_body); msg_ids.push(get_remp_catchain_record_info(&msg)); + + if msg_vect.len() >= MAX_VECT_COUNT { break; } } let payload = ::ton_api::ton::validator_session::blockupdate::BlockUpdate { diff --git a/src/validator/remp_manager.rs b/src/validator/remp_manager.rs index 8ae95628..2f308fd3 100644 --- a/src/validator/remp_manager.rs +++ b/src/validator/remp_manager.rs @@ -42,7 +42,8 @@ use std::time::SystemTime; use adnl::telemetry::Metric; use chrono::{DateTime, Utc}; use crossbeam_channel::TryRecvError; -use rand::Rng; +use rand::{Rng, RngCore}; +use rand::prelude::ThreadRng; pub struct RempInterfaceQueues { message_cache: Arc, @@ -550,9 +551,12 @@ impl RempManager { #[allow(dead_code)] impl RempInterfaceQueues { - pub fn make_test_message(&self) -> Result { + pub fn make_test_message(&self, thread_rng: &mut ThreadRng) -> Result { + let mut data_array = [0 as u8; 128]; + thread_rng.fill_bytes(&mut data_array); + let data = SliceData::new(data_array.to_vec()); Ok(RempMessageWithOrigin { - message: RmqMessage::make_test_message(&SliceData::new_empty())?, + message: RmqMessage::make_test_message(&data)?, origin: RempMessageOrigin::create_empty()? }) } @@ -563,25 +567,26 @@ impl RempInterfaceQueues { * 1. random messages are generated each second, and sent to the REMP input queue * 2. responses after REMP processing are taken from REMP response queue and printed */ - pub async fn test_remp_messages_loop(&self) { + pub async fn test_remp_messages_loop(&self, pause: Duration, batch_size: usize) { log::info!(target: "remp", "Test REMP messages loop is started"); loop { - match self.make_test_message() { - Err(e) => log::error!(target: "remp", "Cannot make test REMP message: `{}`", e), - Ok(msg) => { - if let Err(x) = self.incoming_sender.send(Arc::new(msg)) { - log::error!(target: "remp", "Cannot send test REMP message to RMQ: {}", - x - ); - } - - while let Ok(msg) = self.response_receiver.try_recv() { - log::trace!(target: "remp", "Received test REMP response: {:?}", msg); + for _i in 0..batch_size { + let mut thread_rng = ThreadRng::default(); + match self.make_test_message(&mut thread_rng) { + Err(e) => log::error!(target: "remp", "Cannot make test REMP message: `{}`", e), + Ok(msg) => { + if let Err(x) = self.incoming_sender.send(Arc::new(msg)) { + log::error!(target: "remp", "Cannot send test REMP message to RMQ: {}", x); + } + + while let Ok(msg) = self.response_receiver.try_recv() { + log::trace!(target: "remp", "Received test REMP response: {:?}", msg); + } } } } - tokio::time::sleep(std::time::Duration::from_secs(1)).await; + tokio::time::sleep(pause).await; } } diff --git a/src/validator/validator_manager.rs b/src/validator/validator_manager.rs index 57e6ab1d..f19e4528 100644 --- a/src/validator/validator_manager.rs +++ b/src/validator/validator_manager.rs @@ -1726,11 +1726,19 @@ pub fn start_validator_manager( log::error!(target: "validator_manager", "set_remp_core_interface: {}", e); } + let iface = remp_iface.clone(); runtime.clone().spawn(async move { log::info!(target: "remp", "Starting REMP responses polling loop"); - remp_iface.poll_responses_loop().await; + iface.poll_responses_loop().await; log::info!(target: "remp", "Finishing REMP responses polling loop"); }); + + let iface = remp_iface.clone(); + runtime.clone().spawn(async move { + log::info!(target: "remp", "Starting REMP testing loop"); + iface.test_remp_messages_loop(Duration::from_millis(300), 100).await; + log::info!(target: "remp", "Finishing REMP testing loop"); + }); } if let Err(e) = manager.invoke().await { diff --git a/src/validator/validator_utils.rs b/src/validator/validator_utils.rs index 9df997d1..cb603484 100644 --- a/src/validator/validator_utils.rs +++ b/src/validator/validator_utils.rs @@ -17,13 +17,7 @@ use crate::{ }; use catchain::{BlockPayloadPtr, CatchainNode, PublicKey, PublicKeyHash}; -use ever_block::{ - error, fail, BlockIdExt, BlockInfo, BlockSignatures, BlockSignaturesPure, BuilderData, - ConfigParams, CryptoSignature, CryptoSignaturePair, Deserializable, Ed25519KeyOption, - GlobalCapabilities, HashmapType, KeyId, Message, Result, Serializable, Sha256, ShardIdent, - SigPubKey, UInt256, UnixTime32, ValidatorBaseInfo, ValidatorDescr, ValidatorSet, Workchains, - WorkchainDescr -}; +use ever_block::{error, fail, BlockIdExt, BlockInfo, BlockSignatures, BlockSignaturesPure, BuilderData, ConfigParams, CryptoSignature, CryptoSignaturePair, Deserializable, Ed25519KeyOption, GlobalCapabilities, HashmapType, KeyId, Message, Result, Serializable, Sha256, ShardIdent, SigPubKey, UInt256, UnixTime32, ValidatorBaseInfo, ValidatorDescr, ValidatorSet, Workchains, WorkchainDescr, AccountIdPrefixFull}; use ever_block::CatchainConfig; use std::{collections::HashMap, fmt::Debug, hash::Hash, sync::Arc}; use ton_api::ton::engine::validator::validator::groupmember::GroupMember; @@ -418,12 +412,29 @@ pub fn calc_subset_for_workchain_standard( pub async fn get_shard_by_message(engine: Arc, message: Arc) -> Result { let dst_wc = message.dst_workchain_id() .ok_or_else(|| error!("Can't get workchain id from message"))?; - let dst_address = message.int_dst_account_id() + let mut dst_address = message.int_dst_account_id() .ok_or_else(|| error!("Can't get standart destination address from message"))?; + let last_mc_state = engine.load_last_applied_mc_state().await?; + let mut shards_vec = Vec::new(); + last_mc_state.shards()?.iterate_shards_for_workchain( + dst_wc, |s, _| { shards_vec.push(s); Ok(true) } + )?; + + if shards_vec.len() == 0 { + return Ok(ShardIdent::masterchain()) + } + + let shard_id = shards_vec + .get(dst_address.hash(0).first_u64() as usize % shards_vec.len()) + .ok_or_else(|| error!("No shards"))?; + // find account and related shard - let (_account, shard) = engine.load_account(dst_wc, dst_address.clone()).await?; - Ok(shard) + //let (_account, shard) = engine.load_account(dst_wc, dst_address.clone()).await?; + + log::trace!(target: "remp", "get_shard_by_message: message {:x}: shard {}", dst_address, shard_id); + + Ok(shard_id.clone()) } pub fn get_first_block_seqno_after_prevs(prevs: &Vec) -> Option { From 73eddd9e5e8a85dbcaa6efa8ba34d752a0e22ee3 Mon Sep 17 00:00:00 2001 From: Dmitry Shtukenberg Date: Wed, 15 May 2024 13:50:11 +0300 Subject: [PATCH 40/40] Debug info improved --- src/validator/remp_catchain.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/validator/remp_catchain.rs b/src/validator/remp_catchain.rs index ec58e7ce..40a0e248 100644 --- a/src/validator/remp_catchain.rs +++ b/src/validator/remp_catchain.rs @@ -603,7 +603,7 @@ impl CatchainListener for RempCatchain { } fn process_blocks(&self, blocks: Vec) { - log::trace!(target: "remp", "Processing RMQ {}: new external messages, len = {}", self, blocks.len()); + log::trace!(target: "remp", "Processing RMQ {}: new external messages, len {}", self, blocks.len()); let mut msg_vect: Vec<::ton_api::ton::validator_session::round::Message> = Vec::new(); let mut msg_ids: Vec = Vec::new(); @@ -638,8 +638,8 @@ impl CatchainListener for RempCatchain { catchain::CatchainFactory::create_block_payload( serialized_payload.clone(), ), false, false); - log::trace!(target: "remp", "Point 3. RMQ {} sent messages: '{:?}'", - self, msg_ids + log::trace!(target: "remp", "Point 3. RMQ {} sent messages, cnt {}, len {}: '{:?}'", + self, msg_ids.len(), serialized_payload.len(), msg_ids ); #[cfg(feature = "telemetry")] self.engine.remp_core_telemetry().sent_to_catchain(