diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index cfc9f3a19..14e0121fd 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -406,6 +406,7 @@ tonNode.preparedProof = tonNode.PreparedProof; tonNode.preparedProofLink = tonNode.PreparedProof; tonNode.preparedState = tonNode.PreparedState; tonNode.notFoundState = tonNode.PreparedState; +tonNode.persistentStateSize size:long = tonNode.PersistentStateSize; tonNode.prepared = tonNode.Prepared; tonNode.notFound = tonNode.Prepared; tonNode.data data:bytes = tonNode.Data; @@ -472,6 +473,7 @@ tonNode.prepareKeyBlockProofs blocks:(vector tonNode.blockIdExt) allow_partial:B tonNode.prepareBlock block:tonNode.blockIdExt = tonNode.Prepared; tonNode.prepareBlocks blocks:(vector tonNode.blockIdExt) = tonNode.Prepared; tonNode.preparePersistentState block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt = tonNode.PreparedState; +tonNode.getPersistentStateSize block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt = tonNode.PersistentStateSize; tonNode.prepareZeroState block:tonNode.blockIdExt = tonNode.PreparedState; tonNode.getNextKeyBlockIds block:tonNode.blockIdExt max_size:int = tonNode.KeyBlocks; tonNode.downloadNextBlockFull prev_block:tonNode.blockIdExt = tonNode.DataFull; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 96ecb7751..5d9ccc2d8 100644 Binary files a/tl/generate/scheme/ton_api.tlo and b/tl/generate/scheme/ton_api.tlo differ diff --git a/validator/db/archive-manager.cpp b/validator/db/archive-manager.cpp index 8c7cde170..9b1ebf6c2 100644 --- a/validator/db/archive-manager.cpp +++ b/validator/db/archive-manager.cpp @@ -314,7 +314,15 @@ void ArchiveManager::register_perm_state(FileReferenceShort id) { BlockSeqno masterchain_seqno = 0; id.ref().visit(td::overloaded( [&](const fileref::PersistentStateShort &x) { masterchain_seqno = x.masterchain_seqno; }, [&](const auto &) {})); - perm_states_[{masterchain_seqno, id.hash()}] = id; + td::uint64 size; + auto r_stat = td::stat(db_root_ + "/archive/states/" + id.filename_short()); + if (r_stat.is_error()) { + LOG(WARNING) << "Cannot stat persistent state file " << id.filename_short() << " : " << r_stat.move_as_error(); + size = 0; + } else { + size = r_stat.ok().size_; + } + perm_states_[{masterchain_seqno, id.hash()}] = {.id = id, .size = size}; } void ArchiveManager::add_zero_state(BlockIdExt block_id, td::BufferSlice data, td::Promise promise) { @@ -417,7 +425,7 @@ void ArchiveManager::get_previous_persistent_state_files( BlockSeqno mc_seqno = it->first.first; std::vector> files; while (it->first.first == mc_seqno) { - files.emplace_back(db_root_ + "/archive/states/" + it->second.filename_short(), it->second.shard()); + files.emplace_back(db_root_ + "/archive/states/" + it->second.id.filename_short(), it->second.id.shard()); if (it == perm_states_.begin()) { break; } @@ -452,15 +460,16 @@ void ArchiveManager::get_persistent_state_slice(BlockIdExt block_id, BlockIdExt td::actor::create_actor("readfile", path, offset, max_size, 0, std::move(promise)).release(); } -void ArchiveManager::check_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) { +void ArchiveManager::get_persistent_state_file_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) { auto id = FileReference{fileref::PersistentState{block_id, masterchain_block_id}}; auto hash = id.hash(); - if (perm_states_.find({masterchain_block_id.seqno(), hash}) == perm_states_.end()) { - promise.set_result(false); + auto it = perm_states_.find({masterchain_block_id.seqno(), hash}); + if (it == perm_states_.end()) { + promise.set_error(td::Status::Error(ErrorCode::notready)); return; } - promise.set_result(true); + promise.set_result(it->second.size); } void ArchiveManager::get_block_by_unix_time(AccountIdPrefixFull account_id, UnixTime ts, @@ -1023,15 +1032,15 @@ void ArchiveManager::persistent_state_gc(std::pair last) { int res = 0; BlockSeqno seqno = 0; - F.ref().visit(td::overloaded([&](const fileref::ZeroStateShort &) { res = 1; }, - [&](const fileref::PersistentStateShort &x) { - res = 0; - seqno = x.masterchain_seqno; - }, - [&](const auto &obj) { res = -1; })); + F.id.ref().visit(td::overloaded([&](const fileref::ZeroStateShort &) { res = 1; }, + [&](const fileref::PersistentStateShort &x) { + res = 0; + seqno = x.masterchain_seqno; + }, + [&](const auto &obj) { res = -1; })); if (res == -1) { - td::unlink(db_root_ + "/archive/states/" + F.filename_short()).ignore(); + td::unlink(db_root_ + "/archive/states/" + F.id.filename_short()).ignore(); perm_states_.erase(it); } if (res != 0) { @@ -1081,7 +1090,7 @@ void ArchiveManager::got_gc_masterchain_handle(ConstBlockHandle handle, std::pai CHECK(it != perm_states_.end()); auto &F = it->second; if (to_del) { - td::unlink(db_root_ + "/archive/states/" + F.filename_short()).ignore(); + td::unlink(db_root_ + "/archive/states/" + F.id.filename_short()).ignore(); perm_states_.erase(it); } delay_action( @@ -1202,12 +1211,7 @@ void ArchiveManager::prepare_stats(td::Promise states; for (auto &[key, file] : perm_states_) { BlockSeqno seqno = key.first; - auto r_stat = td::stat(db_root_ + "/archive/states/" + file.filename_short()); - if (r_stat.is_error()) { - LOG(WARNING) << "Cannot stat persistent state file " << file.filename_short() << " : " << r_stat.move_as_error(); - } else { - states[seqno] += r_stat.move_as_ok().size_; - } + states[seqno] += file.size; } td::StringBuilder sb; for (auto &[seqno, size] : states) { @@ -1308,7 +1312,7 @@ void ArchiveManager::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle han auto it = perm_states_.begin(); while (it != perm_states_.end()) { int res = 0; - it->second.ref().visit(td::overloaded( + it->second.id.ref().visit(td::overloaded( [&](const fileref::ZeroStateShort &x) { res = -1; }, [&](const fileref::PersistentStateShort &x) { res = x.masterchain_seqno <= masterchain_seqno ? -1 : 1; }, [&](const auto &obj) { res = 1; })); @@ -1317,7 +1321,7 @@ void ArchiveManager::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle han } else { auto it2 = it; it++; - td::unlink(db_root_ + "/archive/states/" + it2->second.filename_short()).ignore(); + td::unlink(db_root_ + "/archive/states/" + it2->second.id.filename_short()).ignore(); perm_states_.erase(it2); } } diff --git a/validator/db/archive-manager.hpp b/validator/db/archive-manager.hpp index d919e32ee..cd79ccc6e 100644 --- a/validator/db/archive-manager.hpp +++ b/validator/db/archive-manager.hpp @@ -52,7 +52,8 @@ class ArchiveManager : public td::actor::Actor { void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise); void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, td::int64 max_size, td::Promise promise); - void check_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise); + void get_persistent_state_file_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise); void check_zero_state(BlockIdExt block_id, td::Promise promise); void get_previous_persistent_state_files(BlockSeqno cur_mc_seqno, td::Promise>> promise); @@ -189,7 +190,11 @@ class ArchiveManager : public td::actor::Actor { return p.key ? key_files_ : p.temp ? temp_files_ : files_; } - std::map, FileReferenceShort> perm_states_; // Mc block seqno, hash -> state + struct PermState { + FileReferenceShort id; + td::uint64 size; + }; + std::map, PermState> perm_states_; // Mc block seqno, hash -> state void load_package(PackageId seqno); void delete_package(PackageId seqno, td::Promise promise); diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index 8d83e7a7d..2b370eb06 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -310,9 +310,9 @@ void RootDb::get_persistent_state_file_slice(BlockIdExt block_id, BlockIdExt mas offset, max_size, std::move(promise)); } -void RootDb::check_persistent_state_file_exists(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) { - td::actor::send_closure(archive_db_, &ArchiveManager::check_persistent_state, block_id, masterchain_block_id, +void RootDb::get_persistent_state_file_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) { + td::actor::send_closure(archive_db_, &ArchiveManager::get_persistent_state_file_size, block_id, masterchain_block_id, std::move(promise)); } diff --git a/validator/db/rootdb.hpp b/validator/db/rootdb.hpp index 52f6098e4..9e5b0b302 100644 --- a/validator/db/rootdb.hpp +++ b/validator/db/rootdb.hpp @@ -80,8 +80,8 @@ class RootDb : public Db { td::Promise promise) override; void get_persistent_state_file_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, td::int64 max_length, td::Promise promise) override; - void check_persistent_state_file_exists(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) override; + void get_persistent_state_file_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) override; void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) override; void get_zero_state_file(BlockIdExt block_id, td::Promise promise) override; void check_zero_state_file_exists(BlockIdExt block_id, td::Promise promise) override; diff --git a/validator/full-node-master.cpp b/validator/full-node-master.cpp index da49f0e2e..8dc2a6815 100644 --- a/validator/full-node-master.cpp +++ b/validator/full-node-master.cpp @@ -275,20 +275,32 @@ void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNo void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_preparePersistentState &query, td::Promise promise) { - auto P = - td::PromiseCreator::lambda([SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { - if (R.is_error() || !R.move_as_ok()) { + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { auto x = create_serialize_tl_object(); promise.set_value(std::move(x)); return; } - auto x = create_serialize_tl_object(); promise.set_value(std::move(x)); }); auto block_id = create_block_id(query.block_); auto masterchain_block_id = create_block_id(query.masterchain_block_); - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::check_persistent_state_exists, block_id, + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_size, block_id, + masterchain_block_id, std::move(P)); +} + +void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSize &query, + td::Promise promise) { + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + TRY_RESULT_PROMISE(promise, size, std::move(R)); + promise.set_value(create_serialize_tl_object(size)); + }); + auto block_id = create_block_id(query.block_); + auto masterchain_block_id = create_block_id(query.masterchain_block_); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_size, block_id, masterchain_block_id, std::move(P)); } @@ -389,6 +401,21 @@ void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNo ShardIdFull{masterchainId}, std::move(P)); } +void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getShardArchiveInfo &query, + td::Promise promise) { + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_serialize_tl_object()); + } else { + promise.set_value(create_serialize_tl_object(R.move_as_ok())); + } + }); + ShardIdFull shard_prefix = create_shard_id(query.shard_prefix_); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_archive_id, query.masterchain_seqno_, + shard_prefix, std::move(P)); +} + void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveSlice &query, td::Promise promise) { td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_archive_slice, query.archive_id_, diff --git a/validator/full-node-master.hpp b/validator/full-node-master.hpp index ce0aedd35..d6160a94e 100644 --- a/validator/full-node-master.hpp +++ b/validator/full-node-master.hpp @@ -66,6 +66,8 @@ class FullNodeMasterImpl : public FullNodeMaster { td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_preparePersistentState &query, td::Promise promise); + void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSize &query, + td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getNextKeyBlockIds &query, td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadZeroState &query, @@ -80,6 +82,8 @@ class FullNodeMasterImpl : public FullNodeMaster { td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveInfo &query, td::Promise promise); + void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getShardArchiveInfo &query, + td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveSlice &query, td::Promise promise); // void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_prepareNextKeyBlockProof &query, diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index ac0eb7688..a8ddb338a 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -509,13 +509,12 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_preparePersistentState &query, td::Promise promise) { auto P = - td::PromiseCreator::lambda([SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { - if (R.is_error() || !R.move_as_ok()) { + td::PromiseCreator::lambda([SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { auto x = create_serialize_tl_object(); promise.set_value(std::move(x)); return; } - auto x = create_serialize_tl_object(); promise.set_value(std::move(x)); }); @@ -523,7 +522,22 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod auto masterchain_block_id = create_block_id(query.masterchain_block_); VLOG(FULL_NODE_DEBUG) << "Got query preparePersistentState " << block_id.to_str() << " " << masterchain_block_id.to_str() << " from " << src; - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::check_persistent_state_exists, block_id, + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_size, block_id, + masterchain_block_id, std::move(P)); +} + +void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSize &query, + td::Promise promise) { + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + TRY_RESULT_PROMISE(promise, size, std::move(R)); + promise.set_value(create_serialize_tl_object(size)); + }); + auto block_id = create_block_id(query.block_); + auto masterchain_block_id = create_block_id(query.masterchain_block_); + VLOG(FULL_NODE_DEBUG) << "Got query getPersistentStateSize " << block_id.to_str() << " " + << masterchain_block_id.to_str() << " from " << src; + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_size, block_id, masterchain_block_id, std::move(P)); } diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index fb3eef769..8cbd614aa 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -123,6 +123,8 @@ class FullNodeShardImpl : public FullNodeShard { td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_preparePersistentState &query, td::Promise promise); + void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSize &query, + td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getNextKeyBlockIds &query, td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadZeroState &query, diff --git a/validator/interfaces/db.h b/validator/interfaces/db.h index 29ef715b3..9a2eea56e 100644 --- a/validator/interfaces/db.h +++ b/validator/interfaces/db.h @@ -62,8 +62,8 @@ class Db : public td::actor::Actor { td::Promise promise) = 0; virtual void get_persistent_state_file_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, td::int64 max_length, td::Promise promise) = 0; - virtual void check_persistent_state_file_exists(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) = 0; + virtual void get_persistent_state_file_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) = 0; virtual void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) = 0; virtual void get_zero_state_file(BlockIdExt block_id, td::Promise promise) = 0; virtual void check_zero_state_file_exists(BlockIdExt block_id, td::Promise promise) = 0; diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index 62fdc4b43..8e4a4d086 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -200,9 +200,9 @@ void ValidatorManagerImpl::get_zero_state(BlockIdExt block_id, td::Promise promise) { - td::actor::send_closure(db_, &Db::check_persistent_state_file_exists, block_id, masterchain_block_id, +void ValidatorManagerImpl::get_persistent_state_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) { + td::actor::send_closure(db_, &Db::get_persistent_state_file_size, block_id, masterchain_block_id, std::move(promise)); } void ValidatorManagerImpl::get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index cd06bf555..62f54b2e9 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -108,8 +108,8 @@ class ValidatorManagerImpl : public ValidatorManager { void get_block_data(BlockHandle handle, td::Promise promise) override; void check_zero_state_exists(BlockIdExt block_id, td::Promise promise) override; void get_zero_state(BlockIdExt block_id, td::Promise promise) override; - void check_persistent_state_exists(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) override; + void get_persistent_state_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) override; void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise) override; void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 0b8b9e736..486c185f7 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -128,8 +128,8 @@ class ValidatorManagerImpl : public ValidatorManager { void check_zero_state_exists(BlockIdExt block_id, td::Promise promise) override { UNREACHABLE(); } - void check_persistent_state_exists(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) override { + void get_persistent_state_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) override { UNREACHABLE(); } void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, diff --git a/validator/manager.cpp b/validator/manager.cpp index b0ac54092..b1dc4cc9a 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -304,9 +304,9 @@ void ValidatorManagerImpl::get_zero_state(BlockIdExt block_id, td::Promise promise) { - td::actor::send_closure(db_, &Db::check_persistent_state_file_exists, block_id, masterchain_block_id, +void ValidatorManagerImpl::get_persistent_state_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) { + td::actor::send_closure(db_, &Db::get_persistent_state_file_size, block_id, masterchain_block_id, std::move(promise)); } void ValidatorManagerImpl::get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, diff --git a/validator/manager.hpp b/validator/manager.hpp index 418deb350..347b8f7ef 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -377,8 +377,8 @@ class ValidatorManagerImpl : public ValidatorManager { void get_block_data(BlockHandle handle, td::Promise promise) override; void check_zero_state_exists(BlockIdExt block_id, td::Promise promise) override; void get_zero_state(BlockIdExt block_id, td::Promise promise) override; - void check_persistent_state_exists(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) override; + void get_persistent_state_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) override; void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise) override; void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, diff --git a/validator/net/download-state.cpp b/validator/net/download-state.cpp index 6735a2b5f..c481adbbf 100644 --- a/validator/net/download-state.cpp +++ b/validator/net/download-state.cpp @@ -168,6 +168,7 @@ void DownloadState::got_block_state_description(td::BufferSlice data) { }, [&, self = this](ton_api::tonNode_preparedState &f) { if (masterchain_block_id_.is_valid()) { + request_total_size(); got_block_state_part(td::BufferSlice{}, 0); return; } @@ -190,8 +191,37 @@ void DownloadState::got_block_state_description(td::BufferSlice data) { create_serialize_tl_object_suffix(std::move(query)), td::Timestamp::in(3.0), std::move(P)); } + status_.set_status(PSTRING() << block_id_.id.to_str() << " : download started"); })); - status_.set_status(PSTRING() << block_id_.id.to_str() << " : 0 bytes, 0B/s"); +} + +void DownloadState::request_total_size() { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + return; + } + auto res = fetch_tl_object(R.move_as_ok(), true); + if (res.is_error()) { + return; + } + td::actor::send_closure(SelfId, &DownloadState::got_total_size, res.ok()->size_); + }); + + td::BufferSlice query = create_serialize_tl_object( + create_tl_block_id(block_id_), create_tl_block_id(masterchain_block_id_)); + if (client_.empty()) { + td::actor::send_closure(overlays_, &overlay::Overlays::send_query_via, download_from_, local_id_, overlay_id_, + "get size", std::move(P), td::Timestamp::in(3.0), std::move(query), + FullNode::max_state_size(), rldp_); + } else { + td::actor::send_closure(client_, &adnl::AdnlExtClient::send_query, "get size", + create_serialize_tl_object_suffix(std::move(query)), + td::Timestamp::in(3.0), std::move(P)); + } +} + +void DownloadState::got_total_size(td::uint64 size) { + total_size_ = size; } void DownloadState::got_block_state_part(td::BufferSlice data, td::uint32 requested_size) { @@ -203,10 +233,22 @@ void DownloadState::got_block_state_part(td::BufferSlice data, td::uint32 reques if (elapsed > 5.0) { prev_logged_timer_ = td::Timer(); auto speed = (td::uint64)((double)(sum_ - prev_logged_sum_) / elapsed); - LOG(WARNING) << "downloading state " << block_id_.to_str() << ": " << td::format::as_size(sum_) << " (" - << td::format::as_size(speed) << "/s)"; - status_.set_status(PSTRING() << block_id_.id.to_str() << " : " << sum_ << " bytes, " << td::format::as_size(speed) - << "/s"); + td::StringBuilder sb; + sb << td::format::as_size(sum_); + if (total_size_) { + sb << "/" << td::format::as_size(total_size_); + } + sb << " (" << td::format::as_size(speed) << "/s"; + if (total_size_) { + sb << ", " << td::StringBuilder::FixedDouble((double)sum_ / (double)total_size_ * 100.0, 2) << "%"; + if (speed > 0 && total_size_ >= sum_) { + td::uint64 rem = (total_size_ - sum_) / speed; + sb << ", " << rem << "s remaining"; + } + } + sb << ")"; + LOG(WARNING) << "downloading state " << block_id_.to_str() << " : " << sb.as_cslice(); + status_.set_status(PSTRING() << block_id_.id.to_str() << " : " << sb.as_cslice()); prev_logged_sum_ = sum_; } diff --git a/validator/net/download-state.hpp b/validator/net/download-state.hpp index 470c54318..29854ce27 100644 --- a/validator/net/download-state.hpp +++ b/validator/net/download-state.hpp @@ -49,6 +49,8 @@ class DownloadState : public td::actor::Actor { void got_block_handle(BlockHandle handle); void got_node_to_download(adnl::AdnlNodeIdShort node); void got_block_state_description(td::BufferSlice data_description); + void request_total_size(); + void got_total_size(td::uint64 size); void got_block_state_part(td::BufferSlice data, td::uint32 requested_size); void got_block_state(td::BufferSlice data); @@ -77,6 +79,7 @@ class DownloadState : public td::actor::Actor { td::uint64 prev_logged_sum_ = 0; td::Timer prev_logged_timer_; + td::uint64 total_size_ = 0; ProcessStatus status_; }; diff --git a/validator/validator.h b/validator/validator.h index 5d6c0173c..eecb30c07 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -227,8 +227,8 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void get_block_data(BlockHandle handle, td::Promise promise) = 0; virtual void check_zero_state_exists(BlockIdExt block_id, td::Promise promise) = 0; virtual void get_zero_state(BlockIdExt block_id, td::Promise promise) = 0; - virtual void check_persistent_state_exists(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) = 0; + virtual void get_persistent_state_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) = 0; virtual void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise) = 0; virtual void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset,