Add --archive-preload-period (#904)

Co-authored-by: SpyCheese <[email protected]>

validator-engine/validator-engine.cpp

@@ -1365,6 +1365,7 @@ td::Status ValidatorEngine::load_global_config() {
   }
   validator_options_.write().set_celldb_compress_depth(celldb_compress_depth_);
   validator_options_.write().set_max_open_archive_files(max_open_archive_files_);
+  validator_options_.write().set_archive_preload_period(archive_preload_period_);
 
   std::vector<ton::BlockIdExt> h;
   for (auto &x : conf.validator_->hardforks_) {
@@ -3801,6 +3802,16 @@ int main(int argc, char *argv[]) {
         acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_max_open_archive_files, v); });
         return td::Status::OK();
       });
+  p.add_checked_option(
+      '\0', "archive-preload-period", "open archive slices for the past X second on startup (default: 0)",
+      [&](td::Slice s) -> td::Status {
+        auto v = td::to_double(s);
+        if (v < 0) {
+          return td::Status::Error("sync-before should be non-negative");
+        }
+        acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_archive_preload_period, v); });
+        return td::Status::OK();
+      });
   auto S = p.run(argc, argv);
   if (S.is_error()) {
     LOG(ERROR) << "failed to parse options: " << S.move_as_error();

validator-engine/validator-engine.hpp

@@ -205,6 +205,7 @@ class ValidatorEngine : public td::actor::Actor {
   double key_proof_ttl_ = 0;
   td::uint32 celldb_compress_depth_ = 0;
   size_t max_open_archive_files_ = 0;
+  double archive_preload_period_ = 0.0;
   bool read_config_ = false;
   bool started_keyring_ = false;
   bool started_ = false;
@@ -268,6 +269,9 @@ class ValidatorEngine : public td::actor::Actor {
   void set_max_open_archive_files(size_t value) {
     max_open_archive_files_ = value;
   }
+  void set_archive_preload_period(double value) {
+    archive_preload_period_ = value;
+  }
   void start_up() override;
   ValidatorEngine() {
   }

validator/db/archive-manager.cpp

@@ -885,6 +885,24 @@ void ArchiveManager::start_up() {
   }).ensure();
 
   persistent_state_gc(FileHash::zero());
+
+  double open_since = td::Clocks::system() - opts_->get_archive_preload_period();
+  for (auto it = files_.rbegin(); it != files_.rend(); ++it) {
+    if (it->second.file_actor_id().empty()) {
+      continue;
+    }
+    td::actor::send_closure(it->second.file_actor_id(), &ArchiveSlice::open_files);
+    bool stop = true;
+    for (const auto &first_block : it->second.first_blocks) {
+      if ((double)first_block.second.ts >= open_since) {
+        stop = false;
+        break;
+      }
+    }
+    if (stop) {
+      break;
+    }
+  }
 }
 
 void ArchiveManager::run_gc(UnixTime mc_ts, UnixTime gc_ts, UnixTime archive_ttl) {

validator/db/archive-slice.cpp

@@ -525,6 +525,10 @@ void ArchiveSlice::before_query() {
   }
 }
 
+void ArchiveSlice::open_files() {
+  before_query();
+}
+
 void ArchiveSlice::close_files() {
   if (status_ == st_open) {
     if (active_queries_ == 0) {

validator/db/archive-slice.hpp

@@ -108,6 +108,7 @@ class ArchiveSlice : public td::actor::Actor {
 
   void set_async_mode(bool mode, td::Promise<td::Unit> promise);
 
+  void open_files();
   void close_files();
 
  private:

validator/validator-options.hpp

@@ -120,6 +120,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions {
   size_t get_max_open_archive_files() const override {
     return max_open_archive_files_;
   }
+  double get_archive_preload_period() const override {
+    return archive_preload_period_;
+  }
 
   void set_zero_block_id(BlockIdExt block_id) override {
     zero_block_id_ = block_id;
@@ -179,6 +182,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions {
   void set_max_open_archive_files(size_t value) override {
     max_open_archive_files_ = value;
   }
+  void set_archive_preload_period(double value) override {
+    archive_preload_period_ = value;
+  }
 
   ValidatorManagerOptionsImpl *make_copy() const override {
     return new ValidatorManagerOptionsImpl(*this);
@@ -223,6 +229,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions {
   std::string session_logs_file_;
   td::uint32 celldb_compress_depth_{0};
   size_t max_open_archive_files_ = 0;
+  double archive_preload_period_ = 0.0;
 };
 
 }  // namespace validator

validator/validator.h

@@ -83,6 +83,7 @@ struct ValidatorManagerOptions : public td::CntObject {
   virtual std::string get_session_logs_file() const = 0;
   virtual td::uint32 get_celldb_compress_depth() const = 0;
   virtual size_t get_max_open_archive_files() const = 0;
+  virtual double get_archive_preload_period() const = 0;
 
   virtual void set_zero_block_id(BlockIdExt block_id) = 0;
   virtual void set_init_block_id(BlockIdExt block_id) = 0;
@@ -104,6 +105,7 @@ struct ValidatorManagerOptions : public td::CntObject {
   virtual void set_session_logs_file(std::string f) = 0;
   virtual void set_celldb_compress_depth(td::uint32 value) = 0;
   virtual void set_max_open_archive_files(size_t value) = 0;
+  virtual void set_archive_preload_period(double value) = 0;
 
   static td::Ref<ValidatorManagerOptions> create(
       BlockIdExt zero_block_id, BlockIdExt init_block_id,