Merkle management layer (vmware#2947)

* Initial Merkle management layer.

Processing infrastructure for the updates to the Merkle trees.
Added IMerkleBuilder, EmptyMerkleBuilder,
initial BaseMerkleBuilder and SyncMerkleBuilder.

* Added abstraction layer.

The introduced abstraction layer is meant to hide
the details of the exact mechanism used to build and
maintain the proofs for validity of the state.

* Member functions const correctness fix + argument changed to const reference.

* Change builder to processor.

* Formatting changes.

* Added description.

* Move implementations to cpp file.

* Return from getProof changed to std::vector<concord::Byte>.

kvbc/CMakeLists.txt

@@ -57,7 +57,10 @@ if (BUILD_ROCKSDB_STORAGE)
             src/v4blockchain/detail/keys_history.cpp
                                 src/v4blockchain/detail/blocks.cpp
                                 src/v4blockchain/detail/st_chain.cpp
-                                src/v4blockchain/detail/blockchain.cpp)
+                                src/v4blockchain/detail/blockchain.cpp
+                                src/v4blockchain/detail/verifiability_management_layer.cpp
+                                src/merkle_processor/BaseMerkleProcessor.cpp
+                                src/merkle_processor/SyncMerkleProcessor.cpp)
 
 endif (BUILD_ROCKSDB_STORAGE)
 target_link_libraries(kvbc PUBLIC corebft util concord-crypto)

kvbc/include/merkle_processor/BaseMerkleProcessor.h

@@ -0,0 +1,35 @@
+// Concord
+//
+// Copyright (c) 2023 VMware, Inc. All Rights Reserved.
+//
+// This product is licensed to you under the Apache 2.0 license (the "License").
+// You may not use this product except in compliance with the Apache 2.0 License.
+//
+// This product may include a number of subcomponents with separate copyright
+// notices and license terms. Your use of these subcomponents is subject to the
+// terms and conditions of the sub-component's license, as noted in the
+// LICENSE file.
+
+#pragma once
+
+#include "IMerkleProcessor.h"
+#include "kvbc_app_filter/kvbc_key_types.h"
+#include "categorization/base_types.h"
+
+namespace concord {
+namespace kvbc {
+namespace sparse_merkle {
+
+using Hasher = concord::kvbc::categorization::Hasher;
+
+class BaseMerkleProcessor : public IMerkleProcessor {
+ public:
+  bool needProcessing(char type) const;
+  address getAddress(const std::string& key) const;
+
+  virtual void ProcessUpdates(const categorization::Updates& updates) override;
+};
+
+}  // namespace sparse_merkle
+}  // namespace kvbc
+}  // namespace concord

kvbc/include/merkle_processor/IMerkleProcessor.h

@@ -0,0 +1,52 @@
+// Concord
+//
+// Copyright (c) 2023 VMware, Inc. All Rights Reserved.
+//
+// This product is licensed to you under the Apache 2.0 license (the "License").
+// You may not use this product except in compliance with the Apache 2.0 License.
+//
+// This product may include a number of subcomponents with separate copyright
+// notices and license terms. Your use of these subcomponents is subject to the
+// terms and conditions of the sub-component's license, as noted in the
+// LICENSE file.
+
+#pragma once
+
+//#include <stdint.h>
+#include <string>
+#include <vector>
+#include "categorization/updates.h"
+#include "proof_processor/IProofProcessor.h"
+#include "util/sliver.hpp"
+
+namespace concord {
+namespace kvbc {
+namespace sparse_merkle {
+
+// This is an interface used for managing a set of Merle trees' storage.
+class IMerkleProcessor : public IProofProcessor {
+ public:
+  using address = concordUtils::Sliver;
+  static const unsigned int address_size = 20;
+  using value = std::array<unsigned char, 32>;
+
+  virtual ~IMerkleProcessor() = default;
+
+  virtual void Init(uint numVersionsStored) override = 0;  // from IProofProcessor
+
+  virtual void BeginVersionUpdateBatch() = 0;
+  virtual void UpdateAccountTree(const address& addr, const std::string& key, const value& data) = 0;
+  virtual void CommitVersionUpdateBatch() = 0;
+
+  virtual void ProcessUpdates(const categorization::Updates& updates) override = 0;  // from IProofProcessor
+
+  virtual std::vector<std::string> GetAccountMerkleRootPath(const address& addr) = 0;
+  virtual std::vector<std::string> GetAccountStorageKeyMerklePath(const address& addr, const std::string& key) = 0;
+  virtual std::vector<concord::Byte> GetProof(const std::string& key) override = 0;  // from IProofProcessor
+  virtual bool VerifyMerkleTreePath(std::string root_hash, std::string key, std::vector<std::string> path) = 0;
+  virtual void WaitForScheduledTasks() override = 0;  // from IProofProcessor
+};
+
+}  // namespace sparse_merkle
+}  // namespace kvbc
+}  // namespace concord

kvbc/include/merkle_processor/SyncMerkleProcessor.h

@@ -0,0 +1,43 @@
+// Concord
+//
+// Copyright (c) 2023 VMware, Inc. All Rights Reserved.
+//
+// This product is licensed to you under the Apache 2.0 license (the "License").
+// You may not use this product except in compliance with the Apache 2.0 License.
+//
+// This product may include a number of subcomponents with separate copyright
+// notices and license terms. Your use of these subcomponents is subject to the
+// terms and conditions of the sub-component's license, as noted in the
+// LICENSE file.
+
+#pragma once
+
+#include "BaseMerkleProcessor.h"
+
+namespace concord {
+namespace kvbc {
+namespace sparse_merkle {
+
+using concordUtils::bufferToHex;
+
+class SyncMerkleProcessor : public BaseMerkleProcessor {
+ public:
+  virtual ~SyncMerkleProcessor() = default;
+
+  virtual void Init(uint numVersionsStored) override;
+
+  virtual void BeginVersionUpdateBatch() override;
+  virtual void UpdateAccountTree(const address& addr, const std::string& key, const value& data) override;
+
+  virtual void CommitVersionUpdateBatch() override;
+
+  virtual std::vector<std::string> GetAccountMerkleRootPath(const address& addr) override;
+  virtual std::vector<std::string> GetAccountStorageKeyMerklePath(const address& addr, const std::string& key) override;
+  virtual std::vector<concord::Byte> GetProof(const std::string& key) override;
+  virtual bool VerifyMerkleTreePath(std::string root_hash, std::string key, std::vector<std::string> path) override;
+  virtual void WaitForScheduledTasks() override;
+};
+
+}  // namespace sparse_merkle
+}  // namespace kvbc
+}  // namespace concord

kvbc/include/proof_processor/EmptyProofProcessor.h

@@ -0,0 +1,34 @@
+// Concord
+//
+// Copyright (c) 2023 VMware, Inc. All Rights Reserved.
+//
+// This product is licensed to you under the Apache 2.0 license (the "License").
+// You may not use this product except in compliance with the Apache 2.0 License.
+//
+// This product may include a number of subcomponents with separate copyright
+// notices and license terms. Your use of these subcomponents is subject to the
+// terms and conditions of the sub-component's license, as noted in the
+// LICENSE file.
+
+#pragma once
+
+#include "IProofProcessor.h"
+#include "kvbc_app_filter/kvbc_key_types.h"
+
+namespace concord {
+namespace kvbc {
+
+class EmptyProofProcessor : public IProofProcessor {
+ public:
+  virtual ~EmptyProofProcessor() = default;
+
+  virtual void Init(uint numVersionsStored) override {}
+
+  virtual void ProcessUpdates(const categorization::Updates& updates) override {}
+
+  virtual std::vector<concord::Byte> GetProof(const std::string& key) override { return {}; }
+  virtual void WaitForScheduledTasks() override {}
+};
+
+}  // namespace kvbc
+}  // namespace concord

kvbc/include/proof_processor/IProofProcessor.h

@@ -0,0 +1,41 @@
+// Concord
+//
+// Copyright (c) 2023 VMware, Inc. All Rights Reserved.
+//
+// This product is licensed to you under the Apache 2.0 license (the "License").
+// You may not use this product except in compliance with the Apache 2.0 License.
+//
+// This product may include a number of subcomponents with separate copyright
+// notices and license terms. Your use of these subcomponents is subject to the
+// terms and conditions of the sub-component's license, as noted in the
+// LICENSE file.
+
+#pragma once
+
+//#include <stdint.h>
+#include <string>
+#include <vector>
+#include "categorization/updates.h"
+#include "util/sliver.hpp"
+#include "util/types.hpp"
+
+namespace concord {
+namespace kvbc {
+
+// This is an interface of a general proof builder.
+class IProofProcessor {
+ public:
+  virtual ~IProofProcessor() = default;
+
+  // Initialize with proper parameters.
+  virtual void Init(uint numVersionsStored) = 0;
+  // Process all the updates that will be persisted.
+  virtual void ProcessUpdates(const categorization::Updates& updates) = 0;
+  // Get a serialized object serving as proof for correctness.
+  virtual std::vector<concord::Byte> GetProof(const std::string& key) = 0;
+  // In case of async processing we use this call to sync and wait for all async tasks to complete.
+  virtual void WaitForScheduledTasks() = 0;
+};
+
+}  // namespace kvbc
+}  // namespace concord

kvbc/include/v4blockchain/detail/verifiability_management_layer.h

@@ -0,0 +1,39 @@
+// Concord
+//
+// Copyright (c) 2022 VMware, Inc. All Rights Reserved.
+//
+// This product is licensed to you under the Apache 2.0 license (the
+// "License").  You may not use this product except in compliance with the
+// Apache 2.0 License.
+//
+// This product may include a number of subcomponents with separate copyright
+// notices and license terms. Your use of these subcomponents is subject to the
+// terms and conditions of the subcomponent's license, as noted in the LICENSE
+// file.
+
+#pragma once
+
+#include "rocksdb/native_client.h"
+#include <memory>
+#include <unordered_map>
+#include "categorization/updates.h"
+#include "v4blockchain/detail/categories.h"
+#include <rocksdb/compaction_filter.h>
+#include "rocksdb/snapshot.h"
+#include "proof_processor/IProofProcessor.h"
+
+namespace concord::kvbc::v4blockchain::detail {
+
+using namespace concord::kvbc;
+
+class VerifiabilityManagementLayer {
+ public:
+  VerifiabilityManagementLayer();
+
+  IProofProcessor* getProcessor() { return proofProcessor_.get(); }
+
+ private:
+  std::unique_ptr<IProofProcessor> proofProcessor_;
+};
+
+}  // namespace concord::kvbc::v4blockchain::detail

kvbc/include/v4blockchain/v4_blockchain.h

@@ -18,6 +18,7 @@
 #include "v4blockchain/detail/st_chain.h"
 #include "v4blockchain/detail/latest_keys.h"
 #include "v4blockchain/detail/blockchain.h"
+#include "v4blockchain/detail/verifiability_management_layer.h"
 #include "v4blockchain/detail/keys_history.h"
 #include <memory>
 #include <string>
@@ -86,6 +87,9 @@ class KeyValueBlockchain {
   const v4blockchain::detail::Blockchain &getBlockchain() const { return block_chain_; };
   const v4blockchain::detail::StChain &getStChain() const { return state_transfer_chain_; };
   const v4blockchain::detail::LatestKeys &getLatestKeys() const { return latest_keys_; };
+  v4blockchain::detail::VerifiabilityManagementLayer &getVerifiabilityManagementLayer() {
+    return verifiability_management_layer_;
+  }
   /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
   // IReader
   std::optional<categorization::Value> get(const std::string &category_id,
@@ -274,6 +278,8 @@ class KeyValueBlockchain {
   v4blockchain::detail::StChain state_transfer_chain_;
   v4blockchain::detail::LatestKeys latest_keys_;
   v4blockchain::detail::KeysHistory keys_history_;
+  v4blockchain::detail::VerifiabilityManagementLayer verifiability_management_layer_;
+
   // flag to mark whether a checkpoint is being taken.
   std::optional<uint64_t> last_block_sn_;
   const float updates_to_final_size_ration_{2.5};

kvbc/src/merkle_processor/BaseMerkleProcessor.cpp

@@ -0,0 +1,68 @@
+// Concord
+//
+// Copyright (c) 2023 VMware, Inc. All Rights Reserved.
+//
+// This product is licensed to you under the Apache 2.0 license (the "License").
+// You may not use this product except in compliance with the Apache 2.0 License.
+//
+// This product may include a number of subcomponents with separate copyright
+// notices and license terms. Your use of these subcomponents is subject to the
+// terms and conditions of the sub-component's license, as noted in the
+// LICENSE file.
+
+#include "merkle_processor/BaseMerkleProcessor.h"
+
+namespace concord {
+namespace kvbc {
+namespace sparse_merkle {
+
+bool BaseMerkleProcessor::needProcessing(char type) const {
+  if (type == kKvbKeyEthBalance || type == kKvbKeyEthCode || type == kKvbKeyEthStorage || type == kKvbKeyEthNonce) {
+    return true;
+  }
+  return false;
+}
+
+IMerkleProcessor::address BaseMerkleProcessor::getAddress(const std::string& key) const {
+  ConcordAssert(key.size() > IMerkleProcessor::address_size);  // Include 1 byte indicating key type
+  return address(&key[1], IMerkleProcessor::address_size);
+}
+
+void BaseMerkleProcessor::ProcessUpdates(const categorization::Updates& updates) {
+  BeginVersionUpdateBatch();
+  for (const auto& k : updates.categoryUpdates().kv) {
+    if (const categorization::BlockMerkleInput* pval = std::get_if<categorization::BlockMerkleInput>(&k.second)) {
+      for (const auto& v : pval->kv) {
+        if (needProcessing(v.first[0])) {
+          auto hasher = Hasher{};
+          const auto value_hash = hasher.digest(v.second.data(), v.second.size());
+          LOG_INFO(V4_BLOCK_LOG, "MerkleProcessor: categorization::BlockMerkleInput");
+          UpdateAccountTree(getAddress(v.first), v.first, value_hash);
+        }
+      }
+    } else if (const categorization::VersionedInput* pval = std::get_if<categorization::VersionedInput>(&k.second)) {
+      for (const auto& v : pval->kv) {
+        if (needProcessing(v.first[0])) {
+          auto hasher = Hasher{};
+          const auto value_hash = hasher.digest(v.second.data.data(), v.second.data.size());
+          LOG_INFO(V4_BLOCK_LOG, "MerkleProcessor: categorization::VersionedInput");
+          UpdateAccountTree(getAddress(v.first), v.first, value_hash);
+        }
+      }
+    } else if (const categorization::ImmutableInput* pval = std::get_if<categorization::ImmutableInput>(&k.second)) {
+      for (const auto& v : pval->kv) {
+        if (needProcessing(v.first[0])) {
+          auto hasher = Hasher{};
+          const auto value_hash = hasher.digest(v.second.data.data(), v.second.data.size());
+          LOG_INFO(V4_BLOCK_LOG, "MerkleProcessor: categorization::ImmutableInput");
+          UpdateAccountTree(getAddress(v.first), v.first, value_hash);
+        }
+      }
+    }
+  }
+  CommitVersionUpdateBatch();
+}
+
+}  // namespace sparse_merkle
+}  // namespace kvbc
+}  // namespace concord