diff --git a/kvbc/CMakeLists.txt b/kvbc/CMakeLists.txt index 5e881b60dc..89050edfd5 100644 --- a/kvbc/CMakeLists.txt +++ b/kvbc/CMakeLists.txt @@ -57,7 +57,10 @@ if (BUILD_ROCKSDB_STORAGE) src/v4blockchain/detail/keys_history.cpp src/v4blockchain/detail/blocks.cpp src/v4blockchain/detail/st_chain.cpp - src/v4blockchain/detail/blockchain.cpp) + src/v4blockchain/detail/blockchain.cpp + src/v4blockchain/detail/verifiability_management_layer.cpp + src/merkle_processor/BaseMerkleProcessor.cpp + src/merkle_processor/SyncMerkleProcessor.cpp) endif (BUILD_ROCKSDB_STORAGE) target_link_libraries(kvbc PUBLIC corebft util concord-crypto) diff --git a/kvbc/include/merkle_processor/BaseMerkleProcessor.h b/kvbc/include/merkle_processor/BaseMerkleProcessor.h new file mode 100644 index 0000000000..e2b272ec14 --- /dev/null +++ b/kvbc/include/merkle_processor/BaseMerkleProcessor.h @@ -0,0 +1,35 @@ +// Concord +// +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// +// This product is licensed to you under the Apache 2.0 license (the "License"). +// You may not use this product except in compliance with the Apache 2.0 License. +// +// This product may include a number of subcomponents with separate copyright +// notices and license terms. Your use of these subcomponents is subject to the +// terms and conditions of the sub-component's license, as noted in the +// LICENSE file. + +#pragma once + +#include "IMerkleProcessor.h" +#include "kvbc_app_filter/kvbc_key_types.h" +#include "categorization/base_types.h" + +namespace concord { +namespace kvbc { +namespace sparse_merkle { + +using Hasher = concord::kvbc::categorization::Hasher; + +class BaseMerkleProcessor : public IMerkleProcessor { + public: + bool needProcessing(char type) const; + address getAddress(const std::string& key) const; + + virtual void ProcessUpdates(const categorization::Updates& updates) override; +}; + +} // namespace sparse_merkle +} // namespace kvbc +} // namespace concord diff --git a/kvbc/include/merkle_processor/IMerkleProcessor.h b/kvbc/include/merkle_processor/IMerkleProcessor.h new file mode 100644 index 0000000000..a87a1e6b74 --- /dev/null +++ b/kvbc/include/merkle_processor/IMerkleProcessor.h @@ -0,0 +1,52 @@ +// Concord +// +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// +// This product is licensed to you under the Apache 2.0 license (the "License"). +// You may not use this product except in compliance with the Apache 2.0 License. +// +// This product may include a number of subcomponents with separate copyright +// notices and license terms. Your use of these subcomponents is subject to the +// terms and conditions of the sub-component's license, as noted in the +// LICENSE file. + +#pragma once + +//#include +#include +#include +#include "categorization/updates.h" +#include "proof_processor/IProofProcessor.h" +#include "util/sliver.hpp" + +namespace concord { +namespace kvbc { +namespace sparse_merkle { + +// This is an interface used for managing a set of Merle trees' storage. +class IMerkleProcessor : public IProofProcessor { + public: + using address = concordUtils::Sliver; + static const unsigned int address_size = 20; + using value = std::array; + + virtual ~IMerkleProcessor() = default; + + virtual void Init(uint numVersionsStored) override = 0; // from IProofProcessor + + virtual void BeginVersionUpdateBatch() = 0; + virtual void UpdateAccountTree(const address& addr, const std::string& key, const value& data) = 0; + virtual void CommitVersionUpdateBatch() = 0; + + virtual void ProcessUpdates(const categorization::Updates& updates) override = 0; // from IProofProcessor + + virtual std::vector GetAccountMerkleRootPath(const address& addr) = 0; + virtual std::vector GetAccountStorageKeyMerklePath(const address& addr, const std::string& key) = 0; + virtual std::vector GetProof(const std::string& key) override = 0; // from IProofProcessor + virtual bool VerifyMerkleTreePath(std::string root_hash, std::string key, std::vector path) = 0; + virtual void WaitForScheduledTasks() override = 0; // from IProofProcessor +}; + +} // namespace sparse_merkle +} // namespace kvbc +} // namespace concord diff --git a/kvbc/include/merkle_processor/SyncMerkleProcessor.h b/kvbc/include/merkle_processor/SyncMerkleProcessor.h new file mode 100644 index 0000000000..5d90c724bd --- /dev/null +++ b/kvbc/include/merkle_processor/SyncMerkleProcessor.h @@ -0,0 +1,43 @@ +// Concord +// +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// +// This product is licensed to you under the Apache 2.0 license (the "License"). +// You may not use this product except in compliance with the Apache 2.0 License. +// +// This product may include a number of subcomponents with separate copyright +// notices and license terms. Your use of these subcomponents is subject to the +// terms and conditions of the sub-component's license, as noted in the +// LICENSE file. + +#pragma once + +#include "BaseMerkleProcessor.h" + +namespace concord { +namespace kvbc { +namespace sparse_merkle { + +using concordUtils::bufferToHex; + +class SyncMerkleProcessor : public BaseMerkleProcessor { + public: + virtual ~SyncMerkleProcessor() = default; + + virtual void Init(uint numVersionsStored) override; + + virtual void BeginVersionUpdateBatch() override; + virtual void UpdateAccountTree(const address& addr, const std::string& key, const value& data) override; + + virtual void CommitVersionUpdateBatch() override; + + virtual std::vector GetAccountMerkleRootPath(const address& addr) override; + virtual std::vector GetAccountStorageKeyMerklePath(const address& addr, const std::string& key) override; + virtual std::vector GetProof(const std::string& key) override; + virtual bool VerifyMerkleTreePath(std::string root_hash, std::string key, std::vector path) override; + virtual void WaitForScheduledTasks() override; +}; + +} // namespace sparse_merkle +} // namespace kvbc +} // namespace concord diff --git a/kvbc/include/proof_processor/EmptyProofProcessor.h b/kvbc/include/proof_processor/EmptyProofProcessor.h new file mode 100644 index 0000000000..80456567da --- /dev/null +++ b/kvbc/include/proof_processor/EmptyProofProcessor.h @@ -0,0 +1,34 @@ +// Concord +// +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// +// This product is licensed to you under the Apache 2.0 license (the "License"). +// You may not use this product except in compliance with the Apache 2.0 License. +// +// This product may include a number of subcomponents with separate copyright +// notices and license terms. Your use of these subcomponents is subject to the +// terms and conditions of the sub-component's license, as noted in the +// LICENSE file. + +#pragma once + +#include "IProofProcessor.h" +#include "kvbc_app_filter/kvbc_key_types.h" + +namespace concord { +namespace kvbc { + +class EmptyProofProcessor : public IProofProcessor { + public: + virtual ~EmptyProofProcessor() = default; + + virtual void Init(uint numVersionsStored) override {} + + virtual void ProcessUpdates(const categorization::Updates& updates) override {} + + virtual std::vector GetProof(const std::string& key) override { return {}; } + virtual void WaitForScheduledTasks() override {} +}; + +} // namespace kvbc +} // namespace concord diff --git a/kvbc/include/proof_processor/IProofProcessor.h b/kvbc/include/proof_processor/IProofProcessor.h new file mode 100644 index 0000000000..48f65d3a43 --- /dev/null +++ b/kvbc/include/proof_processor/IProofProcessor.h @@ -0,0 +1,41 @@ +// Concord +// +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// +// This product is licensed to you under the Apache 2.0 license (the "License"). +// You may not use this product except in compliance with the Apache 2.0 License. +// +// This product may include a number of subcomponents with separate copyright +// notices and license terms. Your use of these subcomponents is subject to the +// terms and conditions of the sub-component's license, as noted in the +// LICENSE file. + +#pragma once + +//#include +#include +#include +#include "categorization/updates.h" +#include "util/sliver.hpp" +#include "util/types.hpp" + +namespace concord { +namespace kvbc { + +// This is an interface of a general proof builder. +class IProofProcessor { + public: + virtual ~IProofProcessor() = default; + + // Initialize with proper parameters. + virtual void Init(uint numVersionsStored) = 0; + // Process all the updates that will be persisted. + virtual void ProcessUpdates(const categorization::Updates& updates) = 0; + // Get a serialized object serving as proof for correctness. + virtual std::vector GetProof(const std::string& key) = 0; + // In case of async processing we use this call to sync and wait for all async tasks to complete. + virtual void WaitForScheduledTasks() = 0; +}; + +} // namespace kvbc +} // namespace concord diff --git a/kvbc/include/v4blockchain/detail/verifiability_management_layer.h b/kvbc/include/v4blockchain/detail/verifiability_management_layer.h new file mode 100644 index 0000000000..2887056689 --- /dev/null +++ b/kvbc/include/v4blockchain/detail/verifiability_management_layer.h @@ -0,0 +1,39 @@ +// Concord +// +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// +// This product is licensed to you under the Apache 2.0 license (the +// "License"). You may not use this product except in compliance with the +// Apache 2.0 License. +// +// This product may include a number of subcomponents with separate copyright +// notices and license terms. Your use of these subcomponents is subject to the +// terms and conditions of the subcomponent's license, as noted in the LICENSE +// file. + +#pragma once + +#include "rocksdb/native_client.h" +#include +#include +#include "categorization/updates.h" +#include "v4blockchain/detail/categories.h" +#include +#include "rocksdb/snapshot.h" +#include "proof_processor/IProofProcessor.h" + +namespace concord::kvbc::v4blockchain::detail { + +using namespace concord::kvbc; + +class VerifiabilityManagementLayer { + public: + VerifiabilityManagementLayer(); + + IProofProcessor* getProcessor() { return proofProcessor_.get(); } + + private: + std::unique_ptr proofProcessor_; +}; + +} // namespace concord::kvbc::v4blockchain::detail diff --git a/kvbc/include/v4blockchain/v4_blockchain.h b/kvbc/include/v4blockchain/v4_blockchain.h index acc0008330..22fa23db43 100644 --- a/kvbc/include/v4blockchain/v4_blockchain.h +++ b/kvbc/include/v4blockchain/v4_blockchain.h @@ -18,6 +18,7 @@ #include "v4blockchain/detail/st_chain.h" #include "v4blockchain/detail/latest_keys.h" #include "v4blockchain/detail/blockchain.h" +#include "v4blockchain/detail/verifiability_management_layer.h" #include "v4blockchain/detail/keys_history.h" #include #include @@ -86,6 +87,9 @@ class KeyValueBlockchain { const v4blockchain::detail::Blockchain &getBlockchain() const { return block_chain_; }; const v4blockchain::detail::StChain &getStChain() const { return state_transfer_chain_; }; const v4blockchain::detail::LatestKeys &getLatestKeys() const { return latest_keys_; }; + v4blockchain::detail::VerifiabilityManagementLayer &getVerifiabilityManagementLayer() { + return verifiability_management_layer_; + } ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // IReader std::optional get(const std::string &category_id, @@ -274,6 +278,8 @@ class KeyValueBlockchain { v4blockchain::detail::StChain state_transfer_chain_; v4blockchain::detail::LatestKeys latest_keys_; v4blockchain::detail::KeysHistory keys_history_; + v4blockchain::detail::VerifiabilityManagementLayer verifiability_management_layer_; + // flag to mark whether a checkpoint is being taken. std::optional last_block_sn_; const float updates_to_final_size_ration_{2.5}; diff --git a/kvbc/src/merkle_processor/BaseMerkleProcessor.cpp b/kvbc/src/merkle_processor/BaseMerkleProcessor.cpp new file mode 100644 index 0000000000..a1e83ebabf --- /dev/null +++ b/kvbc/src/merkle_processor/BaseMerkleProcessor.cpp @@ -0,0 +1,68 @@ +// Concord +// +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// +// This product is licensed to you under the Apache 2.0 license (the "License"). +// You may not use this product except in compliance with the Apache 2.0 License. +// +// This product may include a number of subcomponents with separate copyright +// notices and license terms. Your use of these subcomponents is subject to the +// terms and conditions of the sub-component's license, as noted in the +// LICENSE file. + +#include "merkle_processor/BaseMerkleProcessor.h" + +namespace concord { +namespace kvbc { +namespace sparse_merkle { + +bool BaseMerkleProcessor::needProcessing(char type) const { + if (type == kKvbKeyEthBalance || type == kKvbKeyEthCode || type == kKvbKeyEthStorage || type == kKvbKeyEthNonce) { + return true; + } + return false; +} + +IMerkleProcessor::address BaseMerkleProcessor::getAddress(const std::string& key) const { + ConcordAssert(key.size() > IMerkleProcessor::address_size); // Include 1 byte indicating key type + return address(&key[1], IMerkleProcessor::address_size); +} + +void BaseMerkleProcessor::ProcessUpdates(const categorization::Updates& updates) { + BeginVersionUpdateBatch(); + for (const auto& k : updates.categoryUpdates().kv) { + if (const categorization::BlockMerkleInput* pval = std::get_if(&k.second)) { + for (const auto& v : pval->kv) { + if (needProcessing(v.first[0])) { + auto hasher = Hasher{}; + const auto value_hash = hasher.digest(v.second.data(), v.second.size()); + LOG_INFO(V4_BLOCK_LOG, "MerkleProcessor: categorization::BlockMerkleInput"); + UpdateAccountTree(getAddress(v.first), v.first, value_hash); + } + } + } else if (const categorization::VersionedInput* pval = std::get_if(&k.second)) { + for (const auto& v : pval->kv) { + if (needProcessing(v.first[0])) { + auto hasher = Hasher{}; + const auto value_hash = hasher.digest(v.second.data.data(), v.second.data.size()); + LOG_INFO(V4_BLOCK_LOG, "MerkleProcessor: categorization::VersionedInput"); + UpdateAccountTree(getAddress(v.first), v.first, value_hash); + } + } + } else if (const categorization::ImmutableInput* pval = std::get_if(&k.second)) { + for (const auto& v : pval->kv) { + if (needProcessing(v.first[0])) { + auto hasher = Hasher{}; + const auto value_hash = hasher.digest(v.second.data.data(), v.second.data.size()); + LOG_INFO(V4_BLOCK_LOG, "MerkleProcessor: categorization::ImmutableInput"); + UpdateAccountTree(getAddress(v.first), v.first, value_hash); + } + } + } + } + CommitVersionUpdateBatch(); +} + +} // namespace sparse_merkle +} // namespace kvbc +} // namespace concord diff --git a/kvbc/src/merkle_processor/SyncMerkleProcessor.cpp b/kvbc/src/merkle_processor/SyncMerkleProcessor.cpp new file mode 100644 index 0000000000..a9dc5f88dc --- /dev/null +++ b/kvbc/src/merkle_processor/SyncMerkleProcessor.cpp @@ -0,0 +1,54 @@ +// Concord +// +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// +// This product is licensed to you under the Apache 2.0 license (the "License"). +// You may not use this product except in compliance with the Apache 2.0 License. +// +// This product may include a number of subcomponents with separate copyright +// notices and license terms. Your use of these subcomponents is subject to the +// terms and conditions of the sub-component's license, as noted in the +// LICENSE file. + +#include "merkle_processor/SyncMerkleProcessor.h" +#include "kvbc_app_filter/kvbc_key_types.h" +#include "util/hex_tools.hpp" + +namespace concord { +namespace kvbc { +namespace sparse_merkle { + +using concordUtils::bufferToHex; + +void SyncMerkleProcessor::Init(uint numVersionsStored) {} + +void SyncMerkleProcessor::BeginVersionUpdateBatch() {} + +void SyncMerkleProcessor::UpdateAccountTree(const address& addr, const std::string& key, const value& data) { + LOG_INFO(V4_BLOCK_LOG, + "MerkleProcessor: UpdateAccountTree(addr=" << bufferToHex(addr.data(), sizeof(addr)) + << ", key=" << bufferToHex(key.c_str(), key.size())); +} + +void SyncMerkleProcessor::CommitVersionUpdateBatch() { + LOG_INFO(V4_BLOCK_LOG, "MerkleProcessor: CommitVersionUpdateBatch"); +} + +std::vector SyncMerkleProcessor::GetAccountMerkleRootPath(const address& addr) { return {}; } + +std::vector SyncMerkleProcessor::GetAccountStorageKeyMerklePath(const address& addr, + const std::string& key) { + return {}; +} + +std::vector SyncMerkleProcessor::GetProof(const std::string& key) { return {}; } + +bool SyncMerkleProcessor::VerifyMerkleTreePath(std::string root_hash, std::string key, std::vector path) { + return true; +} + +void SyncMerkleProcessor::WaitForScheduledTasks() {} + +} // namespace sparse_merkle +} // namespace kvbc +} // namespace concord diff --git a/kvbc/src/v4blockchain/detail/verifiability_management_layer.cpp b/kvbc/src/v4blockchain/detail/verifiability_management_layer.cpp new file mode 100644 index 0000000000..97b61ec3b7 --- /dev/null +++ b/kvbc/src/v4blockchain/detail/verifiability_management_layer.cpp @@ -0,0 +1,34 @@ +// Concord +// +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// +// This product is licensed to you under the Apache 2.0 license (the +// "License"). You may not use this product except in compliance with the +// Apache 2.0 License. +// +// This product may include a number of subcomponents with separate copyright +// notices and license terms. Your use of these subcomponents is subject to the +// terms and conditions of the subcomponent's license, as noted in the LICENSE +// file. + +#include "v4blockchain/detail/verifiability_management_layer.h" +#include "v4blockchain/detail/column_families.h" +#include "log/logger.hpp" +#include "v4blockchain/detail/blockchain.h" +#include "rocksdb/details.h" +#include "proof_processor/EmptyProofProcessor.h" +#include "merkle_processor/SyncMerkleProcessor.h" + +using namespace concord::kvbc; +namespace concord::kvbc::v4blockchain::detail { + +static const int NUM_MAINTAINED_TREE_VERSIONS = 10; + +VerifiabilityManagementLayer::VerifiabilityManagementLayer() { + // TODO: instantiate based on configuration. + proofProcessor_ = std::make_unique(); + // TODO: get configuration parameters from config manager + proofProcessor_->Init(NUM_MAINTAINED_TREE_VERSIONS); +} + +} // namespace concord::kvbc::v4blockchain::detail diff --git a/kvbc/src/v4blockchain/v4_blockchain.cpp b/kvbc/src/v4blockchain/v4_blockchain.cpp index 1515c1516d..b2b90de77e 100644 --- a/kvbc/src/v4blockchain/v4_blockchain.cpp +++ b/kvbc/src/v4blockchain/v4_blockchain.cpp @@ -107,6 +107,7 @@ BlockId KeyValueBlockchain::add(categorization::Updates &&updates) { BlockId KeyValueBlockchain::add(const categorization::Updates &updates, v4blockchain::detail::Block &block, storage::rocksdb::NativeWriteBatch &write_batch) { + getVerifiabilityManagementLayer().getProcessor()->ProcessUpdates(updates); BlockId block_id{}; { block_id = block_chain_.addBlock(block, write_batch); } { latest_keys_.addBlockKeys(updates, block_id, write_batch); }