-
Notifications
You must be signed in to change notification settings - Fork 148
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Use raw key in the block merkle latest version CF
Use the actual raw key instead of the key's hash in the block merkle's BLOCK_MERKLE_LATEST_KEY_VERSION_CF column family. We need that so we can serve a state snapshot in lexicographic order on keys. Hashes prevented us, because: * we lose the key itself when we hash and then prune the block the key was added in (as we only persist the key itself in blocks) * we won't be able to stream in lexicographic order if we store the key hashes only The BLOCK_MERKLE_KEYS_CF column family and other parts of block merkle don't change in any way. The migration keeps track of what steps have been executed such that if the migration process crashes, it can start fresh or continue from where it left off. Persisting migration state is implemented by introducing the `Migration` EDBKeyType. **Important note**: when this change is merged, the DB will no longer be backwards compatible. Therefore, a tool that migrates an existing DB so that it is compatible with the change in this PR is provided too. In terms of the migration tool, we provide: * ability to recover if migration crashed in the middle * unit tests * automatically detect if migration is needed
- Loading branch information
1 parent
3d4ebf6
commit d7a9d38
Showing
15 changed files
with
879 additions
and
45 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
77 changes: 77 additions & 0 deletions
77
kvbc/include/migrations/block_merkle_latest_ver_cf_migration.h
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,77 @@ | ||
| // Concord | ||
| // | ||
| // Copyright (c) 2021 VMware, Inc. All Rights Reserved. | ||
| // | ||
| // This product is licensed to you under the Apache 2.0 license (the "License"). | ||
| // You may not use this product except in compliance with the Apache 2.0 License. | ||
| // | ||
| // This product may include a number of subcomponents with separate copyright | ||
| // notices and license terms. Your use of these subcomponents is subject to the | ||
| // terms and conditions of the sub-component's license, as noted in the | ||
| // LICENSE file. | ||
|
|
||
| #pragma once | ||
|
|
||
| #include "rocksdb/native_client.h" | ||
|
|
||
| #include <rocksdb/utilities/checkpoint.h> | ||
|
|
||
| #include <exception> | ||
| #include <memory> | ||
| #include <optional> | ||
| #include <string> | ||
| #include <string_view> | ||
|
|
||
| namespace concord::kvbc::migrations { | ||
|
|
||
| // Migrates a RocksDB DB from using key hashes in the `block_merkle_latest_key_version` to using raw keys. | ||
| // Keeps track of what steps have been executed such that if the migration process crashes, it can start fresh or | ||
| // continue from where it left off. | ||
| class BlockMerkleLatestVerCfMigration { | ||
| public: | ||
| // Note: `export_path` must be on the same filesystem as `db_path`. | ||
| BlockMerkleLatestVerCfMigration(const std::string& db_path, const std::string& export_path); | ||
|
|
||
| public: | ||
| static const std::string& temporaryColumnFamily(); | ||
| static const std::string& migrationKey(); | ||
|
|
||
| // Migration states. | ||
| static inline const std::string kStateImportedTempCf{"imported-temp-cf"}; | ||
| static inline const std::string kStateMigrated{"migrated"}; | ||
| static inline const std::string kStateMigrationNotNeededOrCompleted{"migration-not-needed-or-completed"}; | ||
|
|
||
| enum class ExecutionStatus { | ||
| kExecuted, // executed as part of this call | ||
| kNotNeededOrAlreadyExecuted, // migration is not needed or already executed and, therefore, nothing done in this | ||
| // call | ||
| }; | ||
|
|
||
| public: | ||
| // Executes the migration, throwing on error. | ||
| // If execute() returns, it is always a success. The ExecutionStatus gives indication as to what the actual outcome | ||
| // is. | ||
| ExecutionStatus execute(); | ||
|
|
||
| std::shared_ptr<storage::rocksdb::NativeClient> db() { return db_; } | ||
|
|
||
| // Following methods are used for testing purposes only. Do not use in production. | ||
| public: | ||
| void removeExportDir(); | ||
| void checkpointDB(); | ||
| void exportLatestVerCf(); | ||
| void importTempLatestVerCf(); | ||
| void clearExistingLatestVerCf(); | ||
| void iterateAndMigrate(); | ||
| void dropTempLatestVerCf(); | ||
| void commitComplete(); | ||
|
|
||
| private: | ||
| const std::string db_path_; | ||
| const std::string export_path_; | ||
| std::shared_ptr<storage::rocksdb::NativeClient> db_; | ||
| std::unique_ptr<::rocksdb::Checkpoint> checkpoint_; | ||
| std::unique_ptr<::rocksdb::ExportImportFilesMetaData> export_metadata_; | ||
| }; | ||
|
|
||
| } // namespace concord::kvbc::migrations |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.