feat(rocksdb): remove getters for internal rocksdb handles, expose `b…

…ackup` instead (#2535)

- **Revert "chore(rocksdb): getter for inner database handle (#2532)"**
- **fix(rocksdb): expose backup and restore functionality instead of
inner dbs**

## Linked Issues/PRs
<!-- List of related issues/PRs -->
- #2532 (comment)

## Description
<!-- List of detailed changes -->
- creates a feature `backup` and 2 functions, `backup` and `restore`. 
- not too many config options available at the moment
- bubbled it to `CombinedDatabase`, similarly to the `prune` api from
rocksdb.


## Checklist
- [x] Breaking changes are clearly marked as such in the PR description
and changelog
- [x] New behavior is reflected in tests
- [x] [The specification](https://github.com/FuelLabs/fuel-specs/)
matches the implemented behavior (link update PR if changes are needed)

### Before requesting review
- [ ] I have reviewed the code myself
- [ ] I have created follow-up issues caused by this PR and linked them
here

### After merging, notify other teams

[Add or remove entries as needed]

- [ ] [Rust SDK](https://github.com/FuelLabs/fuels-rs/)
- [ ] [Sway compiler](https://github.com/FuelLabs/sway/)
- [ ] [Platform
documentation](https://github.com/FuelLabs/devrel-requests/issues/new?assignees=&labels=new+request&projects=&template=NEW-REQUEST.yml&title=%5BRequest%5D%3A+)
(for out-of-organization contributors, the person merging the PR will do
this)
- [ ] Someone else?

---------

Co-authored-by: Green Baneling <[email protected]>

CHANGELOG.md

@@ -36,6 +36,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 - [2526](https://github.com/FuelLabs/fuel-core/pull/2526): Add possibility to not have any cache set for RocksDB. Add an option to either load the RocksDB columns families on creation of the database or when the column is used.
 - [2532](https://github.com/FuelLabs/fuel-core/pull/2532): Getters for inner rocksdb database handles.
 - [2524](https://github.com/FuelLabs/fuel-core/pull/2524): Adds a new lock type which is optimized for certain workloads to the txpool and p2p services.
+- [2535](https://github.com/FuelLabs/fuel-core/pull/2535): Expose `backup` and `restore` APIs on the `CombinedDatabase` struct to create portable backups and restore from them.
 
 ### Fixed
 - [2365](https://github.com/FuelLabs/fuel-core/pull/2365): Fixed the error during dry run in the case of race condition.

crates/database/Cargo.toml

@@ -27,3 +27,4 @@ fuel-core-trace = { path = "../trace" }
 
 [features]
 test-helpers = []
+backup = []

crates/database/src/lib.rs

@@ -72,6 +72,18 @@ pub enum Error {
     #[display(fmt = "Reached the end of the history")]
     ReachedEndOfHistory,
 
+    #[cfg(feature = "backup")]
+    #[display(fmt = "BackupEngine initialization error: {}", _0)]
+    BackupEngineInitError(anyhow::Error),
+
+    #[cfg(feature = "backup")]
+    #[display(fmt = "Backup error: {}", _0)]
+    BackupError(anyhow::Error),
+
+    #[cfg(feature = "backup")]
+    #[display(fmt = "Restore error: {}", _0)]
+    RestoreError(anyhow::Error),
+
     /// Not related to database error.
     #[from]
     Other(anyhow::Error),

crates/fuel-core/Cargo.toml

@@ -101,6 +101,7 @@ p2p = ["dep:fuel-core-p2p", "dep:fuel-core-sync"]
 relayer = ["dep:fuel-core-relayer"]
 shared-sequencer = ["dep:fuel-core-shared-sequencer", "dep:cosmrs"]
 rocksdb = ["dep:rocksdb", "dep:tempfile", "dep:num_cpus", "dep:postcard"]
+backup = ["rocksdb", "fuel-core-database/backup"]
 test-helpers = [
   "fuel-core-database/test-helpers",
   "fuel-core-p2p?/test-helpers",

crates/fuel-core/src/combined_database.rs

@@ -23,6 +23,8 @@ use fuel_core_chain_config::{
     StateConfig,
     StateConfigBuilder,
 };
+#[cfg(feature = "backup")]
+use fuel_core_services::TraceErr;
 #[cfg(feature = "test-helpers")]
 use fuel_core_storage::tables::{
     Coins,
@@ -78,6 +80,94 @@ impl CombinedDatabase {
         Ok(())
     }
 
+    #[cfg(feature = "backup")]
+    pub fn backup(
+        db_dir: &std::path::Path,
+        backup_dir: &std::path::Path,
+    ) -> crate::database::Result<()> {
+        use tempfile::TempDir;
+
+        let temp_backup_dir = TempDir::new()
+            .trace_err("Failed to create temporary backup directory")
+            .map_err(|e| anyhow::anyhow!(e))?;
+
+        Self::backup_databases(db_dir, temp_backup_dir.path())?;
+
+        std::fs::rename(temp_backup_dir.path(), backup_dir)
+            .trace_err("Failed to move temporary backup directory")
+            .map_err(|e| anyhow::anyhow!(e))?;
+
+        Ok(())
+    }
+
+    #[cfg(feature = "backup")]
+    fn backup_databases(
+        db_dir: &std::path::Path,
+        temp_dir: &std::path::Path,
+    ) -> crate::database::Result<()> {
+        crate::state::rocks_db::RocksDb::<OnChain>::backup(db_dir, temp_dir)
+            .trace_err("Failed to backup on-chain database")?;
+
+        crate::state::rocks_db::RocksDb::<OffChain>::backup(db_dir, temp_dir)
+            .trace_err("Failed to backup off-chain database")?;
+
+        crate::state::rocks_db::RocksDb::<Relayer>::backup(db_dir, temp_dir)
+            .trace_err("Failed to backup relayer database")?;
+
+        crate::state::rocks_db::RocksDb::<GasPriceDatabase>::backup(db_dir, temp_dir)
+            .trace_err("Failed to backup gas-price database")?;
+
+        Ok(())
+    }
+
+    #[cfg(feature = "backup")]
+    pub fn restore(
+        restore_to: &std::path::Path,
+        backup_dir: &std::path::Path,
+    ) -> crate::database::Result<()> {
+        use tempfile::TempDir;
+
+        let temp_restore_dir = TempDir::new()
+            .trace_err("Failed to create temporary restore directory")
+            .map_err(|e| anyhow::anyhow!(e))?;
+
+        Self::restore_database(backup_dir, temp_restore_dir.path())?;
+
+        std::fs::rename(temp_restore_dir.path(), restore_to)
+            .trace_err("Failed to move temporary restore directory")
+            .map_err(|e| anyhow::anyhow!(e))?;
+
+        // we don't return a CombinedDatabase here
+        // because the consumer can use any db config while opening it
+        Ok(())
+    }
+
+    #[cfg(feature = "backup")]
+    fn restore_database(
+        backup_dir: &std::path::Path,
+        temp_restore_dir: &std::path::Path,
+    ) -> crate::database::Result<()> {
+        crate::state::rocks_db::RocksDb::<OnChain>::restore(temp_restore_dir, backup_dir)
+            .trace_err("Failed to backup on-chain database")?;
+
+        crate::state::rocks_db::RocksDb::<OffChain>::restore(
+            temp_restore_dir,
+            backup_dir,
+        )
+        .trace_err("Failed to backup off-chain database")?;
+
+        crate::state::rocks_db::RocksDb::<Relayer>::restore(temp_restore_dir, backup_dir)
+            .trace_err("Failed to backup relayer database")?;
+
+        crate::state::rocks_db::RocksDb::<GasPriceDatabase>::restore(
+            temp_restore_dir,
+            backup_dir,
+        )
+        .trace_err("Failed to backup gas-price database")?;
+
+        Ok(())
+    }
+
     #[cfg(feature = "rocksdb")]
     pub fn open(
         path: &std::path::Path,
@@ -426,3 +516,189 @@ impl CombinedGenesisDatabase {
         &self.off_chain
     }
 }
+
+#[allow(non_snake_case)]
+#[cfg(feature = "backup")]
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use fuel_core_storage::StorageAsMut;
+    use fuel_core_types::{
+        entities::coins::coin::CompressedCoin,
+        fuel_tx::UtxoId,
+    };
+    use tempfile::TempDir;
+
+    #[test]
+    fn backup_and_restore__works_correctly__happy_path() {
+        // given
+        let db_dir = TempDir::new().unwrap();
+        let mut combined_db = CombinedDatabase::open(
+            db_dir.path(),
+            StateRewindPolicy::NoRewind,
+            DatabaseConfig::config_for_tests(),
+        )
+        .unwrap();
+        let key = UtxoId::new(Default::default(), Default::default());
+        let expected_value = CompressedCoin::default();
+
+        let on_chain_db = combined_db.on_chain_mut();
+        on_chain_db
+            .storage_as_mut::<Coins>()
+            .insert(&key, &expected_value)
+            .unwrap();
+        drop(combined_db);
+
+        // when
+        let backup_dir = TempDir::new().unwrap();
+        CombinedDatabase::backup(db_dir.path(), backup_dir.path()).unwrap();
+
+        // then
+        let restore_dir = TempDir::new().unwrap();
+        CombinedDatabase::restore(restore_dir.path(), backup_dir.path()).unwrap();
+        let restored_db = CombinedDatabase::open(
+            restore_dir.path(),
+            StateRewindPolicy::NoRewind,
+            DatabaseConfig::config_for_tests(),
+        )
+        .unwrap();
+
+        let mut restored_on_chain_db = restored_db.on_chain();
+        let restored_value = restored_on_chain_db
+            .storage::<Coins>()
+            .get(&key)
+            .unwrap()
+            .unwrap()
+            .into_owned();
+        assert_eq!(expected_value, restored_value);
+
+        // cleanup
+        std::fs::remove_dir_all(db_dir.path()).unwrap();
+        std::fs::remove_dir_all(backup_dir.path()).unwrap();
+        std::fs::remove_dir_all(restore_dir.path()).unwrap();
+    }
+
+    #[test]
+    fn backup__when_backup_fails_it_should_not_leave_any_residue() {
+        use std::os::unix::fs::PermissionsExt;
+
+        // given
+        let db_dir = TempDir::new().unwrap();
+        let mut combined_db = CombinedDatabase::open(
+            db_dir.path(),
+            StateRewindPolicy::NoRewind,
+            DatabaseConfig::config_for_tests(),
+        )
+        .unwrap();
+        let key = UtxoId::new(Default::default(), Default::default());
+        let expected_value = CompressedCoin::default();
+
+        let on_chain_db = combined_db.on_chain_mut();
+        on_chain_db
+            .storage_as_mut::<Coins>()
+            .insert(&key, &expected_value)
+            .unwrap();
+        drop(combined_db);
+
+        // when
+        // we set the permissions of db_dir to not allow reading
+        std::fs::set_permissions(db_dir.path(), std::fs::Permissions::from_mode(0o030))
+            .unwrap();
+        let backup_dir = TempDir::new().unwrap();
+
+        // then
+        CombinedDatabase::backup(db_dir.path(), backup_dir.path())
+            .expect_err("Backup should fail");
+        let backup_dir_contents = std::fs::read_dir(backup_dir.path()).unwrap();
+        assert_eq!(backup_dir_contents.count(), 0);
+
+        // cleanup
+        std::fs::set_permissions(db_dir.path(), std::fs::Permissions::from_mode(0o770))
+            .unwrap();
+        std::fs::remove_dir_all(db_dir.path()).unwrap();
+        std::fs::remove_dir_all(backup_dir.path()).unwrap();
+    }
+
+    #[test]
+    fn restore__when_restore_fails_it_should_not_leave_any_residue() {
+        use std::os::unix::fs::PermissionsExt;
+
+        // given
+        let db_dir = TempDir::new().unwrap();
+        let mut combined_db = CombinedDatabase::open(
+            db_dir.path(),
+            StateRewindPolicy::NoRewind,
+            DatabaseConfig::config_for_tests(),
+        )
+        .unwrap();
+        let key = UtxoId::new(Default::default(), Default::default());
+        let expected_value = CompressedCoin::default();
+
+        let on_chain_db = combined_db.on_chain_mut();
+        on_chain_db
+            .storage_as_mut::<Coins>()
+            .insert(&key, &expected_value)
+            .unwrap();
+        drop(combined_db);
+
+        let backup_dir = TempDir::new().unwrap();
+        CombinedDatabase::backup(db_dir.path(), backup_dir.path()).unwrap();
+
+        // when
+        // we set the permissions of backup_dir to not allow reading
+        std::fs::set_permissions(
+            backup_dir.path(),
+            std::fs::Permissions::from_mode(0o030),
+        )
+        .unwrap();
+        let restore_dir = TempDir::new().unwrap();
+
+        // then
+        CombinedDatabase::restore(restore_dir.path(), backup_dir.path())
+            .expect_err("Restore should fail");
+        let restore_dir_contents = std::fs::read_dir(restore_dir.path()).unwrap();
+        assert_eq!(restore_dir_contents.count(), 0);
+
+        // cleanup
+        std::fs::set_permissions(
+            backup_dir.path(),
+            std::fs::Permissions::from_mode(0o770),
+        )
+        .unwrap();
+        std::fs::remove_dir_all(db_dir.path()).unwrap();
+        std::fs::remove_dir_all(backup_dir.path()).unwrap();
+        std::fs::remove_dir_all(restore_dir.path()).unwrap();
+    }
+
+    #[test]
+    fn backup__cannot_backup_while_db_is_opened() {
+        // given
+        let db_dir = TempDir::new().unwrap();
+        let mut combined_db = CombinedDatabase::open(
+            db_dir.path(),
+            StateRewindPolicy::NoRewind,
+            DatabaseConfig::config_for_tests(),
+        )
+        .unwrap();
+        let key = UtxoId::new(Default::default(), Default::default());
+        let expected_value = CompressedCoin::default();
+
+        let on_chain_db = combined_db.on_chain_mut();
+        on_chain_db
+            .storage_as_mut::<Coins>()
+            .insert(&key, &expected_value)
+            .unwrap();
+
+        // when
+        let backup_dir = TempDir::new().unwrap();
+        // no drop for combined_db
+
+        // then
+        CombinedDatabase::backup(db_dir.path(), backup_dir.path())
+            .expect_err("Backup should fail");
+
+        // cleanup
+        std::fs::remove_dir_all(db_dir.path()).unwrap();
+        std::fs::remove_dir_all(backup_dir.path()).unwrap();
+    }
+}

crates/fuel-core/src/database.rs

@@ -207,18 +207,6 @@ where
         state_rewind_policy: StateRewindPolicy,
         database_config: DatabaseConfig,
     ) -> Result<Self> {
-        let db =
-            Self::open_as_historical_rocksdb(path, state_rewind_policy, database_config)?;
-
-        Ok(Self::new(Arc::new(db)))
-    }
-
-    #[cfg(feature = "rocksdb")]
-    pub fn open_as_historical_rocksdb(
-        path: &Path,
-        state_rewind_policy: StateRewindPolicy,
-        database_config: DatabaseConfig,
-    ) -> Result<HistoricalRocksDB<Description>> {
         use anyhow::Context;
 
         let db = HistoricalRocksDB::<Description>::default_open(
@@ -234,7 +222,7 @@ where
             )
         })?;
 
-        Ok(db)
+        Ok(Self::new(Arc::new(db)))
     }
 
     /// Converts the regular database to an unchecked database.

crates/fuel-core/src/state/historical_rocksdb.rs

@@ -106,10 +106,6 @@ where
         })
     }
 
-    pub fn inner(&self) -> &RocksDb<Historical<Description>> {
-        &self.db
-    }
-
     pub fn default_open<P: AsRef<Path>>(
         path: P,
         state_rewind_policy: StateRewindPolicy,