Feat: index Merkle roots for BSPs and Buckets (#260)

* indexer: add and track merkle roots * pnpm typegen
Moonsong-Labs · Nov 19, 2024 · 0c5096a · 0c5096a
1 parent 2f1f6d8
commit 0c5096a
Show file tree

Hide file tree

Showing 20 changed files with 118 additions and 15 deletions.
diff --git a/api-augment/dist/interfaces/lookup.js b/api-augment/dist/interfaces/lookup.js
diff --git a/api-augment/dist/interfaces/lookup.js.map b/api-augment/dist/interfaces/lookup.js.map
diff --git a/api-augment/dist/types/interfaces/augment-api-events.d.ts b/api-augment/dist/types/interfaces/augment-api-events.d.ts
@@ -694,6 +694,7 @@ declare module "@polkadot/api-base/types/events" {
           mspId: Option<H256>,
           bucketId: H256,
           name: Bytes,
+          root: H256,
           collectionId: Option<u32>,
           private: bool,
           valuePropId: Option<H256>
@@ -703,6 +704,7 @@ declare module "@polkadot/api-base/types/events" {
           mspId: Option<H256>;
           bucketId: H256;
           name: Bytes;
+          root: H256;
           collectionId: Option<u32>;
           private: bool;
           valuePropId: Option<H256>;
@@ -2172,14 +2174,16 @@ declare module "@polkadot/api-base/types/events" {
       >;
       /**
        * Event emitted when a Backup Storage Provider has confirmed its sign up successfully. Provides information about
-       * that BSP's account id, the total data it can store according to its stake, and its multiaddress.
+       * that BSP's account id, the initial root of the Merkle Patricia Trie that it stores, the total data it can store
+       * according to its stake, and its multiaddress.
        **/
       BspSignUpSuccess: AugmentedEvent<
         ApiType,
-        [who: AccountId32, bspId: H256, multiaddresses: Vec<Bytes>, capacity: u64],
+        [who: AccountId32, bspId: H256, root: H256, multiaddresses: Vec<Bytes>, capacity: u64],
         {
           who: AccountId32;
           bspId: H256;
+          root: H256;
           multiaddresses: Vec<Bytes>;
           capacity: u64;
         }

diff --git a/api-augment/dist/types/interfaces/lookup.d.ts b/api-augment/dist/types/interfaces/lookup.d.ts
@@ -1345,6 +1345,7 @@ declare const _default: {
       BspSignUpSuccess: {
         who: string;
         bspId: string;
+        root: string;
         multiaddresses: string;
         capacity: string;
       };
@@ -1429,6 +1430,7 @@ declare const _default: {
         mspId: string;
         bucketId: string;
         name: string;
+        root: string;
         collectionId: string;
         private: string;
         valuePropId: string;

diff --git a/api-augment/dist/types/interfaces/types-lookup.d.ts b/api-augment/dist/types/interfaces/types-lookup.d.ts
@@ -1744,6 +1744,7 @@ declare module "@polkadot/types/lookup" {
     readonly asBspSignUpSuccess: {
       readonly who: AccountId32;
       readonly bspId: H256;
+      readonly root: H256;
       readonly multiaddresses: Vec<Bytes>;
       readonly capacity: u64;
     } & Struct;
@@ -1845,6 +1846,7 @@ declare module "@polkadot/types/lookup" {
       readonly mspId: Option<H256>;
       readonly bucketId: H256;
       readonly name: Bytes;
+      readonly root: H256;
       readonly collectionId: Option<u32>;
       readonly private: bool;
       readonly valuePropId: Option<H256>;

diff --git a/api-augment/src/interfaces/augment-api-events.ts b/api-augment/src/interfaces/augment-api-events.ts
@@ -543,6 +543,7 @@ declare module "@polkadot/api-base/types/events" {
           mspId: Option<H256>,
           bucketId: H256,
           name: Bytes,
+          root: H256,
           collectionId: Option<u32>,
           private: bool,
           valuePropId: Option<H256>
@@ -552,6 +553,7 @@ declare module "@polkadot/api-base/types/events" {
           mspId: Option<H256>;
           bucketId: H256;
           name: Bytes;
+          root: H256;
           collectionId: Option<u32>;
           private: bool;
           valuePropId: Option<H256>;
@@ -1678,12 +1680,13 @@ declare module "@polkadot/api-base/types/events" {
       >;
       /**
        * Event emitted when a Backup Storage Provider has confirmed its sign up successfully. Provides information about
-       * that BSP's account id, the total data it can store according to its stake, and its multiaddress.
+       * that BSP's account id, the initial root of the Merkle Patricia Trie that it stores, the total data it can store
+       * according to its stake, and its multiaddress.
        **/
       BspSignUpSuccess: AugmentedEvent<
         ApiType,
-        [who: AccountId32, bspId: H256, multiaddresses: Vec<Bytes>, capacity: u64],
-        { who: AccountId32; bspId: H256; multiaddresses: Vec<Bytes>; capacity: u64 }
+        [who: AccountId32, bspId: H256, root: H256, multiaddresses: Vec<Bytes>, capacity: u64],
+        { who: AccountId32; bspId: H256; root: H256; multiaddresses: Vec<Bytes>; capacity: u64 }
       >;
       /**
        * Event emitted when a bucket's root has been changed.

diff --git a/api-augment/src/interfaces/lookup.ts b/api-augment/src/interfaces/lookup.ts
@@ -1361,6 +1361,7 @@ export default {
       BspSignUpSuccess: {
         who: "AccountId32",
         bspId: "H256",
+        root: "H256",
         multiaddresses: "Vec<Bytes>",
         capacity: "u64"
       },
@@ -1445,6 +1446,7 @@ export default {
         mspId: "Option<H256>",
         bucketId: "H256",
         name: "Bytes",
+        root: "H256",
         collectionId: "Option<u32>",
         private: "bool",
         valuePropId: "Option<H256>"

diff --git a/api-augment/src/interfaces/types-lookup.ts b/api-augment/src/interfaces/types-lookup.ts
@@ -1827,6 +1827,7 @@ declare module "@polkadot/types/lookup" {
     readonly asBspSignUpSuccess: {
       readonly who: AccountId32;
       readonly bspId: H256;
+      readonly root: H256;
       readonly multiaddresses: Vec<Bytes>;
       readonly capacity: u64;
     } & Struct;
@@ -1932,6 +1933,7 @@ declare module "@polkadot/types/lookup" {
       readonly mspId: Option<H256>;
       readonly bucketId: H256;
       readonly name: Bytes;
+      readonly root: H256;
       readonly collectionId: Option<u32>;
       readonly private: bool;
       readonly valuePropId: Option<H256>;

diff --git a/api-augment/storagehub.json b/api-augment/storagehub.json
diff --git a/client/indexer-db/migrations/2024-11-15-160045_track_merkle_roots/down.sql b/client/indexer-db/migrations/2024-11-15-160045_track_merkle_roots/down.sql
@@ -0,0 +1,3 @@
+-- Remove merkle_root columns from bucket and bsp tables
+ALTER TABLE bucket DROP COLUMN merkle_root;
+ALTER TABLE bsp DROP COLUMN merkle_root;
diff --git a/client/indexer-db/migrations/2024-11-15-160045_track_merkle_roots/up.sql b/client/indexer-db/migrations/2024-11-15-160045_track_merkle_roots/up.sql
@@ -0,0 +1,3 @@
+-- Add merkle_root columns to bucket and bsp tables
+ALTER TABLE bucket ADD COLUMN merkle_root BYTEA NOT NULL;
+ALTER TABLE bsp ADD COLUMN merkle_root BYTEA NOT NULL;
diff --git a/client/indexer-db/src/models/bsp.rs b/client/indexer-db/src/models/bsp.rs
@@ -23,6 +23,7 @@ pub struct Bsp {
     pub created_at: NaiveDateTime,
     pub updated_at: NaiveDateTime,
     pub onchain_bsp_id: String,
+    pub merkle_root: Vec<u8>,
 }
 
 /// Association table between BSP and MultiAddress
@@ -40,6 +41,7 @@ impl Bsp {
         conn: &mut DbConnection<'a>,
         account: String,
         capacity: BigDecimal,
+        merkle_root: Vec<u8>,
         multiaddresses: Vec<MultiAddress>,
         onchain_bsp_id: String,
         stake: BigDecimal,
@@ -49,6 +51,7 @@ impl Bsp {
                 bsp::account.eq(account),
                 bsp::capacity.eq(capacity),
                 bsp::onchain_bsp_id.eq(onchain_bsp_id),
+                bsp::merkle_root.eq(merkle_root),
                 bsp::stake.eq(stake),
             ))
             .returning(Bsp::as_select())
@@ -144,6 +147,19 @@ impl Bsp {
             .await?;
         Ok(())
     }
+
+    pub async fn update_merkle_root<'a>(
+        conn: &mut DbConnection<'a>,
+        onchain_bsp_id: String,
+        merkle_root: Vec<u8>,
+    ) -> Result<(), diesel::result::Error> {
+        diesel::update(bsp::table)
+            .filter(bsp::onchain_bsp_id.eq(onchain_bsp_id))
+            .set(bsp::merkle_root.eq(merkle_root))
+            .execute(conn)
+            .await?;
+        Ok(())
+    }
 }
 
 #[derive(Debug, Queryable, Insertable, Selectable)]

diff --git a/client/indexer-db/src/models/bucket.rs b/client/indexer-db/src/models/bucket.rs
@@ -20,6 +20,7 @@ pub struct Bucket {
     pub private: bool,
     pub created_at: NaiveDateTime,
     pub updated_at: NaiveDateTime,
+    pub merkle_root: Vec<u8>,
 }
 
 impl Bucket {
@@ -31,6 +32,7 @@ impl Bucket {
         name: Vec<u8>,
         collection_id: Option<String>,
         private: bool,
+        merkle_root: Vec<u8>,
     ) -> Result<Self, diesel::result::Error> {
         let bucket = diesel::insert_into(bucket::table)
             .values((
@@ -40,6 +42,7 @@ impl Bucket {
                 bucket::name.eq(name),
                 bucket::collection_id.eq(collection_id),
                 bucket::private.eq(private),
+                bucket::merkle_root.eq(merkle_root),
             ))
             .returning(Bucket::as_select())
             .get_result(conn)
@@ -81,6 +84,30 @@ impl Bucket {
         Ok(bucket)
     }
 
+    pub async fn update_merkle_root<'a>(
+        conn: &mut DbConnection<'a>,
+        onchain_bucket_id: String,
+        merkle_root: Vec<u8>,
+    ) -> Result<(), diesel::result::Error> {
+        diesel::update(bucket::table)
+            .filter(bucket::onchain_bucket_id.eq(onchain_bucket_id))
+            .set(bucket::merkle_root.eq(merkle_root))
+            .execute(conn)
+            .await?;
+        Ok(())
+    }
+
+    pub async fn delete<'a>(
+        conn: &mut DbConnection<'a>,
+        onchain_bucket_id: String,
+    ) -> Result<(), diesel::result::Error> {
+        diesel::delete(bucket::table)
+            .filter(bucket::onchain_bucket_id.eq(onchain_bucket_id))
+            .execute(conn)
+            .await?;
+        Ok(())
+    }
+
     pub async fn get_by_onchain_bucket_id<'a>(
         conn: &mut DbConnection<'a>,
         onchain_bucket_id: String,

diff --git a/client/indexer-db/src/schema.rs b/client/indexer-db/src/schema.rs
@@ -10,6 +10,7 @@ diesel::table! {
         created_at -> Timestamp,
         updated_at -> Timestamp,
         onchain_bsp_id -> Varchar,
+        merkle_root -> Bytea,
     }
 }
 
@@ -38,6 +39,7 @@ diesel::table! {
         private -> Bool,
         created_at -> Timestamp,
         updated_at -> Timestamp,
+        merkle_root -> Bytea,
     }
 }
 

diff --git a/client/indexer-service/src/handler.rs b/client/indexer-service/src/handler.rs
@@ -186,6 +186,7 @@ impl IndexerService {
                 collection_id,
                 private,
                 value_prop_id: _,
+                root,
             } => {
                 let msp = match msp_id {
                     Some(msp_id) => {
@@ -201,6 +202,7 @@ impl IndexerService {
                     name.to_vec(),
                     collection_id.map(|id| id.to_string()),
                     *private,
+                    root.as_ref().to_vec(),
                 )
                 .await?;
             }
@@ -225,18 +227,22 @@ impl IndexerService {
             }
             pallet_file_system::Event::BspConfirmStoppedStoring {
                 bsp_id,
-                file_key,
-                new_root: _,
+                file_key: _,
+                new_root,
             } => {
-                BspFile::delete(conn, file_key, bsp_id.to_string()).await?;
+                Bsp::update_merkle_root(conn, bsp_id.to_string(), new_root.as_ref().to_vec())
+                    .await?;
             }
             pallet_file_system::Event::BspConfirmedStoring {
                 who: _,
+                bsp_id,
                 confirmed_file_keys,
                 skipped_file_keys: _,
-                new_root: _,
-                bsp_id,
+                new_root,
             } => {
+                Bsp::update_merkle_root(conn, bsp_id.to_string(), new_root.as_ref().to_vec())
+                    .await?;
+
                 let bsp = Bsp::get_by_onchain_bsp_id(conn, bsp_id.to_string()).await?;
                 for file_key in confirmed_file_keys {
                     let file = File::get_by_file_key(conn, file_key.as_ref().to_vec()).await?;
@@ -306,7 +312,13 @@ impl IndexerService {
             pallet_file_system::Event::MoveBucketRequestExpired { .. } => {}
             pallet_file_system::Event::MoveBucketRejected { .. } => {}
             pallet_file_system::Event::MspStoppedStoringBucket { .. } => {}
-            pallet_file_system::Event::BucketDeleted { .. } => {}
+            pallet_file_system::Event::BucketDeleted {
+                who: _,
+                bucket_id,
+                maybe_collection_id: _,
+            } => {
+                Bucket::delete(conn, bucket_id.to_string()).await?;
+            }
             pallet_file_system::Event::__Ignore(_, _) => {}
         }
         Ok(())
@@ -419,6 +431,7 @@ impl IndexerService {
             pallet_storage_providers::Event::BspSignUpSuccess {
                 who,
                 bsp_id,
+                root,
                 multiaddresses,
                 capacity,
             } => {
@@ -441,6 +454,7 @@ impl IndexerService {
                     conn,
                     who.to_string(),
                     capacity.into(),
+                    root.as_ref().to_vec(),
                     sql_multiaddresses,
                     bsp_id.to_string(),
                     stake,
@@ -513,7 +527,14 @@ impl IndexerService {
             } => {
                 Msp::delete(conn, who.to_string()).await?;
             }
-            pallet_storage_providers::Event::BucketRootChanged { .. } => {}
+            pallet_storage_providers::Event::BucketRootChanged {
+                bucket_id,
+                old_root: _,
+                new_root,
+            } => {
+                Bucket::update_merkle_root(conn, bucket_id.to_string(), new_root.as_ref().to_vec())
+                    .await?;
+            }
             pallet_storage_providers::Event::Slashed {
                 provider_id,
                 amount_slashed: _amount_slashed,

diff --git a/pallets/file-system/src/lib.rs b/pallets/file-system/src/lib.rs
@@ -478,6 +478,7 @@ pub mod pallet {
             msp_id: Option<ProviderIdFor<T>>,
             bucket_id: BucketIdFor<T>,
             name: BucketNameFor<T>,
+            root: MerkleHash<T>,
             collection_id: Option<CollectionIdFor<T>>,
             private: bool,
             value_prop_id: Option<ValuePropId<T>>,
@@ -819,6 +820,7 @@ pub mod pallet {
                 msp_id,
                 bucket_id,
                 name,
+                root: <T::ProofDealer as shp_traits::ProofsDealerInterface>::MerkleHash::default(),
                 collection_id: maybe_collection_id,
                 private,
                 value_prop_id,