Fix sequence value at API

digital_asset_types/src/dao/scopes/asset.rs

@@ -369,6 +369,18 @@ fn convert_rocks_asset_model(
         .max()
         .unwrap(); // unwrap here is safe, because vec is not empty
 
+    // there are instructions where we update only assetLeaf seq value
+    // and there is burn instruction where we update only assetDynamic seq value
+    // that's why we need to take max value from both of them
+    let seq = {
+        let dynamic_seq = dynamic_data
+            .seq
+            .clone()
+            .and_then(|u| u.value.try_into().ok());
+        let leaf_seq = leaf.leaf_seq.map(|seq| seq as i64);
+        std::cmp::max(dynamic_seq, leaf_seq)
+    };
+
     Ok(asset::Model {
         id: static_data.pubkey.to_bytes().to_vec(),
         alt_id: None,
@@ -389,10 +401,7 @@ fn convert_rocks_asset_model(
         supply_mint: Some(static_data.pubkey.to_bytes().to_vec()),
         compressed: dynamic_data.is_compressed.value,
         compressible: dynamic_data.is_compressible.value,
-        seq: dynamic_data
-            .seq
-            .clone()
-            .and_then(|u| u.value.try_into().ok()),
+        seq,
         tree_id,
         leaf: leaf.leaf.clone(),
         nonce: leaf.nonce.map(|nonce| nonce as i64),

nft_ingester/src/index_syncronizer.rs

@@ -146,7 +146,7 @@ where
 #[cfg(test)]
 mod tests {
     use super::*;
-    use entities::models::AssetIndex;
+    use entities::models::{AssetIndex, UrlWithStatus};
     use metrics_utils::{
         ApiMetricsConfig, BackfillerMetricsConfig, IngesterMetricsConfig,
         JsonDownloaderMetricsConfig, JsonMigratorMetricsConfig, MetricState, MetricsTrait,
@@ -182,7 +182,10 @@ mod tests {
             is_compressed: false,
             is_frozen: false,
             supply: Some(1),
-            metadata_url: Some("https://www.google.com".to_string()),
+            metadata_url: Some(UrlWithStatus {
+                metadata_url: "https://www.google.com".to_string(),
+                is_downloaded: true,
+            }),
             slot_updated: 123456,
         }
     }

nft_ingester/tests/artifacts/expected_compression.json

@@ -0,0 +1,82 @@
+{
+    "HZPrFymDBjKcYnUGFtQR6uZEoqN9MQCiEvYa75u3xTeX": {
+        "eligible": false,
+        "compressed": true,
+        "data_hash": "849g69cMcDNekmDzfp8RtW7fXhbmsn2h1kr67D8YndJq",
+        "creator_hash": "syh1xpL9yb5sC3vJRDKHkS32ryAnYJjVCiSACRCjp2V",
+        "asset_hash": "4cYPV6NDuhDn9HGD72wqUnftK5yogYAm1FMkMmmzmSZq",
+        "tree": "AGMiLKtXX7PiVneM8S1KkTmCnF7X5zh6bKq4t1Mhrwpb",
+        "seq": 19,
+        "leaf_id": 0
+    },
+    "F8XkxardSug8FSViEGF5XMSXQ93wBYmqmiGuWiYt2uV8": {
+        "eligible": false,
+        "compressed": true,
+        "data_hash": "C9ybY1voubqwFY3RfgDroXvXd6reuM6ZhjC2jAVrQyAa",
+        "creator_hash": "7WJjsibXBHs5jBo5W1J2f8NYrGEc1eDCMgavasEinEcY",
+        "asset_hash": "FUjwuP2fPiDuUGipSmzJaS17T7CWZY2qFm1zZUBc2DTp",
+        "tree": "AGMiLKtXX7PiVneM8S1KkTmCnF7X5zh6bKq4t1Mhrwpb",
+        "seq": 9,
+        "leaf_id": 2
+    },
+    "8roA736sYuZccUMGpT67LqtX86yZ4fb99Ga2Up6adZ9y": {
+        "eligible": false,
+        "compressed": true,
+        "data_hash": "3HQtW9tfwKiXErZefSrJvK7kfEbm32F7AHPiAg2UVtiU",
+        "creator_hash": "7y8PaCbNffb1zkuo22Uu8nJM8deNtUbNGzqErAMj5rLn",
+        "asset_hash": "GAReCjyZeyT23bzy5Fn4ECkCbBZE5svQRZvfrAq1RexG",
+        "tree": "AGMiLKtXX7PiVneM8S1KkTmCnF7X5zh6bKq4t1Mhrwpb",
+        "seq": 21,
+        "leaf_id": 8
+    },
+    "Df2fyPsqeDhCkyteWVwsL1tTfhFygchuDBDy8YUhHJPk": {
+        "eligible": false,
+        "compressed": true,
+        "data_hash": "C9ybY1voubqwFY3RfgDroXvXd6reuM6ZhjC2jAVrQyAa",
+        "creator_hash": "7WJjsibXBHs5jBo5W1J2f8NYrGEc1eDCMgavasEinEcY",
+        "asset_hash": "EJryzovW6HnwuF4S2jvkQm8VY2nABC9aLcRdDVgaNEHa",
+        "tree": "AGMiLKtXX7PiVneM8S1KkTmCnF7X5zh6bKq4t1Mhrwpb",
+        "seq": 4,
+        "leaf_id": 3
+    },
+    "Ahsn3GcwiwuXCavbet26YCMGifG8XAEEZAkH6A8KM8sa": {
+        "eligible": false,
+        "compressed": true,
+        "data_hash": "849g69cMcDNekmDzfp8RtW7fXhbmsn2h1kr67D8YndJq",
+        "creator_hash": "syh1xpL9yb5sC3vJRDKHkS32ryAnYJjVCiSACRCjp2V",
+        "asset_hash": "7aYKaDpbqXCNSfk3Yz2WVYBZEBhg4ZG8rCMqVV7JD5FR",
+        "tree": "AGMiLKtXX7PiVneM8S1KkTmCnF7X5zh6bKq4t1Mhrwpb",
+        "seq": 2,
+        "leaf_id": 1
+    },
+    "9ZEFH8WVtfRqAbLdzkfYXANWVtrafrQjrUkZcswsZgcG": {
+        "eligible": false,
+        "compressed": true,
+        "data_hash": "E6AGG4URZaizT8pmF6k3RHppHWgGU6VCE285CPm7cbAB",
+        "creator_hash": "7WJjsibXBHs5jBo5W1J2f8NYrGEc1eDCMgavasEinEcY",
+        "asset_hash": "7ent4TvbuvkT5MJ2jJEcLKaFFUPg7y6cMv12idqi1vCB",
+        "tree": "AGMiLKtXX7PiVneM8S1KkTmCnF7X5zh6bKq4t1Mhrwpb",
+        "seq": 18,
+        "leaf_id": 4
+    },
+    "HWxE2EReU9VEixVNSw9e1QGUAGQJ32VACAPSH6dPSXbU": {
+        "eligible": false,
+        "compressed": true,
+        "data_hash": "E6AGG4URZaizT8pmF6k3RHppHWgGU6VCE285CPm7cbAB",
+        "creator_hash": "7WJjsibXBHs5jBo5W1J2f8NYrGEc1eDCMgavasEinEcY",
+        "asset_hash": "EVFvXE3UnnHUZMocueeHMHmyyJfCkW8BQybnUZxVkuMA",
+        "tree": "AGMiLKtXX7PiVneM8S1KkTmCnF7X5zh6bKq4t1Mhrwpb",
+        "seq": 6,
+        "leaf_id": 5
+    },
+    "7aKVZtBGW37kR3usNCUTdGHbUtwiLwVwkbc1DT7ikPaw": {
+        "eligible": false,
+        "compressed": true,
+        "data_hash": "E6AGG4URZaizT8pmF6k3RHppHWgGU6VCE285CPm7cbAB",
+        "creator_hash": "7WJjsibXBHs5jBo5W1J2f8NYrGEc1eDCMgavasEinEcY",
+        "asset_hash": "4gB2o5HJdZuikbttCPp5CW3Jemjd9RZnc4Kzyaex92Jj",
+        "tree": "AGMiLKtXX7PiVneM8S1KkTmCnF7X5zh6bKq4t1Mhrwpb",
+        "seq": 8,
+        "leaf_id": 7
+    }
+}

nft_ingester/tests/bubblegum_tests.rs

@@ -1,17 +1,18 @@
 #[cfg(test)]
 #[cfg(feature = "integration_tests")]
 mod tests {
+    use entities::api_req_params::{GetAsset, GetAssetProof};
     use metrics_utils::{ApiMetricsConfig, BackfillerMetricsConfig, IngesterMetricsConfig};
     use nft_ingester::{
         backfiller::{DirectBlockParser, TransactionsParser},
         bubblegum_updates_processor::BubblegumTxProcessor,
         buffer::Buffer,
         transaction_ingester,
     };
-    use rocks_db::Storage;
-    use std::fs::File;
+    use rocks_db::{offchain_data::OffChainData, Storage};
     use std::io::{self, Read};
     use std::sync::Arc;
+    use std::{fs::File, sync::atomic::AtomicBool};
     use testcontainers::clients::Cli;
     use tokio::sync::Mutex;
     use tokio::task::JoinSet;
@@ -62,6 +63,7 @@ mod tests {
             env.rocks_env.storage.clone(),
             Arc::new(IngesterMetricsConfig::new()),
             buffer.json_tasks.clone(),
+            true,
         ));
 
         let tx_ingester = Arc::new(transaction_ingester::BackfillTransactionIngester::new(
@@ -74,21 +76,143 @@ mod tests {
         ));
         let producer = rocks_storage.clone();
 
-        let transactions_parser = Arc::new(TransactionsParser::new(
-            env.rocks_env.storage.clone(),
-            consumer,
-            producer,
+        let keep_running = Arc::new(AtomicBool::new(true));
+
+        TransactionsParser::parse_slots(
+            consumer.clone(),
+            producer.clone(),
             Arc::new(BackfillerMetricsConfig::new()),
             1,
-            100,
+            slots_to_parse,
+            keep_running,
+        )
+        .await
+        .unwrap();
+
+        let file = File::open("./tests/artifacts/expected_proofs.json").unwrap();
+        let mut reader = io::BufReader::new(file);
+
+        let mut contents = String::new();
+        reader.read_to_string(&mut contents).unwrap();
+
+        let expected_results: serde_json::Value = serde_json::from_str(&contents).unwrap();
+
+        let assets_to_test_proof_for = vec![
+            "HZPrFymDBjKcYnUGFtQR6uZEoqN9MQCiEvYa75u3xTeX",
+            "F8XkxardSug8FSViEGF5XMSXQ93wBYmqmiGuWiYt2uV8",
+            "8roA736sYuZccUMGpT67LqtX86yZ4fb99Ga2Up6adZ9y",
+            "Df2fyPsqeDhCkyteWVwsL1tTfhFygchuDBDy8YUhHJPk",
+            "Ahsn3GcwiwuXCavbet26YCMGifG8XAEEZAkH6A8KM8sa",
+            "9ZEFH8WVtfRqAbLdzkfYXANWVtrafrQjrUkZcswsZgcG",
+            "HWxE2EReU9VEixVNSw9e1QGUAGQJ32VACAPSH6dPSXbU",
+            "7aKVZtBGW37kR3usNCUTdGHbUtwiLwVwkbc1DT7ikPaw",
+        ];
+
+        for asset in assets_to_test_proof_for.iter() {
+            let payload = GetAssetProof {
+                id: asset.to_string(),
+            };
+            let proof_result = api.get_asset_proof(payload).await.unwrap();
+
+            assert_eq!(proof_result, expected_results[*asset]);
+        }
+
+        env.teardown().await;
+    }
+
+    #[tokio::test]
+    #[tracing_test::traced_test]
+    async fn test_asset_compression_info() {
+        // write slots we need to parse because backfiller dropped it during raw transactions saving
+        let slots_to_parse: Vec<u64> = vec![
+            242049108, 242049247, 242049255, 242050728, 242050746, 242143893, 242143906, 242239091,
+            242239108, 242248687, 242560746, 242847845, 242848373, 242853752, 242856151, 242943141,
+            242943774, 242947970, 242948187, 242949333, 242949940, 242951695, 242952638,
+        ];
+
+        let tasks = JoinSet::new();
+        let mutexed_tasks = Arc::new(Mutex::new(tasks));
+
+        let tx_storage_dir = tempfile::TempDir::new().unwrap();
+
+        let storage_archieve = File::open("./tests/artifacts/test_rocks.zip").unwrap();
+
+        zip_extract::extract(storage_archieve, tx_storage_dir.path(), false).unwrap();
+
+        let transactions_storage = Storage::open(
+            &format!(
+                "{}{}",
+                tx_storage_dir.path().to_str().unwrap(),
+                "/test_rocks"
+            ),
+            mutexed_tasks.clone(),
+        )
+        .unwrap();
+
+        let rocks_storage = Arc::new(transactions_storage);
+
+        let cnt = 20;
+        let cli = Cli::default();
+        let (env, _generated_assets) = setup::TestEnvironment::create(&cli, cnt, 100).await;
+        let api = nft_ingester::api::api_impl::DasApi::new(
+            env.pg_env.client.clone(),
+            env.rocks_env.storage.clone(),
+            Arc::new(ApiMetricsConfig::new()),
+        );
+
+        let buffer = Arc::new(Buffer::new());
+
+        let bubblegum_updates_processor = Arc::new(BubblegumTxProcessor::new(
+            env.rocks_env.storage.clone(),
+            Arc::new(IngesterMetricsConfig::new()),
+            buffer.json_tasks.clone(),
+            true,
+        ));
+
+        let tx_ingester = Arc::new(transaction_ingester::BackfillTransactionIngester::new(
+            bubblegum_updates_processor.clone(),
+        ));
+
+        let consumer = Arc::new(DirectBlockParser::new(
+            tx_ingester.clone(),
+            Arc::new(BackfillerMetricsConfig::new()),
         ));
+        let producer = rocks_storage.clone();
+
+        let keep_running = Arc::new(AtomicBool::new(true));
+
+        TransactionsParser::parse_slots(
+            consumer.clone(),
+            producer.clone(),
+            Arc::new(BackfillerMetricsConfig::new()),
+            1,
+            slots_to_parse,
+            keep_running,
+        )
+        .await
+        .unwrap();
+
+        let metadata = OffChainData {
+            url: "https://supersweetcollection.notarealurl/token.json".to_string(),
+            metadata: "{\"msg\": \"hallo\"}".to_string(),
+        };
+        env.rocks_env
+            .storage
+            .asset_offchain_data
+            .put(metadata.url.clone(), metadata)
+            .unwrap();
 
-        transactions_parser
-            .parse_slots(slots_to_parse)
-            .await
+        let metadata = OffChainData {
+            url: "https://arweave.net/nbCWy-OEu7MG5ORuJMurP5A-65qO811R-vL_8l_JHQM".to_string(),
+            metadata: "{\"msg\": \"hallo\"}".to_string(),
+        };
+        env.rocks_env
+            .storage
+            .asset_offchain_data
+            .put(metadata.url.clone(), metadata)
             .unwrap();
 
-        let file = File::open("./tests/artifacts/expected_response.json").unwrap();
+        let file = File::open("./tests/artifacts/expected_compression.json").unwrap();
         let mut reader = io::BufReader::new(file);
 
         let mut contents = String::new();
@@ -108,12 +232,12 @@ mod tests {
         ];
 
         for asset in assets_to_test_proof_for.iter() {
-            let payload = nft_ingester::api::GetAssetProof {
+            let payload = GetAsset {
                 id: asset.to_string(),
             };
-            let proof_result = api.get_asset_proof(payload).await.unwrap();
+            let asset_info = api.get_asset(payload).await.unwrap();
 
-            assert_eq!(proof_result, expected_results[*asset]);
+            assert_eq!(asset_info["compression"], expected_results[*asset]);
         }
 
         env.teardown().await;