Merge pull request #17 from taikoxyz/increase-tx-size

Static file watcher + Improve sync API

crates/cli/commands/src/common.rs

@@ -86,7 +86,7 @@ impl EnvironmentArgs {
             ),
             AccessRights::RO => (
                 Arc::new(open_db_read_only(&db_path, self.db.database_args())?),
-                StaticFileProvider::read_only(sf_path)?,
+                StaticFileProvider::read_only(sf_path, false)?,
             ),
         };
 

crates/cli/commands/src/db/stats.rs

@@ -168,7 +168,7 @@ impl Command {
         }
 
         let static_files = iter_static_files(data_dir.static_files())?;
-        let static_file_provider = StaticFileProvider::read_only(data_dir.static_files())?;
+        let static_file_provider = StaticFileProvider::read_only(data_dir.static_files(), false)?;
 
         let mut total_data_size = 0;
         let mut total_index_size = 0;

crates/gwyneth/src/exex.rs

@@ -2,6 +2,8 @@ use std::{collections::HashMap, marker::PhantomData, sync::Arc};
 
 use alloy_rlp::Decodable;
 use alloy_sol_types::{sol, SolEventInterface};
+use reth_network::NetworkInfo;
+use reth_rpc_api::eth::helpers::EthApiSpec;
 
 use crate::{
     engine_api::EngineApiContext, GwynethEngineTypes, GwynethNode, GwynethPayloadAttributes,
@@ -30,7 +32,7 @@ use reth_transaction_pool::{
 };
 use RollupContract::{BlockProposed, RollupContractEvents};
 use reth_provider::BlockReaderIdExt;
-use reth_provider::ROLLUP_SYNC_DATA;
+use reth_provider::{GWYNETH_SYNCED_L1_BLOCK_IDX, GWYNETH_SYNCED_L2_BLOCK_IDX};
 
 const ROLLUP_CONTRACT_ADDRESS: Address = address!("9fCF7D13d10dEdF17d0f24C62f0cf4ED462f65b7");
 pub const BASE_CHAIN_ID: u64 = 167010;
@@ -86,6 +88,9 @@ impl<Node: reth_node_api::FullNodeComponents> Rollup<Node> {
                 _marker: PhantomData::<GwynethEngineTypes>,
             };
             engine_apis.push(engine_api);
+
+            let mut l2_block_indices = GWYNETH_SYNCED_L2_BLOCK_IDX.lock().unwrap();
+            l2_block_indices.insert(node.chain_spec().chain().id(), 0);
         }
         Ok(Self { ctx, nodes, /* payload_event_stream, */ engine_apis, num_l2_blocks: 0 })
     }
@@ -104,8 +109,8 @@ impl<Node: reth_node_api::FullNodeComponents> Rollup<Node> {
 
                 // Update the sync data
                 unsafe {
-                    ROLLUP_SYNC_DATA = committed_chain.tip().number;
-                    println!("Updated sync data: {}", ROLLUP_SYNC_DATA);
+                    GWYNETH_SYNCED_L1_BLOCK_IDX = committed_chain.tip().number;
+                    println!("Updated L1 sync data: {}", GWYNETH_SYNCED_L1_BLOCK_IDX);
                 }
 
                 self.ctx.events.send(ExExEvent::FinishedHeight(committed_chain.tip().number))?;
@@ -288,7 +293,6 @@ impl<Node: reth_node_api::FullNodeComponents> Rollup<Node> {
 
                 // loop {
                 //     // wait for the block to commit
-                //     tokio::time::sleep(std::time::Duration::from_millis(1000)).await;
                 //     if let Some(latest_block) =
                 //         self.nodes[node_idx].provider.block_by_number_or_tag(BlockNumberOrTag::Latest)?
                 //     {
@@ -299,11 +303,15 @@ impl<Node: reth_node_api::FullNodeComponents> Rollup<Node> {
                 //             break
                 //         }
                 //     }
-                //     println!("waiting on L2 block for {}: {}", node_chain_id, payload.block().number)
+                //     println!("waiting on L2 block for {}: {}", node_chain_id, payload.block().number);
+                //     tokio::time::sleep(std::time::Duration::from_millis(2)).await;
                 // }
 
                 println!("[L1 block {}] Done with block {}: {}", block.number, node_chain_id, payload.block().number);
 
+                let mut l2_block_indices = GWYNETH_SYNCED_L2_BLOCK_IDX.lock().unwrap();
+                l2_block_indices.insert(self.nodes[node_idx].chain_spec().chain().id(), payload.block().number);
+
                 self.num_l2_blocks += 1;
             }
         }

crates/rpc/rpc-eth-api/src/core.rs

@@ -18,7 +18,7 @@ use reth_rpc_types::{
 };
 use reth_transaction_pool::{PoolTransaction, TransactionPool};
 use tracing::trace;
-use reth_provider::ROLLUP_SYNC_DATA;
+use reth_provider::{GWYNETH_SYNCED_L1_BLOCK_IDX, GWYNETH_SYNCED_L2_BLOCK_IDX};
 
 use crate::{
     helpers::{
@@ -360,9 +360,13 @@ pub trait EthApi<T: RpcObject, B: RpcObject> {
         block_number: Option<BlockId>,
     ) -> RpcResult<EIP1186AccountProofResponse>;
 
-    /// Returns the L1 block to which all exexes are synced
-    #[method(name = "getSyncData")]
-    async fn get_sync_data(&self) -> RpcResult<U64>;
+    /// Returns the L1 block for which all L2 blocks have been processed
+    #[method(name = "getSyncedL1BlockIdx")]
+    async fn get_synced_l1_block_idx(&self) -> RpcResult<U64>;
+
+    /// Returns the L2 block to which this L2 is synced (but perhaps not yet fully processed)
+    #[method(name = "getSyncedL2BlockIdx")]
+    async fn get_synced_l2_block_idx(&self) -> RpcResult<U64>;
 }
 
 #[async_trait::async_trait]
@@ -842,11 +846,20 @@ where
         Ok(EthState::get_proof(self, address, keys, block_number)?.await?)
     }
 
-    /// Handler for: `eth_getSyncData`
-    async fn get_sync_data(&self) -> RpcResult<U64> {
-        let sync_data = unsafe {
-            ROLLUP_SYNC_DATA
+    /// Handler for: `eth_getSyncedL1BlockIdx`
+    async fn get_synced_l1_block_idx(&self) -> RpcResult<U64> {
+        let l1_block_idx = unsafe {
+            GWYNETH_SYNCED_L1_BLOCK_IDX
+        };
+        Ok(U64::from(l1_block_idx))
+    }
+
+    /// Handler for: `eth_getSyncedL2BlockIdx`
+    async fn get_synced_l2_block_idx(&self) -> RpcResult<U64> {
+        let chain_id = EthApiSpec::chain_id(self).into_limbs()[0];
+        let l2_block_idx = unsafe {
+            *GWYNETH_SYNCED_L2_BLOCK_IDX.lock().unwrap().get(&chain_id).unwrap_or(&0)
         };
-        Ok(U64::from(sync_data))
+        Ok(U64::from(l2_block_idx))
     }
 }

crates/storage/nippy-jar/src/lib.rs

@@ -49,7 +49,7 @@ const NIPPY_JAR_VERSION: usize = 1;
 
 const INDEX_FILE_EXTENSION: &str = "idx";
 const OFFSETS_FILE_EXTENSION: &str = "off";
-const CONFIG_FILE_EXTENSION: &str = "conf";
+pub const CONFIG_FILE_EXTENSION: &str = "conf";
 
 /// A [`RefRow`] is a list of column value slices pointing to either an internal buffer or a
 /// memory-mapped file.

crates/storage/provider/Cargo.toml

@@ -55,11 +55,13 @@ dashmap = { workspace = true, features = ["inline"] }
 strum.workspace = true
 
 # test-utils
-once_cell = { workspace = true, optional = true }
+once_cell.workspace = true
 
 # parallel utils
 rayon.workspace = true
 
+notify = "6.1.1"
+
 [dev-dependencies]
 reth-db = { workspace = true, features = ["test-utils"] }
 reth-primitives = { workspace = true, features = ["arbitrary", "test-utils"] }
@@ -70,7 +72,7 @@ parking_lot.workspace = true
 tempfile.workspace = true
 assert_matches.workspace = true
 rand.workspace = true
-once_cell.workspace = true
+#once_cell.workspace = true
 eyre.workspace = true
 
 [features]
@@ -81,5 +83,5 @@ test-utils = [
     "reth-nippy-jar/test-utils",
     "reth-trie/test-utils",
     "reth-chain-state/test-utils",
-    "once_cell",
+    #"once_cell",
 ]

crates/storage/provider/src/lib.rs

@@ -19,6 +19,7 @@ use std::{collections::HashMap, sync::{Arc, LazyLock}};
 use providers::BlockchainProvider;
 use reth_db::{test_utils::TempDatabase, DatabaseEnv};
 use std::sync::Mutex;
+use once_cell::sync::Lazy;
 pub use traits::*;
 
 /// Provider trait implementations.
@@ -77,4 +78,8 @@ pub static NODES: LazyLock<Mutex<HashMap<u64, BlockchainProvider<Arc<DatabaseEnv
 
 // pub static NODES: LazyLock<Mutex<HashMap<u64, ProviderFactory<BlockchainProvider<Arc<TempDatabase<DatabaseEnv>>>>>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
 
-pub static mut ROLLUP_SYNC_DATA: u64 = 0;
+pub static mut GWYNETH_SYNCED_L1_BLOCK_IDX: u64 = 0;
+pub static GWYNETH_SYNCED_L2_BLOCK_IDX: Lazy<Mutex<HashMap<u64, u64>>> = Lazy::new(|| {
+    let map = HashMap::new();
+    Mutex::new(map)
+});