Add raw batch data syncing

.env.sample

@@ -0,0 +1,18 @@
+#Orderbook DB Credentials
+BARN_DB_URL=
+PROD_DB_URL=
+ANALYTICS_DB_URL=
+
+
+NETWORK=
+
+NODE_URL_ETHEREUM=
+NODE_URL_GNOSIS=
+NODE_URL_ARBITRUM=
+
+EPSILON_LOWER_ETHEREUM=
+EPSILON_UPPER_ETHEREUM=
+EPSILON_LOWER_GNOSIS=
+EPSILON_UPPER_GNOSIS=
+EPSILON_LOWER_ARBITRUM=
+EPSILON_UPPER_ARBITRUM=

requirements.txt

@@ -10,7 +10,7 @@ SQLAlchemy>=2.0,<3.0
 web3==6.20.3
 pandas-stubs>=1.5.1.221024
 boto3-stubs>=1.26.12
-black==23.12.0
+black>=24.3.0
 mypy==1.3.0
 mypy-extensions==1.0.0
 pylint>=2.14.4

src/main.py

@@ -1,19 +1,16 @@
 """Main Entry point for app_hash sync"""
+
 import argparse
 import asyncio
 import os
 from dataclasses import dataclass
-from pathlib import Path
 
 from dotenv import load_dotenv
 from web3 import Web3
 
 from src.fetch.orderbook import OrderbookFetcher
 from src.logger import set_log
 from src.models.tables import SyncTable
-from src.sync.config import (
-    BatchDataSyncConfig,
-)
 from src.sync.batch_data import sync_batch_data
 from src.utils import node_suffix
 
@@ -37,31 +34,24 @@ def __init__(self) -> None:
         arguments, _ = parser.parse_known_args()
         self.sync_table: SyncTable = arguments.sync_table
 
+
 def main() -> None:
     """
     Main function
     """
     load_dotenv()
     args = ScriptArgs()
     orderbook = OrderbookFetcher()
-    network = os.environ.get("NETWORK", "mainnet")
+    network = node_suffix(os.environ.get("NETWORK", "mainnet"))
     log.info(f"Network is set to: {network}")
-    web3 = Web3(
-        Web3.HTTPProvider(os.environ.get("NODE_URL" + "_" + node_suffix(network)))
-    )
-
-    # if args.sync_table == SyncTable.BATCH_DATA:
-    #     table = os.environ["BATCH_DATA_TARGET_TABLE"]
-    #     assert table, "BATCH DATA sync needs a BATCH_DATA_TARGET_TABLE env"
-    #     asyncio.run(
-    #         sync_batch_data(
-    #             web3,
-    #             orderbook,
-    #             config=BatchDataSyncConfig(table),
-    #         )
-    #     )
-    # else:
-    #     log.error(f"unsupported sync_table '{args.sync_table}'")
+    web3 = Web3(Web3.HTTPProvider(os.environ.get("NODE_URL" + "_" + network)))
+
+    if args.sync_table == SyncTable.BATCH_DATA:
+        asyncio.run(
+            sync_batch_data(web3, orderbook, network, recompute_previous_month=False)
+        )
+    else:
+        log.error(f"unsupported sync_table '{args.sync_table}'")
 
 
 if __name__ == "__main__":

src/scripts/batch_data_past_two_months.py

@@ -0,0 +1,40 @@
+"""
+Script to recompute batch rewards for the current and previous month.
+"""
+
+import argparse
+import os
+from web3 import Web3
+from dotenv import load_dotenv
+import asyncio
+
+
+from src.models.tables import SyncTable
+from src.fetch.orderbook import OrderbookFetcher
+from src.logger import set_log
+from src.utils import node_suffix
+from src.sync.batch_data import sync_batch_data
+
+
+log = set_log(__name__)
+
+if __name__ == "__main__":
+    load_dotenv()
+    parser = argparse.ArgumentParser("Script Arguments")
+    parser.add_argument(  # pylint: disable=duplicate-code
+        "--sync-table",
+        type=SyncTable,
+        required=True,
+        choices=list(SyncTable),
+    )
+    args, _ = parser.parse_known_args()
+
+    orderbook = OrderbookFetcher()
+    network = node_suffix(os.environ.get("NETWORK", "mainnet"))
+    log.info(f"Network is set to: {network}")
+    web3 = Web3(Web3.HTTPProvider(os.environ.get("NODE_URL" + "_" + network)))
+
+    if args.sync_table == SyncTable.BATCH_DATA:
+        asyncio.run(sync_batch_data(web3, orderbook, network, recompute_previous_month=True))
+    else:
+        log.error(f"unsupported sync_table '{args.sync_table}'")

src/sql/orderbook/create_batch_table.sql

@@ -0,0 +1,18 @@
+-- sample table name for creating the intermediate tables used in the analytics db to store batch data
+create table raw_batch_data_latest_odd_month_gnosis (
+    environment varchar(6) not null,
+    auction_id bigint not null,
+    settlement_block bigint,
+    block_deadline bigint not null,
+    tx_hash bytea,
+    solver bytea not null,
+    execution_cost numeric(78,0),
+    surplus numeric(78,0),
+    protocol_fee numeric(78,0),
+    network_fee numeric(78,0),
+    uncapped_payment_native_token numeric(78,0) not null,
+    capped_payment  numeric (78,0) not null,
+    winning_score numeric(78,0) not null,
+    reference_score numeric(78,0) not null,
+    PRIMARY KEY (block_deadline, auction_id, environment)
+);

src/sync/batch_data.py

@@ -1,42 +1,50 @@
 """Main Entry point for batch data sync"""
+
 import os
 from dotenv import load_dotenv
-from dune_client.client import DuneClient
 from web3 import Web3
 from src.fetch.orderbook import OrderbookFetcher, OrderbookEnv
 from src.logger import set_log
-from src.sync.config import BatchDataSyncConfig
 from src.sync.common import compute_block_and_month_range
 from src.models.block_range import BlockRange
 
 
 log = set_log(__name__)
 
 
-# async def sync_batch_data(
-#     node: Web3,
-#     orderbook: OrderbookFetcher,
-#     config: BatchDataSyncConfig,
-# ) -> None:
-#     """Batch data Sync Logic"""
-#     load_dotenv()
-#     network = os.environ["NETWORK"]
+async def sync_batch_data(
+    node: Web3,
+    orderbook: OrderbookFetcher,
+    network: str,
+    recompute_previous_month: bool,
+) -> None:
+    """
+    Batch data Sync Logic. The recompute_previous_month flag, when enabled, forces a recomputation
+    of the previous month. If it is set to False, previous month is still recomputed when the current
+    date is the first day of the current month.
+    """
 
-#     block_range_list, months_list, is_even = compute_block_and_month_range(node)
-#     for i, _ in enumerate(block_range_list):
-#         start_block = block_range_list[i][0]
-#         end_block = block_range_list[i][1]
-#         if is_even[i]:
-#             table_name = "raw_batch_data_latest_even_month_" + str(network)
-#         else:
-#             table_name = "raw_batch_data_latest_odd_month_" + str(network)
-#         block_range = BlockRange(block_from=start_block, block_to=end_block)
-#         log.info(
-#             f"About to process block range ({start_block}, {end_block}) for month {months_list[i]}"
-#         )
-#         batch_data = orderbook.get_batch_data(block_range)
-#         log.info("SQL query successfully executed. About to update analytics table.")
-#         batch_data.to_sql(table_name,orderbook._pg_engine(OrderbookEnv.ANALYTICS),if_exists= 'replace')
-#         log.info(
-#             f"batch data sync run completed successfully for month {months_list[i]}"
-#         )
+    block_range_list, months_list, is_even = compute_block_and_month_range(
+        node, recompute_previous_month
+    )
+    for i, _ in enumerate(block_range_list):
+        start_block = block_range_list[i][0]
+        end_block = block_range_list[i][1]
+        if is_even[i]:
+            table_name = "raw_batch_data_latest_even_month_" + network.lower()
+        else:
+            table_name = "raw_batch_data_latest_odd_month_" + network.lower()
+        block_range = BlockRange(block_from=start_block, block_to=end_block)
+        log.info(
+            f"About to process block range ({start_block}, {end_block}) for month {months_list[i]}"
+        )
+        batch_data = orderbook.get_batch_data(block_range)
+        log.info("SQL query successfully executed. About to update analytics table.")
+        batch_data.to_sql(
+            table_name,
+            orderbook._pg_engine(OrderbookEnv.ANALYTICS),
+            if_exists="replace",
+        )
+        log.info(
+            f"batch data sync run completed successfully for month {months_list[i]}"
+        )