Refactor exchange info to use symbols DB

api/account/account.py

@@ -7,69 +7,12 @@
     json_response_message,
     json_response_error,
 )
-from database.db import setup_db
-from requests_cache import CachedSession, MongoCache
-from pymongo import MongoClient
-from decimal import Decimal
 
 
 class Account(BinbotApi):
-    db = setup_db()
-
     def __init__(self):
         pass
 
-    def setup_mongocache(self):
-        mongo: MongoClient = MongoClient(
-            host=os.getenv("MONGO_HOSTNAME"),
-            port=int(os.getenv("MONGO_PORT", 2017)),
-            authSource="admin",
-            username=os.getenv("MONGO_AUTH_USERNAME"),
-            password=os.getenv("MONGO_AUTH_PASSWORD"),
-        )
-        mongo_cache = MongoCache(connection=mongo)
-        return mongo_cache
-
-    def calculate_price_precision(self, symbol) -> int:
-        precision = -1 * (
-            Decimal(str(self.price_filter_by_symbol(symbol, "tickSize")))
-            .as_tuple()
-            .exponent
-        )
-        price_precision = int(precision)
-        return price_precision
-
-    def calculate_qty_precision(self, symbol) -> int:
-        precision = -1 * (
-            Decimal(str(self.lot_size_by_symbol(symbol, "stepSize")))
-            .as_tuple()
-            .exponent
-        )
-        qty_precision = int(precision)
-        return qty_precision
-
-    def _exchange_info(self, symbol=None):
-        """
-        This must be a separate method because classes use it with inheritance
-
-        This request is used in many places to retrieve data about symbols, precisions etc.
-        It is a high weight endpoint, thus Binance could ban our IP
-        However it is not real-time updated data, so cache is used to avoid hitting endpoint
-        too many times and still be able to re-request data everywhere.
-
-        In addition, it uses MongoDB, with a separate database called "mongo_cache"
-        """
-        params = {}
-        if symbol:
-            params["symbol"] = symbol
-
-        mongo_cache = self.setup_mongocache()
-        # set up a cache that expires in 1440'' (24hrs)
-        session = CachedSession("http_cache", backend=mongo_cache, expire_after=1440)
-        exchange_info_res = session.get(url=f"{self.exchangeinfo_url}", params=params)
-        exchange_info = handle_binance_errors(exchange_info_res)
-        return exchange_info
-
     def _get_price_from_book_order(self, data: dict, order_side: bool, index: int):
         """
         Buy order = get bid prices = True
@@ -88,24 +31,6 @@ def get_ticker_price(self, symbol: str):
         data = handle_binance_errors(res)
         return data["price"]
 
-    def find_quoteAsset(self, symbol):
-        """
-        e.g. BNBBTC: base asset = BTC
-        """
-        symbols = self._exchange_info(symbol)
-        quote_asset = symbols["symbols"][0]
-        if quote_asset:
-            quote_asset = quote_asset["quoteAsset"]
-        return quote_asset
-
-    def find_baseAsset(self, symbol):
-        """
-        e.g. BNBBTC: base asset = BNB
-        """
-        symbols = self._exchange_info(symbol)
-        base_asset = symbols["symbols"][0]["baseAsset"]
-        return base_asset
-
     def find_base_asset_json(self, symbol):
         data = self.find_baseAsset(symbol)
         return json_response({"data": data})
@@ -161,48 +86,6 @@ def get_quote_asset_precision(self, symbol, quote=True):
         )
         return asset_precision
 
-    def price_filter_by_symbol(self, symbol, filter_limit):
-        """
-        PRICE_FILTER restrictions from /exchangeinfo
-        @params:
-            - symbol: string - pair/market e.g. BNBBTC
-            - filter_limit: string - minPrice or maxPrice
-        """
-        symbols = self._exchange_info(symbol)
-        market = symbols["symbols"][0]
-        price_filter = next(
-            (m for m in market["filters"] if m["filterType"] == "PRICE_FILTER")
-        )
-        return price_filter[filter_limit].rstrip(".0")
-
-    def lot_size_by_symbol(self, symbol, lot_size_limit):
-        """
-        LOT_SIZE (quantity) restrictions from /exchangeinfo
-        @params:
-            - symbol: string - pair/market e.g. BNBBTC
-            - lot_size_limit: string - minQty, maxQty, stepSize
-        """
-        symbols = self._exchange_info(symbol)
-        market = symbols["symbols"][0]
-        quantity_filter: list = next(
-            (m for m in market["filters"] if m["filterType"] == "LOT_SIZE")
-        )
-        return quantity_filter[lot_size_limit].rstrip(".0")
-
-    def min_notional_by_symbol(self, symbol, min_notional_limit="minNotional"):
-        """
-        MIN_NOTIONAL (price x quantity) restrictions from /exchangeinfo
-        @params:
-            - symbol: string - pair/market e.g. BNBBTC
-            - min_notional_limit: string - minNotional
-        """
-        symbols = self._exchange_info(symbol)
-        market = symbols["symbols"][0]
-        min_notional_filter = next(
-            (m for m in market["filters"] if m["filterType"] == "MIN_NOTIONAL")
-        )
-        return min_notional_filter[min_notional_limit]
-
     def get_raw_balance(self) -> list:
         """
         Unrestricted balance

api/account/assets.py

@@ -7,10 +7,9 @@
 from deals.factory import DealAbstract
 from tools.handle_error import json_response, json_response_error, json_response_message
 from tools.round_numbers import round_numbers, ts_to_day, round_timestamp
-from tools.exceptions import BinanceErrors, LowBalanceCleanupError, BinbotErrors
+from tools.exceptions import BinanceErrors, LowBalanceCleanupError
 from tools.enum_definitions import Strategy
 from database.bot_crud import BotTableCrud
-from database.symbols_crud import SymbolsCrud
 
 
 class Assets(Account):
@@ -190,27 +189,6 @@ def map_balance_with_benchmark(self, start_date, end_date):
         )
         return resp
 
-    def refresh_symbols_table(self):
-        """
-        Refresh the symbols table
-        """
-        data = self.ticker()
-        symbol_controller = SymbolsCrud()
-        for item in data:
-            try:
-                symbol = symbol_controller.get_symbol(item["symbol"])
-            except BinbotErrors:
-                pass
-
-            # Only store fiat market, exclude other fiats.
-            if item["symbol"].endswith("USDC") and not symbol and item["symbol"].startswith(("DOWN", "UP", "AUD", "USDT", "EUR", "GBP")):
-                if item["symbol"] == "BTCUSDC":
-                    symbol_controller.add_symbol(symbol=item["symbol"], active=False)
-                else:
-                    symbol_controller.add_symbol(item["symbol"])
-
-        return json_response_message("Successfully refreshed symbols table.")
-
     def clean_balance_assets(self, bypass=False):
         """
         Check if there are many small assets (0.000.. BTC)

api/apis.py

@@ -7,6 +7,9 @@
 from tools.handle_error import handle_binance_errors, json_response, json_response_error
 from tools.exceptions import IsolateBalanceError
 from py3cw.request import Py3CW
+from requests_cache import MongoCache
+from pymongo import MongoClient
+from decimal import Decimal
 
 
 class BinanceApi:
@@ -68,6 +71,17 @@ class BinanceApi:
     max_borrow_url = f"{BASE}/sapi/v1/margin/maxBorrowable"
     interest_history_url = f"{BASE}/sapi/v1/margin/interestHistory"
 
+    def setup_mongocache(self):
+        mongo: MongoClient = MongoClient(
+            host=os.getenv("MONGO_HOSTNAME"),
+            port=int(os.getenv("MONGO_PORT", 2017)),
+            authSource="admin",
+            username=os.getenv("MONGO_AUTH_USERNAME"),
+            password=os.getenv("MONGO_AUTH_PASSWORD"),
+        )
+        mongo_cache = MongoCache(connection=mongo)
+        return mongo_cache
+
     def request(
         self,
         url,
@@ -137,6 +151,106 @@ def get_listen_key(self):
     No security endpoints
     """
 
+    def _exchange_info(self, symbol=None):
+        """
+        This must be a separate method because classes use it with inheritance
+
+        This request is used in many places to retrieve data about symbols, precisions etc.
+        It is a high weight endpoint, thus Binance could ban our IP
+        However it is not real-time updated data, so cache is used to avoid hitting endpoint
+        too many times and still be able to re-request data everywhere.
+
+        In addition, it uses MongoDB, with a separate database called "mongo_cache"
+        """
+        params = {}
+        if symbol:
+            params["symbol"] = symbol
+
+        # mongo_cache = self.setup_mongocache()
+        # set up a cache that expires in 1440'' (24hrs)
+        # session = CachedSession("http_cache", backend=mongo_cache, expire_after=1440)
+        exchange_info_res = self.request(url=f"{self.exchangeinfo_url}", params=params)
+        # exchange_info = handle_binance_errors(exchange_info_res)
+        return exchange_info_res
+
+    def price_filter_by_symbol(self, symbol, filter_limit):
+        """
+        PRICE_FILTER restrictions from /exchangeinfo
+        @params:
+            - symbol: string - pair/market e.g. BNBBTC
+            - filter_limit: string - minPrice or maxPrice
+        """
+        symbols = self._exchange_info(symbol)
+        market = symbols["symbols"][0]
+        price_filter = next(
+            (m for m in market["filters"] if m["filterType"] == "PRICE_FILTER")
+        )
+        return price_filter[filter_limit].rstrip(".0")
+
+    def lot_size_by_symbol(self, symbol, lot_size_limit):
+        """
+        LOT_SIZE (quantity) restrictions from /exchangeinfo
+        @params:
+            - symbol: string - pair/market e.g. BNBBTC
+            - lot_size_limit: string - minQty, maxQty, stepSize
+        """
+        symbols = self._exchange_info(symbol)
+        market = symbols["symbols"][0]
+        quantity_filter: list = next(
+            (m for m in market["filters"] if m["filterType"] == "LOT_SIZE")
+        )
+        return quantity_filter[lot_size_limit].rstrip(".0")
+
+    def min_notional_by_symbol(self, symbol, min_notional_limit="minNotional"):
+        """
+        MIN_NOTIONAL (price x quantity) restrictions from /exchangeinfo
+        @params:
+            - symbol: string - pair/market e.g. BNBBTC
+            - min_notional_limit: string - minNotional
+        """
+        symbols = self._exchange_info(symbol)
+        market = symbols["symbols"][0]
+        min_notional_filter = next(
+            (m for m in market["filters"] if m["filterType"] == "MIN_NOTIONAL")
+        )
+        return min_notional_filter[min_notional_limit]
+
+    def calculate_price_precision(self, symbol) -> int:
+        precision = -1 * (
+            Decimal(str(self.price_filter_by_symbol(symbol, "tickSize")))
+            .as_tuple()
+            .exponent
+        )
+        price_precision = int(precision)
+        return price_precision
+
+    def calculate_qty_precision(self, symbol) -> int:
+        precision = -1 * (
+            Decimal(str(self.lot_size_by_symbol(symbol, "stepSize")))
+            .as_tuple()
+            .exponent
+        )
+        qty_precision = int(precision)
+        return qty_precision
+
+    def find_quoteAsset(self, symbol):
+        """
+        e.g. BNBBTC: base asset = BTC
+        """
+        symbols = self._exchange_info(symbol)
+        quote_asset = symbols["symbols"][0]
+        if quote_asset:
+            quote_asset = quote_asset["quoteAsset"]
+        return quote_asset
+
+    def find_baseAsset(self, symbol):
+        """
+        e.g. BNBBTC: base asset = BNB
+        """
+        symbols = self._exchange_info(symbol)
+        base_asset = symbols["symbols"][0]["baseAsset"]
+        return base_asset
+
     def ticker(self, symbol: str | None = None, json: bool = True):
         params = {}
         if symbol:

api/cronjobs.py

@@ -5,7 +5,7 @@
 from account.assets import Assets
 from charts.controllers import MarketDominationController
 from database.utils import independent_session
-
+from database.symbols_crud import SymbolsCrud
 
 logging.Formatter.converter = time.gmtime  # date time in GMT/UTC
 logging.basicConfig(
@@ -24,14 +24,6 @@ def main():
 
     # Jobs should be distributed as far as possible from each other
     # to avoid overloading RAM and also avoid hitting rate limits due to high weight
-    scheduler.add_job(
-        func=assets.refresh_symbols_table,
-        trigger="cron",
-        timezone=timezone,
-        hour=4,
-        minute=1,
-        id="store_balance",
-    )
     scheduler.add_job(
         func=assets.store_balance,
         trigger="cron",

api/database/api_db.py

@@ -19,6 +19,7 @@
 from alembic import command
 from database.utils import engine
 from account.assets import Assets
+from database.symbols_crud import SymbolsCrud
 
 
 class ApiDb:
@@ -29,6 +30,7 @@ class ApiDb:
     def __init__(self):
         self.session = Session(engine)
         self.assets_collection = Assets(self.session)
+        self.symbols = SymbolsCrud(self.session)
         pass
 
     def init_db(self):
@@ -37,7 +39,7 @@ def init_db(self):
         self.init_users()
         self.create_dummy_bot()
         self.init_autotrade_settings()
-        self.assets_collection.refresh_symbols_table()
+        self.symbols.refresh_symbols_table()
         self.assets_collection.store_balance()
         self.session.close()
         logging.info("Finishing db operations")

api/database/models/symbol_table.py

@@ -15,3 +15,16 @@ class SymbolTable(SQLModel, table=True):
     )
     active: bool = Field(default=True, description="Blacklisted items = false")
     blacklist_reason: str = Field(default="")
+    quote_asset: str = Field(
+        default="", description="in BTCUSDC, BTC would be quote asset"
+    )
+    base_asset: str = Field(
+        default="", description="in BTCUSDC, USDC would be base asset"
+    )
+    price_precision: int = Field(
+        default=0,
+        description="Usually there are 2 price precisions, one for base and another for quote, here we usually indicate quote, since we always use the same base: USDC",
+    )
+    qty_precision: int = Field(default=0)
+    min_qty: float = Field(default=0, description="Minimum qty (quote asset) value")
+    min_notional: float = Field(default=0, description="Minimum price x qty value")