diff --git a/api/account/account.py b/api/account/account.py index 25dcf749..fd8ae78b 100644 --- a/api/account/account.py +++ b/api/account/account.py @@ -7,69 +7,12 @@ json_response_message, json_response_error, ) -from database.db import setup_db -from requests_cache import CachedSession, MongoCache -from pymongo import MongoClient -from decimal import Decimal class Account(BinbotApi): - db = setup_db() - def __init__(self): pass - def setup_mongocache(self): - mongo: MongoClient = MongoClient( - host=os.getenv("MONGO_HOSTNAME"), - port=int(os.getenv("MONGO_PORT", 2017)), - authSource="admin", - username=os.getenv("MONGO_AUTH_USERNAME"), - password=os.getenv("MONGO_AUTH_PASSWORD"), - ) - mongo_cache = MongoCache(connection=mongo) - return mongo_cache - - def calculate_price_precision(self, symbol) -> int: - precision = -1 * ( - Decimal(str(self.price_filter_by_symbol(symbol, "tickSize"))) - .as_tuple() - .exponent - ) - price_precision = int(precision) - return price_precision - - def calculate_qty_precision(self, symbol) -> int: - precision = -1 * ( - Decimal(str(self.lot_size_by_symbol(symbol, "stepSize"))) - .as_tuple() - .exponent - ) - qty_precision = int(precision) - return qty_precision - - def _exchange_info(self, symbol=None): - """ - This must be a separate method because classes use it with inheritance - - This request is used in many places to retrieve data about symbols, precisions etc. - It is a high weight endpoint, thus Binance could ban our IP - However it is not real-time updated data, so cache is used to avoid hitting endpoint - too many times and still be able to re-request data everywhere. - - In addition, it uses MongoDB, with a separate database called "mongo_cache" - """ - params = {} - if symbol: - params["symbol"] = symbol - - mongo_cache = self.setup_mongocache() - # set up a cache that expires in 1440'' (24hrs) - session = CachedSession("http_cache", backend=mongo_cache, expire_after=1440) - exchange_info_res = session.get(url=f"{self.exchangeinfo_url}", params=params) - exchange_info = handle_binance_errors(exchange_info_res) - return exchange_info - def _get_price_from_book_order(self, data: dict, order_side: bool, index: int): """ Buy order = get bid prices = True @@ -88,24 +31,6 @@ def get_ticker_price(self, symbol: str): data = handle_binance_errors(res) return data["price"] - def find_quoteAsset(self, symbol): - """ - e.g. BNBBTC: base asset = BTC - """ - symbols = self._exchange_info(symbol) - quote_asset = symbols["symbols"][0] - if quote_asset: - quote_asset = quote_asset["quoteAsset"] - return quote_asset - - def find_baseAsset(self, symbol): - """ - e.g. BNBBTC: base asset = BNB - """ - symbols = self._exchange_info(symbol) - base_asset = symbols["symbols"][0]["baseAsset"] - return base_asset - def find_base_asset_json(self, symbol): data = self.find_baseAsset(symbol) return json_response({"data": data}) @@ -161,48 +86,6 @@ def get_quote_asset_precision(self, symbol, quote=True): ) return asset_precision - def price_filter_by_symbol(self, symbol, filter_limit): - """ - PRICE_FILTER restrictions from /exchangeinfo - @params: - - symbol: string - pair/market e.g. BNBBTC - - filter_limit: string - minPrice or maxPrice - """ - symbols = self._exchange_info(symbol) - market = symbols["symbols"][0] - price_filter = next( - (m for m in market["filters"] if m["filterType"] == "PRICE_FILTER") - ) - return price_filter[filter_limit].rstrip(".0") - - def lot_size_by_symbol(self, symbol, lot_size_limit): - """ - LOT_SIZE (quantity) restrictions from /exchangeinfo - @params: - - symbol: string - pair/market e.g. BNBBTC - - lot_size_limit: string - minQty, maxQty, stepSize - """ - symbols = self._exchange_info(symbol) - market = symbols["symbols"][0] - quantity_filter: list = next( - (m for m in market["filters"] if m["filterType"] == "LOT_SIZE") - ) - return quantity_filter[lot_size_limit].rstrip(".0") - - def min_notional_by_symbol(self, symbol, min_notional_limit="minNotional"): - """ - MIN_NOTIONAL (price x quantity) restrictions from /exchangeinfo - @params: - - symbol: string - pair/market e.g. BNBBTC - - min_notional_limit: string - minNotional - """ - symbols = self._exchange_info(symbol) - market = symbols["symbols"][0] - min_notional_filter = next( - (m for m in market["filters"] if m["filterType"] == "MIN_NOTIONAL") - ) - return min_notional_filter[min_notional_limit] - def get_raw_balance(self) -> list: """ Unrestricted balance diff --git a/api/account/assets.py b/api/account/assets.py index b4fee291..883621d1 100644 --- a/api/account/assets.py +++ b/api/account/assets.py @@ -7,10 +7,9 @@ from deals.factory import DealAbstract from tools.handle_error import json_response, json_response_error, json_response_message from tools.round_numbers import round_numbers, ts_to_day, round_timestamp -from tools.exceptions import BinanceErrors, LowBalanceCleanupError, BinbotErrors +from tools.exceptions import BinanceErrors, LowBalanceCleanupError from tools.enum_definitions import Strategy from database.bot_crud import BotTableCrud -from database.symbols_crud import SymbolsCrud class Assets(Account): @@ -190,27 +189,6 @@ def map_balance_with_benchmark(self, start_date, end_date): ) return resp - def refresh_symbols_table(self): - """ - Refresh the symbols table - """ - data = self.ticker() - symbol_controller = SymbolsCrud() - for item in data: - try: - symbol = symbol_controller.get_symbol(item["symbol"]) - except BinbotErrors: - pass - - # Only store fiat market, exclude other fiats. - if item["symbol"].endswith("USDC") and not symbol and item["symbol"].startswith(("DOWN", "UP", "AUD", "USDT", "EUR", "GBP")): - if item["symbol"] == "BTCUSDC": - symbol_controller.add_symbol(symbol=item["symbol"], active=False) - else: - symbol_controller.add_symbol(item["symbol"]) - - return json_response_message("Successfully refreshed symbols table.") - def clean_balance_assets(self, bypass=False): """ Check if there are many small assets (0.000.. BTC) diff --git a/api/apis.py b/api/apis.py index 3d115d29..5be6d4fe 100644 --- a/api/apis.py +++ b/api/apis.py @@ -7,6 +7,9 @@ from tools.handle_error import handle_binance_errors, json_response, json_response_error from tools.exceptions import IsolateBalanceError from py3cw.request import Py3CW +from requests_cache import MongoCache +from pymongo import MongoClient +from decimal import Decimal class BinanceApi: @@ -68,6 +71,17 @@ class BinanceApi: max_borrow_url = f"{BASE}/sapi/v1/margin/maxBorrowable" interest_history_url = f"{BASE}/sapi/v1/margin/interestHistory" + def setup_mongocache(self): + mongo: MongoClient = MongoClient( + host=os.getenv("MONGO_HOSTNAME"), + port=int(os.getenv("MONGO_PORT", 2017)), + authSource="admin", + username=os.getenv("MONGO_AUTH_USERNAME"), + password=os.getenv("MONGO_AUTH_PASSWORD"), + ) + mongo_cache = MongoCache(connection=mongo) + return mongo_cache + def request( self, url, @@ -137,6 +151,106 @@ def get_listen_key(self): No security endpoints """ + def _exchange_info(self, symbol=None): + """ + This must be a separate method because classes use it with inheritance + + This request is used in many places to retrieve data about symbols, precisions etc. + It is a high weight endpoint, thus Binance could ban our IP + However it is not real-time updated data, so cache is used to avoid hitting endpoint + too many times and still be able to re-request data everywhere. + + In addition, it uses MongoDB, with a separate database called "mongo_cache" + """ + params = {} + if symbol: + params["symbol"] = symbol + + # mongo_cache = self.setup_mongocache() + # set up a cache that expires in 1440'' (24hrs) + # session = CachedSession("http_cache", backend=mongo_cache, expire_after=1440) + exchange_info_res = self.request(url=f"{self.exchangeinfo_url}", params=params) + # exchange_info = handle_binance_errors(exchange_info_res) + return exchange_info_res + + def price_filter_by_symbol(self, symbol, filter_limit): + """ + PRICE_FILTER restrictions from /exchangeinfo + @params: + - symbol: string - pair/market e.g. BNBBTC + - filter_limit: string - minPrice or maxPrice + """ + symbols = self._exchange_info(symbol) + market = symbols["symbols"][0] + price_filter = next( + (m for m in market["filters"] if m["filterType"] == "PRICE_FILTER") + ) + return price_filter[filter_limit].rstrip(".0") + + def lot_size_by_symbol(self, symbol, lot_size_limit): + """ + LOT_SIZE (quantity) restrictions from /exchangeinfo + @params: + - symbol: string - pair/market e.g. BNBBTC + - lot_size_limit: string - minQty, maxQty, stepSize + """ + symbols = self._exchange_info(symbol) + market = symbols["symbols"][0] + quantity_filter: list = next( + (m for m in market["filters"] if m["filterType"] == "LOT_SIZE") + ) + return quantity_filter[lot_size_limit].rstrip(".0") + + def min_notional_by_symbol(self, symbol, min_notional_limit="minNotional"): + """ + MIN_NOTIONAL (price x quantity) restrictions from /exchangeinfo + @params: + - symbol: string - pair/market e.g. BNBBTC + - min_notional_limit: string - minNotional + """ + symbols = self._exchange_info(symbol) + market = symbols["symbols"][0] + min_notional_filter = next( + (m for m in market["filters"] if m["filterType"] == "MIN_NOTIONAL") + ) + return min_notional_filter[min_notional_limit] + + def calculate_price_precision(self, symbol) -> int: + precision = -1 * ( + Decimal(str(self.price_filter_by_symbol(symbol, "tickSize"))) + .as_tuple() + .exponent + ) + price_precision = int(precision) + return price_precision + + def calculate_qty_precision(self, symbol) -> int: + precision = -1 * ( + Decimal(str(self.lot_size_by_symbol(symbol, "stepSize"))) + .as_tuple() + .exponent + ) + qty_precision = int(precision) + return qty_precision + + def find_quoteAsset(self, symbol): + """ + e.g. BNBBTC: base asset = BTC + """ + symbols = self._exchange_info(symbol) + quote_asset = symbols["symbols"][0] + if quote_asset: + quote_asset = quote_asset["quoteAsset"] + return quote_asset + + def find_baseAsset(self, symbol): + """ + e.g. BNBBTC: base asset = BNB + """ + symbols = self._exchange_info(symbol) + base_asset = symbols["symbols"][0]["baseAsset"] + return base_asset + def ticker(self, symbol: str | None = None, json: bool = True): params = {} if symbol: diff --git a/api/cronjobs.py b/api/cronjobs.py index 19bfcfe3..9e0ba5da 100644 --- a/api/cronjobs.py +++ b/api/cronjobs.py @@ -5,7 +5,7 @@ from account.assets import Assets from charts.controllers import MarketDominationController from database.utils import independent_session - +from database.symbols_crud import SymbolsCrud logging.Formatter.converter = time.gmtime # date time in GMT/UTC logging.basicConfig( @@ -24,14 +24,6 @@ def main(): # Jobs should be distributed as far as possible from each other # to avoid overloading RAM and also avoid hitting rate limits due to high weight - scheduler.add_job( - func=assets.refresh_symbols_table, - trigger="cron", - timezone=timezone, - hour=4, - minute=1, - id="store_balance", - ) scheduler.add_job( func=assets.store_balance, trigger="cron", diff --git a/api/database/api_db.py b/api/database/api_db.py index 624513eb..d20f5f37 100644 --- a/api/database/api_db.py +++ b/api/database/api_db.py @@ -19,6 +19,7 @@ from alembic import command from database.utils import engine from account.assets import Assets +from database.symbols_crud import SymbolsCrud class ApiDb: @@ -29,6 +30,7 @@ class ApiDb: def __init__(self): self.session = Session(engine) self.assets_collection = Assets(self.session) + self.symbols = SymbolsCrud(self.session) pass def init_db(self): @@ -37,7 +39,7 @@ def init_db(self): self.init_users() self.create_dummy_bot() self.init_autotrade_settings() - self.assets_collection.refresh_symbols_table() + self.symbols.refresh_symbols_table() self.assets_collection.store_balance() self.session.close() logging.info("Finishing db operations") diff --git a/api/database/models/symbol_table.py b/api/database/models/symbol_table.py index 0a36ea66..d6038456 100644 --- a/api/database/models/symbol_table.py +++ b/api/database/models/symbol_table.py @@ -15,3 +15,16 @@ class SymbolTable(SQLModel, table=True): ) active: bool = Field(default=True, description="Blacklisted items = false") blacklist_reason: str = Field(default="") + quote_asset: str = Field( + default="", description="in BTCUSDC, BTC would be quote asset" + ) + base_asset: str = Field( + default="", description="in BTCUSDC, USDC would be base asset" + ) + price_precision: int = Field( + default=0, + description="Usually there are 2 price precisions, one for base and another for quote, here we usually indicate quote, since we always use the same base: USDC", + ) + qty_precision: int = Field(default=0) + min_qty: float = Field(default=0, description="Minimum qty (quote asset) value") + min_notional: float = Field(default=0, description="Minimum price x qty value") diff --git a/api/database/symbols_crud.py b/api/database/symbols_crud.py index 126d1906..1f3fb68c 100644 --- a/api/database/symbols_crud.py +++ b/api/database/symbols_crud.py @@ -3,11 +3,12 @@ from database.models.symbol_table import SymbolTable from typing import Optional from tools.exceptions import BinbotErrors +from apis import BinanceApi class SymbolsCrud: """ - Database operations for Autotrade settings + Database operations for SymbolTable """ def __init__( @@ -43,26 +44,65 @@ def get_symbol(self, symbol: str) -> SymbolTable: else: raise BinbotErrors("Symbol not found") - def add_symbol(self, symbol: str, active: bool = False, reason: Optional[str] = ""): + def add_symbol( + self, + symbol: str, + active: bool = True, + reason: Optional[str] = "", + price_precision: int = 0, + qty_precision: int = 0, + min_qty: float = 0, + min_notional: float = 0, + ): """ Add a new blacklisted item """ - blacklist = SymbolTable(id=symbol, blacklist_reason=reason, active=active) - self.session.add(blacklist) + symbol = SymbolTable( + id=symbol, + blacklist_reason=reason, + active=active, + price_precision=price_precision, + qty_precision=qty_precision, + min_qty=min_qty, + min_notional=min_notional, + ) + self.session.add(symbol) self.session.commit() - self.session.refresh(blacklist) + self.session.refresh(symbol) self.session.close() - return blacklist + return symbol - def edit_symbol_item(self, symbol: str, active: bool, reason: Optional[str] = None): + def edit_symbol_item( + self, + symbol: str, + active: bool, + reason: Optional[str] = None, + price_precision: int = 0, + qty_precision: int = 0, + min_qty: float = 0, + min_notional: float = 0, + ): """ Edit a blacklisted item """ symbol_model = self.get_symbol(symbol) symbol_model.active = active + if reason: symbol_model.blacklist_reason = reason + if price_precision > 0: + symbol_model.price_precision = price_precision + + if qty_precision > 0: + symbol_model.qty_precision = qty_precision + + if min_qty > 0: + symbol_model.min_qty = min_qty + + if min_notional > 0: + symbol_model.min_notional = min_notional + self.session.add(symbol_model) self.session.commit() self.session.refresh(symbol_model) @@ -78,3 +118,48 @@ def delete_symbol(self, symbol: str): self.session.commit() self.session.close() return symbol_model + + def refresh_symbols_table(self): + """ + Refresh the symbols table + + Uses ticker instead of exchange_info + because weight considerably lower + """ + binance_api = BinanceApi() + data = binance_api._exchange_info()["symbols"] + + for item in data: + if item["status"] != "TRADING": + continue + try: + symbol = self.get_symbol(item["symbol"]) + except BinbotErrors: + symbol = None + pass + + # Only store fiat market, exclude other fiats. + if ( + item["symbol"].endswith("USDC") + and not symbol + and not item["symbol"].startswith( + ("DOWN", "UP", "AUD", "USDT", "EUR", "GBP") + ) + ): + price_precision = binance_api.calculate_price_precision(item["symbol"]) + qty_precision = binance_api.calculate_qty_precision(item["symbol"]) + min_notional = binance_api.min_notional_by_symbol(item["symbol"]) + min_qty = binance_api.lot_size_by_symbol(item["symbol"]) + active = True + + if item["symbol"] == "BTCUSDC": + active = False + + self.add_symbol( + item["symbol"], + active=active, + price_precision=price_precision, + qty_precision=qty_precision, + min_qty=min_qty, + min_notional=min_notional, + ) diff --git a/api/symbols/routes.py b/api/symbols/routes.py index 0b05e667..87c6ed60 100644 --- a/api/symbols/routes.py +++ b/api/symbols/routes.py @@ -1,81 +1,86 @@ -from fastapi import APIRouter +from fastapi import APIRouter, Depends from database.symbols_crud import SymbolsCrud from symbols.models import SymbolsResponse, GetOneSymbolResponse -from apis import BinanceApi -from tools.exceptions import BinbotErrors +from database.utils import get_session +from sqlmodel import Session +from tools.handle_error import StandardResponse, BinbotErrors research_blueprint = APIRouter() @research_blueprint.get("/", response_model=SymbolsResponse, tags=["Symbols"]) -def get_all_symbols(): +def get_all_symbols(session: Session = Depends(get_session)): """ Get all active/not blacklisted symbols/pairs """ - data = SymbolsCrud().get_all(active=True) + data = SymbolsCrud(session=session).get_all(active=True) return SymbolsResponse(message="Successfully retrieved blacklist", data=data) @research_blueprint.post("/", response_model=GetOneSymbolResponse, tags=["Symbols"]) -def add_symbol(symbol: str, reason: str = "", active: bool = True): +def add_symbol( + symbol: str, + reason: str = "", + active: bool = True, + session: Session = Depends(get_session), +): """ Create a new symbol/pair. If active=False, the pair is blacklisted """ - data = SymbolsCrud().add_symbol(symbol=symbol, reason=reason, active=active) + data = SymbolsCrud(session=session).add_symbol( + symbol=symbol, reason=reason, active=active + ) return GetOneSymbolResponse(message="Symbols found!", data=data) @research_blueprint.delete( "/{pair}", response_model=GetOneSymbolResponse, tags=["Symbols"] ) -def delete_symbol(pair: str): +def delete_symbol(pair: str, session: Session = Depends(get_session)): """ Given symbol/pair, delete a symbol Should not be used often. If need to blacklist, simply set active=False """ - data = SymbolsCrud().delete_symbol(pair) + data = SymbolsCrud(session=session).delete_symbol(pair) return GetOneSymbolResponse(message="Symbol deleted", data=data) @research_blueprint.put("/", response_model=GetOneSymbolResponse, tags=["Symbols"]) -def edit_symbol(symbol, active: bool = True, reason: str = ""): +def edit_symbol( + symbol, + active: bool = True, + reason: str = "", + session: Session = Depends(get_session), +): """ Modify a blacklisted item """ - data = SymbolsCrud().edit_symbol_item(symbol=symbol, active=active, reason=reason) + data = SymbolsCrud(session=session).edit_symbol_item( + symbol=symbol, active=active, reason=reason + ) return GetOneSymbolResponse(message="Symbol edited", data=data) @research_blueprint.get("/blacklist", response_model=SymbolsResponse, tags=["Symbols"]) -def get_blacklisted_symbols(): +def get_blacklisted_symbols(session: Session = Depends(get_session)): """ Get all symbols/pairs blacklisted """ - data = SymbolsCrud().get_all(active=False) + data = SymbolsCrud(session=session).get_all(active=False) return SymbolsResponse(message="Successfully retrieved blacklist", data=data) @research_blueprint.get("/store", tags=["Symbols"]) -def store_symbols(): +def store_symbols(session: Session = Depends(get_session)): """ Store all symbols from Binance """ - b_api = BinanceApi() - data = b_api.ticker(json=False) - symbol_controller = SymbolsCrud() - - for item in data: - try: - symbol = symbol_controller.get_symbol(item["symbol"]) - except BinbotErrors: - symbol = None - pass - - if item["symbol"].endswith("USDC") and not symbol: - symbol_controller.add_symbol(item["symbol"]) - - return GetOneSymbolResponse(message="Symbols stored!") + try: + SymbolsCrud(session=session).refresh_symbols_table() + return GetOneSymbolResponse(message="Symbols stored!") + except BinbotErrors as e: + return StandardResponse(message=str(e), error=1) diff --git a/binquant b/binquant index 618d9899..964934d6 160000 --- a/binquant +++ b/binquant @@ -1 +1 @@ -Subproject commit 618d9899fed4af07961a15526b04ccc9bec496da +Subproject commit 964934d65fa1ca572d51a2e0621864e826b1b8d9