diff --git a/pragma/publisher/fetchers/binance.py b/pragma/publisher/fetchers/binance.py index fbf472a3..bf332b4b 100644 --- a/pragma/publisher/fetchers/binance.py +++ b/pragma/publisher/fetchers/binance.py @@ -28,45 +28,20 @@ async def _fetch_pair( self, asset: PragmaSpotAsset, session: ClientSession ) -> Union[SpotEntry, PublisherFetchError]: pair = asset["pair"] - url = f"{self.BASE_URL}?symbol={pair[0]}{pair[1]}" + + # For now still leaving this line, if pair == ("STRK", "USD"): pair = ("STRK", "USDT") - if pair == ("ETH", "STRK"): - url = f"{self.BASE_URL}?symbol=STRKUSDT" - async with session.get(url) as resp: - if resp.status == 404: - return PublisherFetchError( - f"No data found for {'/'.join(pair)} from Binance" - ) - result = await resp.json() - if "code" in result: - return PublisherFetchError( - f"No data found for {'/'.join(pair)} from Binance" - ) - eth_url = f"{self.BASE_URL}?symbol=ETHUSDT" - eth_resp = await session.get(eth_url) - eth_result = await eth_resp.json() - return self._construct( - asset, - result, - ((float(eth_result["bidPrice"]) + float(eth_result["askPrice"]))) - / 2, + url = self.format_url(pair[0], pair[1]) + async with session.get(url) as resp: + if resp.status == 404: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from Binance" ) - else: - async with session.get(url) as resp: - if resp.status == 404: - return PublisherFetchError( - f"No data found for {'/'.join(pair)} from Binance" - ) - result = await resp.json() - if "code" in result: - return PublisherFetchError( - f"No data found for {'/'.join(pair)} from Binance" - ) - eth_url = f"{self.BASE_URL}?symbol=ETHUSDT" - eth_resp = await session.get(eth_url) - eth_result = await eth_resp.json() - return self._construct(asset, result) + result = await resp.json() + if "code" in result: + return await self.operate_usdt_hop(asset, session) + return self._construct(asset, result) def _fetch_pair_sync( self, asset: PragmaSpotAsset @@ -74,39 +49,16 @@ def _fetch_pair_sync( pair = asset["pair"] if pair == ("STRK", "USD"): pair = ("STRK", "USDT") - if pair == ("ETH", "STRK"): - url = f"{self.BASE_URL}?symbol=STRKUSDT" - resp = requests.get(url) - if resp.status_code == 404: - return PublisherFetchError( - f"No data found for {'/'.join(pair)} from Binance" - ) - result = resp.json() - if "code" in result: - return PublisherFetchError( - f"No data found for {'/'.join(pair)} from Binance" - ) - eth_url = f"{self.BASE_URL}?symbol=ETHUSDT" - eth_resp = requests.get(eth_url) - eth_result = eth_resp.json() - return self._construct( - asset, - result, - (float(eth_result["bidPrice"]) + float(eth_result["askPrice"])) / 2, + url = self.format_url(pair[0], pair[1]) + resp = requests.get(url) + if resp.status_code == 404: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from Binance" ) - else: - url = f"{self.BASE_URL}?symbol={pair[0]}{pair[1]}" - resp = requests.get(url) - if resp.status_code == 404: - return PublisherFetchError( - f"No data found for {'/'.join(pair)} from Binance" - ) - result = resp.json() - if "code" in result: - return PublisherFetchError( - f"No data found for {'/'.join(pair)} from Binance" - ) - return self._construct(asset, result) + result = resp.json() + if "code" in result: + return self.operate_usdt_hop_sync(asset) + return self._construct(asset, result) async def fetch( self, session: ClientSession @@ -134,18 +86,68 @@ def format_url(self, quote_asset, base_asset): url = f"{self.BASE_URL}?symbol={quote_asset}{base_asset}" return url - def _construct(self, asset, result, eth_price=None) -> SpotEntry: + async def operate_usdt_hop(self, asset, session) -> SpotEntry: pair = asset["pair"] + url_pair1 = self.format_url(asset["pair"][0], "USDT") + async with session.get(url_pair1) as resp: + if resp.status == 404: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from Binance - hop failed for {pair[0]}" + ) + pair1_usdt = await resp.json() + if "code" in pair1_usdt: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from Binance - hop failed for {pair[0]}" + ) + url_pair2 = self.format_url(asset["pair"][1], "USDT") + async with session.get(url_pair2) as resp: + if resp.status == 404: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from Binance - hop failed for {pair[1]}" + ) + pair2_usdt = await resp.json() + if "code" in pair2_usdt: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from Binance - hop failed for {pair[1]}" + ) + return self._construct(asset, pair2_usdt, pair1_usdt) + + def operate_usdt_hop_sync(self, asset) -> float: + pair = asset["pair"] + url_pair1 = self.format_url(asset["pair"][0], "USDT") + resp = requests.get(url_pair1) + if resp.status_code == 404: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from Binance - hop failed for {pair[0]}" + ) + pair1_usdt = resp.json() + if "code" in pair1_usdt: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from Binance - hop failed for {pair[0]}" + ) + url2 = self.format_url(asset["pair"][1], "USDT") + resp2 = requests.get(url2) + if resp2.status_code == 404: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from Binance - hop failed for {pair[1]}" + ) + pair2_usdt = resp2.json() + if "code" in pair2_usdt: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from Binance - hop failed for {pair[1]}" + ) + return self._construct(asset, pair2_usdt, pair1_usdt) - if pair == ("ETH", "STRK"): - bid = float(result["bidPrice"]) - ask = float(result["askPrice"]) - strk_price = (bid + ask) / 2 - price = eth_price / strk_price - else: - bid = float(result["bidPrice"]) - ask = float(result["askPrice"]) - price = (bid + ask) / 2 + def _construct(self, asset, result, hop_result=None) -> SpotEntry: + pair = asset["pair"] + bid = float(result["bidPrice"]) + ask = float(result["askPrice"]) + price = (bid + ask) / 2 + if hop_result is not None: + hop_bid = float(hop_result["bidPrice"]) + hop_ask = float(hop_result["askPrice"]) + hop_price = (hop_bid + hop_ask) / 2 + price = hop_price / price timestamp = int(time.time()) price_int = int(price * (10 ** asset["decimals"])) pair_id = currency_pair_to_pair_id(*pair) diff --git a/pragma/publisher/fetchers/bybit.py b/pragma/publisher/fetchers/bybit.py index 1cd8364e..9b026135 100644 --- a/pragma/publisher/fetchers/bybit.py +++ b/pragma/publisher/fetchers/bybit.py @@ -30,46 +30,16 @@ async def _fetch_pair( pair = asset["pair"] if pair == ("STRK", "USD"): pair = ("STRK", "USDT") - if pair == ("ETH", "STRK"): - url = f"{self.BASE_URL}symbol=STRKUSDT" - async with session.get(url) as resp: - if resp.status == 404: - return PublisherFetchError( - f"No data found for {'/'.join(pair)} from Bybit" - ) - result = await resp.json() - if result["retCode"] == 10001: - return PublisherFetchError( - f"No data found for {'/'.join(pair)} from Bybit" - ) - eth_url = f"{self.BASE_URL}symbol=ETHUSDT" - eth_resp = await session.get(eth_url) - eth_result = await eth_resp.json() - return self._construct( - asset, - result, - ( - ( - float(eth_result["result"]["list"][0]["bid1Price"]) - + float(eth_result["result"]["list"][0]["ask1Price"]) - ) - ) - / 2, + url = self.format_url(pair[0], pair[1]) + async with session.get(url) as resp: + if resp.status == 404: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from Bybit" ) - else: - url = f"{self.BASE_URL}symbol={pair[0]}{pair[1]}" - async with session.get(url) as resp: - if resp.status == 404: - return PublisherFetchError( - f"No data found for {'/'.join(pair)} from Bybit" - ) - result = await resp.json() - if result["retCode"] == 10001: - return PublisherFetchError( - f"No data found for {'/'.join(pair)} from Bybit" - ) - - return self._construct(asset, result) + result = await resp.json() + if result["retCode"] == 10001: + return await self.operate_usdt_hop(asset, session) + return self._construct(asset, result) def _fetch_pair_sync( self, asset: PragmaSpotAsset @@ -77,47 +47,15 @@ def _fetch_pair_sync( pair = asset["pair"] if pair == ("STRK", "USD"): pair = ("STRK", "USDT") - if pair == ("ETH", "STRK"): - url = f"{self.BASE_URL}symbol=STRKUSDT" - resp = requests.get(url) - if resp.status_code == 404: - return PublisherFetchError( - f"No data found for {'/'.join(pair)} from Bybit" - ) - result = resp.json() - if result["retCode"] == 10001: - return PublisherFetchError( - f"No data found for {'/'.join(pair)} from Bybit" - ) - eth_url = f"{self.BASE_URL}symbol=ETHUSDT" - eth_resp = requests.get(eth_url) - eth_result = eth_resp.json() - return self._construct( - asset, - result, - ( - ( - float(eth_result["result"]["list"][0]["bid1Price"]) - + float(eth_result["result"]["list"][0]["ask1Price"]) - ) - ) - / 2, - ) - else: - url = f"{self.BASE_URL}symbol={pair[0]}{pair[1]}" - resp = requests.get(url) - if resp.status_code == 404: - return PublisherFetchError( - f"No data found for {'/'.join(pair)} from Bybit" - ) - result = resp.json() - if result["retCode"] == 10001: - return PublisherFetchError( - f"No data found for {'/'.join(pair)} from Bybit" - ) - - return self._construct(asset, result) + url = self.format_url(pair[0], pair[1]) + resp = requests.get(url) + if resp.status_code == 404: + return PublisherFetchError(f"No data found for {'/'.join(pair)} from Bybit") + result = resp.json() + if result["retCode"] == 10001: + return self.operate_usdt_hop_sync(asset) + return self._construct(asset, result) async def fetch( self, session: ClientSession @@ -140,27 +78,79 @@ def fetch_sync(self) -> List[Union[SpotEntry, PublisherFetchError]]: return entries def format_url(self, quote_asset, base_asset): - url = f"{self.BASE_URL}?symbol={quote_asset}/{base_asset}" + url = f"{self.BASE_URL}symbol={quote_asset}{base_asset}" return url - def _construct(self, asset, result, eth_price=None) -> SpotEntry: + async def operate_usdt_hop(self, asset, session) -> SpotEntry: pair = asset["pair"] - data = result["result"]["list"] - timestamp = int(time.time()) - if pair == ("ETH", "STRK"): - ask = float(data[0]["ask1Price"]) - bid = float(data[0]["bid1Price"]) - price = (ask + bid) / 2.0 - price_int = int((eth_price / price) * (10 ** asset["decimals"])) - else: - ask = float(data[0]["ask1Price"]) - bid = float(data[0]["bid1Price"]) - price = (ask + bid) / 2.0 - price_int = int(price * (10 ** asset["decimals"])) + url_pair1 = self.format_url(asset["pair"][0], "USDT") + async with session.get(url_pair1) as resp: + if resp.status == 404: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from Bybit - hop failed for {pair[0]}" + ) + pair1_usdt = await resp.json() + if pair1_usdt["retCode"] == 10001: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from Bybit - hop failed for {pair[0]}" + ) + url2 = self.format_url(asset["pair"][1], "USDT") + async with session.get(url2) as resp: + if resp.status == 404: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from Bybit - hop failed for {pair[1]}" + ) + pair2_usdt = await resp.json() + if pair2_usdt["retCode"] == 10001: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from Bybit - hop failed for {pair[1]}" + ) + return self._construct(asset, pair2_usdt, pair1_usdt) + def operate_usdt_hop_sync(self, asset) -> SpotEntry: + pair = asset["pair"] + url_pair1 = self.format_url(asset["pair"][0], "USDT") + resp = requests.get(url_pair1) + if resp.status_code == 404: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from Bybit - hop failed for {pair[0]}" + ) + pair1_usdt = resp.json() + if pair1_usdt["retCode"] == 10001: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from Bybit - hop failed for {pair[0]}" + ) + url_pair2 = self.format_url(asset["pair"][1], "USDT") + resp = requests.get(url_pair2) + if resp.status_code == 404: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from Bybit - hop failed for {pair[1]}" + ) + pair2_usdt = resp.json() + if pair2_usdt["retCode"] == 10001: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from Bybit - hop failed for {pair[1]}" + ) + return self._construct(asset, pair2_usdt, pair1_usdt) + + def _construct(self, asset, result, hop_result=None) -> SpotEntry: + pair = asset["pair"] + bid = float(result["result"]["list"][0]["bid1Price"]) + ask = float(result["result"]["list"][0]["ask1Price"]) + price = (bid + ask) / 2 + if hop_result is not None: + hop_bid = float(hop_result["result"]["list"][0]["bid1Price"]) + hop_ask = float(hop_result["result"]["list"][0]["ask1Price"]) + hop_price = (hop_bid + hop_ask) / 2 + price = hop_price / price + timestamp = int(time.time()) + price_int = int(price * (10 ** asset["decimals"])) pair_id = currency_pair_to_pair_id(*pair) - volume = float(data[0]["volume24h"]) / 10 ** asset["decimals"] + volume = ( + float(result["result"]["list"][0]["volume24h"]) if hop_result is None else 0 + ) logger.info("Fetched price %d for %s from Bybit", price, "/".join(pair)) + return SpotEntry( pair_id=pair_id, price=price_int, diff --git a/pragma/publisher/fetchers/defillama.py b/pragma/publisher/fetchers/defillama.py index 3c87b5e3..1ff10cbf 100644 --- a/pragma/publisher/fetchers/defillama.py +++ b/pragma/publisher/fetchers/defillama.py @@ -69,14 +69,10 @@ async def _fetch_pair( return PublisherFetchError( f"Unknown price pair, do not know how to query Coingecko for {pair[0]}" ) - if pair[1] != "USD" and pair[1] != "STRK": - return PublisherFetchError(f"Base asset not supported : {pair[1]}") - - if pair[1] == "STRK" and pair[0] == "ETH": - pair_id = "starknet" + if pair[1] != "USD": + return await self.operate_usd_hop(asset, session) url = self.BASE_URL.format(pair_id=pair_id) - async with session.get(url, headers=self.headers) as resp: if resp.status == 404: return PublisherFetchError( @@ -88,14 +84,9 @@ async def _fetch_pair( f"No data found for {'/'.join(pair)} from Defillama" ) - async with session.get( - self.BASE_URL.format(pair_id="ethereum"), headers=self.headers - ) as resp2: - eth_result = await resp2.json() - - return self._construct( - asset=asset, result=result, eth_result=eth_result - ) + return self._construct( + asset=asset, result=result + ) def _fetch_pair_sync(self, asset: PragmaSpotAsset) -> SpotEntry: pair = asset["pair"] @@ -105,9 +96,8 @@ def _fetch_pair_sync(self, asset: PragmaSpotAsset) -> SpotEntry: f"Unknown price pair, do not know how to query Coingecko for {pair[0]}" ) if pair[1] != "USD": - return PublisherFetchError(f"Base asset not supported : {pair[1]}") + return self.operate_usd_hop_sync(asset) url = self.BASE_URL.format(pair_id=pair_id) - resp = requests.get(url, headers=self.headers) if resp.status_code == 404: return PublisherFetchError( @@ -142,26 +132,85 @@ def format_url(self, quote_asset, base_asset): pair_id = ASSET_MAPPING.get(quote_asset) url = self.BASE_URL.format(pair_id=pair_id) return url + + async def operate_usd_hop(self, asset, session) -> SpotEntry: + pair = asset["pair"] + pair_id_1 = ASSET_MAPPING.get(pair[0]) + pair_id_2 = ASSET_MAPPING.get(pair[1]) + if pair_id_2 is None: + return PublisherFetchError( + f"Unknown price pair, do not know how to query Coingecko for {pair[1]} - hop failed" + ) + url_pair_1 = self.BASE_URL.format(pair_id=pair_id_1) + async with session.get(url_pair_1, headers=self.headers) as resp: + if resp.status == 404: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from Defillama - hop failed for {pair[0]}" + ) + result_base = await resp.json() + if not result_base["coins"]: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from Defillama - hop failed for {pair[0]}" + ) + url_pair_2 = self.BASE_URL.format(pair_id=pair_id_2) + async with session.get(url_pair_2, headers=self.headers) as resp: + if resp.status == 404: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from Defillama - usd hop failed for {pair[1]}" + ) + result_quote= await resp.json() + print(result_quote) + if not result_quote["coins"]: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from Defillama - usd hop failed for {pair[1]}" + ) + return self._construct(asset, result_base,result_quote) + + def operate_usd_hop_sync(self, asset) -> SpotEntry: + pair = asset["pair"] + pair_id_1 = ASSET_MAPPING.get(pair[0]) + pair_id_2 = ASSET_MAPPING.get(pair[1]) + if pair_id_2 is None: + return PublisherFetchError( + f"Unknown price pair, do not know how to query Coingecko for {pair[1]} - hop failed" + ) + url_pair_1 = self.BASE_URL.format(pair_id=pair_id_1) + resp = requests.get(url_pair_1, headers=self.headers) + if resp.status_code == 404: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from Defillama - hop failed for {pair[0]}" + ) + result_base = resp.json() + if not result_base["coins"]: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from Defillama - hop failed for {pair[0]}" + ) + url_pair_2 = self.BASE_URL.format(pair_id=pair_id_2) + resp = requests.get(url_pair_2, headers=self.headers) + if resp.status_code == 404: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from Defillama - usd hop failed for {pair[1]}" + ) + result_quote = resp.json() + if not result_quote["coins"]: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from Defillama - usd hop failed for {pair[1]}" + ) + return self._construct(asset, result_base,result_quote) - def _construct(self, asset, result, eth_result) -> SpotEntry: + def _construct(self, asset, result, hop_result = None) -> SpotEntry: pair = asset["pair"] - cg_id = ASSET_MAPPING.get(pair[0]) - - if pair[1] == "STRK" and pair[0] == "ETH": - cg_id = "starknet" - pair_id = currency_pair_to_pair_id(*pair) - price = result["coins"][f"coingecko:{cg_id}"]["price"] - strk_usd_int = int(price * (10 ** asset["decimals"])) - timestamp = int(result["coins"][f"coingecko:{cg_id}"]["timestamp"]) - # rebase price - eth_price = eth_result["coins"]["coingecko:ethereum"]["price"] - eth_usd_int = int(eth_price * (10 ** asset["decimals"])) - price_int = int(eth_usd_int / strk_usd_int * 10 ** asset["decimals"]) - else: - pair_id = currency_pair_to_pair_id(*pair) - price = result["coins"][f"coingecko:{cg_id}"]["price"] + base_id= ASSET_MAPPING.get(pair[0]) + quote_id = ASSET_MAPPING.get(pair[1]) + pair_id = currency_pair_to_pair_id(*pair) + timestamp = int(result["coins"][f"coingecko:{base_id}"]["timestamp"]) + if hop_result is not None: + price = result["coins"][f"coingecko:{base_id}"]["price"] + hop_price = hop_result["coins"][f"coingecko:{quote_id}"]["price"] + price_int = int((price / hop_price) * (10 ** asset["decimals"])) + else: + price = result["coins"][f"coingecko:{base_id}"]["price"] price_int = int(price * (10 ** asset["decimals"])) - timestamp = int(result["coins"][f"coingecko:{cg_id}"]["timestamp"]) logger.info("Fetched price %d for %s from Coingecko", price, pair_id) diff --git a/pragma/publisher/fetchers/gecko.py b/pragma/publisher/fetchers/gecko.py index 1ae7232f..aca25b4e 100644 --- a/pragma/publisher/fetchers/gecko.py +++ b/pragma/publisher/fetchers/gecko.py @@ -53,16 +53,9 @@ async def _fetch_pair( self, asset: PragmaSpotAsset, session: ClientSession ) -> SpotEntry: pair = asset["pair"] - if pair[1] != "USD" and pair[1] != "STRK": - return PublisherFetchError(f"Base asset not supported : {pair[1]}") - - if pair[1] == "STRK" and pair[0] == "ETH": - pool = ASSET_MAPPING.get("STRK") - elif pair[0] == "STRK" and pair[1] == "USD": - pool = ASSET_MAPPING.get("STRK") - else: - pool = ASSET_MAPPING.get(pair[0]) - + if pair[1] != "USD" : + return await self.operate_usd_hop(asset, session) + pool = ASSET_MAPPING.get(pair[0]) if pool is None: return PublisherFetchError( f"Unknown price pair, do not know how to query GeckoTerminal for {pair[0]}" @@ -84,24 +77,18 @@ async def _fetch_pair( f"No data found for {'/'.join(pair)} from GeckoTerminal" ) - pool_eth = ASSET_MAPPING.get("ETH") - eth_url = self.BASE_URL.format( - network=pool_eth[0], token_address=pool_eth[1] - ) - async with session.get(eth_url, headers=self.headers) as resp2: - eth_result = await resp2.json() - - return self._construct(asset, result, eth_result) + return self._construct(asset, result) def _fetch_pair_sync(self, asset: PragmaSpotAsset) -> SpotEntry: pair = asset["pair"] + if pair[1] != "USD" : + return self.operate_usd_hop_sync(asset) pool = ASSET_MAPPING.get(pair[0]) if pool is None: return PublisherFetchError( f"Unknown price pair, do not know how to query GeckoTerminal for {pair[0]}" ) - if pair[1] != "USD": - return PublisherFetchError(f"Base asset not supported : {pair[1]}") + url = self.BASE_URL.format(network=pool[0], token_address=pool[1]) resp = requests.get(url, headers=self.headers) @@ -117,8 +104,9 @@ def _fetch_pair_sync(self, asset: PragmaSpotAsset) -> SpotEntry: return PublisherFetchError( f"No data found for {'/'.join(pair)} from GeckoTerminal" ) - return self._construct(asset, result) + return self._construct(asset, result) + async def fetch(self, session: ClientSession) -> List[SpotEntry]: entries = [] for asset in self.assets: @@ -141,19 +129,101 @@ def format_url(self, quote_asset, base_asset): pool = ASSET_MAPPING[quote_asset] url = self.BASE_URL.format(network=pool[0], token_address=pool[1]) return url + + async def operate_usd_hop(self, asset, session) -> SpotEntry: + pair = asset["pair"] + pool_1 = ASSET_MAPPING.get(pair[0]) + pool_2 = ASSET_MAPPING.get(pair[1]) + if pool_1 is None or pool_2 is None: + return PublisherFetchError( + f"Unknown price pair, do not know how to query GeckoTerminal for hop {pair[0]} to {pair[1]}" + ) + pair1_url = self.BASE_URL.format(network=pool_1[0], token_address=pool_1[1]) + + async with session.get(pair1_url, headers=self.headers) as resp: + if resp.status == 404: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from GeckoTerminal" + ) + result = await resp.json() + if ( + result.get("errors") is not None + and result["errors"][0]["title"] == "Not Found" + ): + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from GeckoTerminal" + ) + + pair2_url = self.BASE_URL.format( + network=pool_2[0], token_address=pool_2[1] + ) + async with session.get(pair2_url, headers=self.headers) as resp2: + if resp.status == 404: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from GeckoTerminal" + ) + hop_result = await resp2.json() + if ( + result.get("errors") is not None + and result["errors"][0]["title"] == "Not Found" + ): + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from GeckoTerminal" + ) + return self._construct(asset, hop_result, result) - def _construct(self, asset, result, eth_result) -> SpotEntry: + def operate_usd_hop_sync(self, asset) -> SpotEntry: pair = asset["pair"] - data = result["data"]["attributes"] + pool_1 = ASSET_MAPPING.get(pair[0]) + pool_2 = ASSET_MAPPING.get(pair[1]) + if pool_1 is None or pool_2 is None: + return PublisherFetchError( + f"Unknown price pair, do not know how to query GeckoTerminal for hop {pair[0]} to {pair[1]}" + ) + pair1_url = self.BASE_URL.format(network=pool_1[0], token_address=pool_1[1]) - price = float(data["price_usd"]) - if pair[1] == "STRK" and pair[0] == "ETH": - strk_usd_int = int(price * (10 ** asset["decimals"])) - eth_usd_int = int( - float(eth_result["data"]["attributes"]["price_usd"]) * 10**18 + resp = requests.get(pair1_url, headers=self.headers) + if resp.status_code == 404: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from GeckoTerminal" + ) + result = resp.json() + if ( + result.get("errors") is not None + and result["errors"][0]["title"] == "Not Found" + ): + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from GeckoTerminal" + ) + + pair2_url = self.BASE_URL.format( + network=pool_2[0], token_address=pool_2[1] + ) + resp2 = requests.get(pair2_url, headers=self.headers) + if resp.status_code == 404: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from GeckoTerminal" + ) + hop_result = resp2.json() + if ( + result.get("errors") is not None + and result["errors"][0]["title"] == "Not Found" + ): + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from GeckoTerminal" ) - price_int = int(eth_usd_int / strk_usd_int * 10 ** asset["decimals"]) - else: + + return self._construct(asset, hop_result, result) + + + def _construct(self, asset, result, hop_result = None) -> SpotEntry: + pair = asset["pair"] + data = result["data"]["attributes"] + price = float(data["price_usd"]) + if hop_result is not None: + hop_price = float(hop_result["data"]["attributes"]["price_usd"]) + price_int = int(hop_price / price * 10 ** asset["decimals"]) + else: price_int = int(price * (10 ** asset["decimals"])) volume = float(data["volume_usd"]["h24"]) / 10 ** asset["decimals"] @@ -171,3 +241,5 @@ def _construct(self, asset, result, eth_result) -> SpotEntry: publisher=self.publisher, volume=volume, ) + + diff --git a/pragma/publisher/fetchers/huobi.py b/pragma/publisher/fetchers/huobi.py index 950d4be3..3aa84595 100644 --- a/pragma/publisher/fetchers/huobi.py +++ b/pragma/publisher/fetchers/huobi.py @@ -30,45 +30,16 @@ async def _fetch_pair( pair = asset["pair"] if pair == ("STRK", "USD"): pair = ("STRK", "USDT") - if pair == ("ETH", "STRK"): - url = f"{self.BASE_URL}?symbol=strkusdt" - async with session.get(url) as resp: - if resp.status == 404: - return PublisherFetchError( - f"No data found for {'/'.join(pair)} from Huobi" - ) - result = await resp.json() - if result["status"] != "ok": - return PublisherFetchError( - f"No data found for {'/'.join(pair)} from Huobi" - ) - eth_url = f"{self.BASE_URL}?symbol=ethusdt" - eth_resp = await session.get(eth_url) - eth_result = await eth_resp.json() - return self._construct( - asset, - result, - ( - ( - float(eth_result["tick"]["ask"][0]) - + float(eth_result["tick"]["bid"][0]) - ) - ) - / 2, + url = self.format_url(pair[0], pair[1]) + async with session.get(url) as resp: + if resp.status == 404: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from Huobi" ) - else: - url = f"{self.BASE_URL}?symbol={pair[0].lower()}{pair[1].lower()}" - async with session.get(url) as resp: - if resp.status == 404: - return PublisherFetchError( - f"No data found for {'/'.join(pair)} from Huobi" - ) - result = await resp.json() - if result["status"] != "ok": - return PublisherFetchError( - f"No data found for {'/'.join(pair)} from Huobi" - ) - return self._construct(asset, result) + result = await resp.json() + if result["status"] != "ok": + return await self.operate_usdt_hop(asset, session) + return self._construct(asset, result) def _fetch_pair_sync( self, asset: PragmaSpotAsset @@ -76,45 +47,14 @@ def _fetch_pair_sync( pair = asset["pair"] if pair == ("STRK", "USD"): pair = ("STRK", "USDT") - if pair == ("ETH", "STRK"): - url = f"{self.BASE_URL}?symbol=strkusdt" - resp = requests.get(url) - if resp.status_code == 404: - return PublisherFetchError( - f"No data found for {'/'.join(pair)} from Huobi" - ) - result = resp.json() - if result["status"] != "ok": - return PublisherFetchError( - f"No data found for {'/'.join(pair)} from Huobi" - ) - eth_url = f"{self.BASE_URL}?symbol=ethusdt" - eth_resp = requests.get(eth_url) - eth_result = eth_resp.json() - return self._construct( - asset, - result, - ( - ( - float(eth_result["tick"]["ask"][0]) - + float(eth_result["tick"]["bid"][0]) - ) - ) - / 2, - ) - else: - url = f"{self.BASE_URL}?symbol={pair[0].lower()}{pair[1].lower()}" - resp = requests.get(url) - if resp.status_code == 404: - return PublisherFetchError( - f"No data found for {'/'.join(pair)} from Huobi" - ) - result = resp.json() - if result["status"] != "ok": - return PublisherFetchError( - f"No data found for {'/'.join(pair)} from Huobi" - ) - return self._construct(asset, result) + url = self.format_url(pair[0], pair[1]) + resp = requests.get(url) + if resp.status_code == 404: + return PublisherFetchError(f"No data found for {'/'.join(pair)} from Huobi") + result = resp.json() + if result["status"] != "ok": + return self.operate_usdt_hop_sync(asset) + return self._construct(asset, result) async def fetch( self, session: ClientSession @@ -139,29 +79,76 @@ def fetch_sync(self) -> List[Union[SpotEntry, PublisherFetchError]]: return entries def format_url(self, quote_asset, base_asset): - url = f"{self.BASE_URL}?symbol={quote_asset}/{base_asset}" + url = f"{self.BASE_URL}?symbol={quote_asset.lower()}{base_asset.lower()}" return url - def _construct(self, asset, result, eth_price=None) -> SpotEntry: + async def operate_usdt_hop(self, asset, session) -> SpotEntry: + pair = asset["pair"] + url_pair1 = self.format_url(asset["pair"][0], "USDT") + async with session.get(url_pair1) as resp: + if resp.status == 404: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from Huobi - hop failed for {pair[0]}" + ) + pair1_usdt = await resp.json() + if pair1_usdt["status"] != "ok": + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from Huobi - hop failed for {pair[0]}" + ) + url_pair2 = self.format_url(asset["pair"][1], "USDT") + async with session.get(url_pair2) as resp: + if resp.status == 404: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from Huobi - hop failed for {pair[1]}" + ) + pair2_usdt = await resp.json() + if pair2_usdt["status"] != "ok": + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from Huobi - hop failed for {pair[1]}" + ) + return self._construct(asset, pair2_usdt, pair1_usdt) + + def operate_usdt_hop_sync(self, asset) -> SpotEntry: pair = asset["pair"] - data = result["tick"] - - if pair == ("ETH", "STRK"): - ask = float(data["ask"][0]) - bid = float(data["bid"][0]) - price = (ask + bid) / 2.0 - price_int = int((eth_price / price) * (10 ** asset["decimals"])) - else: - ask = float(data["ask"][0]) - bid = float(data["bid"][0]) - price = (ask + bid) / 2.0 - price_int = int(price * (10 ** asset["decimals"])) + url_pair1 = self.format_url(asset["pair"][0], "USDT") + resp = requests.get(url_pair1) + if resp.status_code == 404: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from Huobi - hop failed for {pair[0]}" + ) + pair1_usdt = resp.json() + if pair1_usdt["status"] != "ok": + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from Huobi - hop failed for {pair[0]}" + ) + url_pair2 = self.format_url(asset["pair"][1], "USDT") + resp = requests.get(url_pair2) + if resp.status_code == 404: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from Huobi - hop failed for {pair[1]}" + ) + pair2_usdt = resp.json() + if pair2_usdt["status"] != "ok": + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from Huobi - hop failed for {pair[1]}" + ) + return self._construct(asset, pair2_usdt, pair1_usdt) + def _construct(self, asset, result, hop_result=None) -> SpotEntry: + pair = asset["pair"] + bid = float(result["tick"]["bid"][0]) + ask = float(result["tick"]["ask"][0]) + price = (bid + ask) / 2 + if hop_result is not None: + hop_bid = float(hop_result["tick"]["bid"][0]) + hop_ask = float(hop_result["tick"]["ask"][0]) + hop_price = (hop_bid + hop_ask) / 2 + price = hop_price / price timestamp = int(result["ts"] / 1000) + price_int = int(price * (10 ** asset["decimals"])) pair_id = currency_pair_to_pair_id(*pair) - volume = float(data["vol"]) - - logger.info("Fetched price %d for %s from Huobi", price, "/".join(pair)) + volume = float(result["tick"]["vol"]) if hop_result is None else 0 + logger.info("Fetched price %d for %s from Bybit", price, "/".join(pair)) return SpotEntry( pair_id=pair_id, diff --git a/pragma/publisher/fetchers/kucoin.py b/pragma/publisher/fetchers/kucoin.py index 77c8ee7d..ce71dc41 100644 --- a/pragma/publisher/fetchers/kucoin.py +++ b/pragma/publisher/fetchers/kucoin.py @@ -30,47 +30,16 @@ async def _fetch_pair( pair = asset["pair"] if pair == ("STRK", "USD"): pair = ("STRK", "USDT") - if pair == ("ETH", "STRK"): - url = f"{self.BASE_URL}?symbol=STRK-USDT" - async with session.get(url) as resp: - if resp.status == 404: - return PublisherFetchError( - f"No data found for {'/'.join(pair)} from Kucoin" - ) - result = await resp.json() - if result["data"] == None: - return PublisherFetchError( - f"No data found for {'/'.join(pair)} from Kucoin" - ) - - eth_url = f"{self.BASE_URL}?symbol=ETH-USDT" - eth_resp = await session.get(eth_url) - eth_result = await eth_resp.json() - return self._construct( - asset, - result, - ( - ( - float(eth_result["data"]["bestAsk"]) - + float(eth_result["data"]["bestBid"]) - ) - / 2 - ), + url = self.format_url(pair[0], pair[1]) + async with session.get(url) as resp: + if resp.status == 404: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from Kucoin" ) - else: - url = f"{self.BASE_URL}?symbol={pair[0]}-{pair[1]}" - async with session.get(url) as resp: - if resp.status == 404: - return PublisherFetchError( - f"No data found for {'/'.join(pair)} from Kucoin" - ) - result = await resp.json() - if result["data"] == None: - return PublisherFetchError( - f"No data found for {'/'.join(pair)} from Kucoin" - ) - - return self._construct(asset, result) + result = await resp.json() + if result["data"] == None: + return await self.operate_usdt_hop(asset, session) + return self._construct(asset, result) def _fetch_pair_sync( self, asset: PragmaSpotAsset @@ -78,47 +47,16 @@ def _fetch_pair_sync( pair = asset["pair"] if pair == ("STRK", "USD"): pair = ("STRK", "USDT") - if pair == ("ETH", "STRK"): - url = f"{self.BASE_URL}?symbol=STRK-USDT" - resp = requests.get(url) - if resp.status_code == 404: - return PublisherFetchError( - f"No data found for {'/'.join(pair)} from Kucoin" - ) - result = resp.json() - if result["data"] == None: - return PublisherFetchError( - f"No data found for {'/'.join(pair)} from Kucoin" - ) - - eth_url = f"{self.BASE_URL}?symbol=ETH-USDT" - eth_resp = requests.get(eth_url) - eth_result = eth_resp.json() - return self._construct( - asset, - result, - ( - ( - float(eth_result["data"]["bestAsk"]) - + float(eth_result["data"]["bestBid"]) - ) - / 2 - ), + url = self.format_url(pair[0], pair[1]) + resp = requests.get(url) + if resp.status_code == 404: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from Kucoin" ) - else: - url = f"{self.BASE_URL}?symbol={pair[0]}-{pair[1]}" - resp = requests.get(url) - if resp.status_code == 404: - return PublisherFetchError( - f"No data found for {'/'.join(pair)} from Kucoin" - ) - result = resp.json() - if result["data"] == None: - return PublisherFetchError( - f"No data found for {'/'.join(pair)} from Kucoin" - ) - - return self._construct(asset, result) + result = resp.json() + if result["data"] == None: + return self.operate_usdt_hop_sync(asset) + return self._construct(asset, result) async def fetch( self, session: ClientSession @@ -141,24 +79,70 @@ def fetch_sync(self) -> List[Union[SpotEntry, PublisherFetchError]]: return entries def format_url(self, quote_asset, base_asset): - url = f"{self.BASE_URL}?symbol={quote_asset}/{base_asset}" + url = f"{self.BASE_URL}?symbol={quote_asset}-{base_asset}" return url - def _construct(self, asset, result, eth_price=None) -> SpotEntry: + async def operate_usdt_hop(self, asset, session) -> SpotEntry: pair = asset["pair"] - data = result["data"] + url_pair1 = self.format_url(asset["pair"][0], "USDT") + async with session.get(url_pair1) as resp: + if resp.status == 404: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from Kucoin - hop failed for {pair[0]}" + ) + pair1_usdt = await resp.json() + if pair1_usdt["data"] == None: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from Kucoin - hop failed for {pair[0]}" + ) + url_pair2 = self.format_url(asset["pair"][1], "USDT") + async with session.get(url_pair2) as resp: + if resp.status == 404: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from Kucoin - hop failed for {pair[1]}" + ) + pair2_usdt = await resp.json() + if pair2_usdt["data"] == None: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from Kucoin - hop failed for {pair[1]}" + ) + return self._construct(asset, pair2_usdt, pair1_usdt) - if pair == ("ETH", "STRK"): - price = eth_price / float(data["price"]) - price_int = int(price * (10 ** asset["decimals"])) - timestamp = int(data["time"] / 1000) - else: - price = float(data["price"]) - price_int = int(price * (10 ** asset["decimals"])) + def operate_usdt_hop_sync(self, asset) -> SpotEntry: + pair = asset["pair"] + url_pair1 = self.format_url(asset["pair"][0], "USDT") + resp = requests.get(url_pair1) + if resp.status_code == 404: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from Kucoin - hop failed for {pair[0]}" + ) + pair1_usdt = resp.json() + if pair1_usdt["data"] == None: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from Kucoin - hop failed for {pair[0]}" + ) + url_pair2 = self.format_url(asset["pair"][1], "USDT") + resp = requests.get(url_pair2) + if resp.status_code == 404: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from Kucoin - hop failed for {pair[1]}" + ) + pair2_usdt = resp.json() + if pair2_usdt["data"] == None: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from Kucoin - hop failed for {pair[1]}" + ) + return self._construct(asset, pair2_usdt, pair1_usdt) - timestamp = int(data["time"] / 1000) + def _construct(self, asset, result, hop_result=None) -> SpotEntry: + pair = asset["pair"] + price = float(result["data"]["price"]) + if hop_result is not None: + hop_price = float(hop_result["data"]["price"]) + price = hop_price / price + timestamp = int(result["data"]["time"] / 1000) + price_int = int(price * (10 ** asset["decimals"])) pair_id = currency_pair_to_pair_id(*pair) - logger.info("Fetched price %d for %s from Kucoin", price, "/".join(pair)) return SpotEntry( diff --git a/stagecoach/jobs/publishers/starknet_publisher/app.py b/stagecoach/jobs/publishers/starknet_publisher/app.py index 9641dc27..dd76fcf8 100644 --- a/stagecoach/jobs/publishers/starknet_publisher/app.py +++ b/stagecoach/jobs/publishers/starknet_publisher/app.py @@ -19,8 +19,8 @@ GeckoTerminalFetcher, KaikoFetcher, OkxFetcher, - StarknetAMMFetcher, PropellerFetcher, + StarknetAMMFetcher, ) from pragma.publisher.future_fetchers import ( BinanceFutureFetcher, diff --git a/stagecoach/jobs/randomness/app.py b/stagecoach/jobs/randomness/app.py index 18e996cb..9ed07e84 100644 --- a/stagecoach/jobs/randomness/app.py +++ b/stagecoach/jobs/randomness/app.py @@ -1,6 +1,7 @@ import asyncio import json import os + import boto3 from pragma.core.client import PragmaClient