From 5a1e65d7870efb59b26ebb313ad2302fab6c1655 Mon Sep 17 00:00:00 2001 From: azurwastaken Date: Tue, 18 Jun 2024 18:13:57 +0200 Subject: [PATCH] feat: gate io and mexc fetchers (#118) * added gate io and mexc * fix CI * added volume for both, fixed pair name and error case * fix lint * Update __init__.py * Update app.py * bump 1.4.1 --------- Co-authored-by: 0xevolve --- .github/workflows/checks.yml | 2 +- pragma/core/assets.py | 1 + pragma/publisher/fetchers/__init__.py | 4 + pragma/publisher/fetchers/gateio.py | 120 ++++++++++++++++++ pragma/publisher/fetchers/mexc.py | 120 ++++++++++++++++++ pyproject.toml | 3 +- .../jobs/publishers/starknet_publisher/app.py | 4 + 7 files changed, 252 insertions(+), 2 deletions(-) create mode 100644 pragma/publisher/fetchers/gateio.py create mode 100644 pragma/publisher/fetchers/mexc.py diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 9cdeaef3..c0f9c8b4 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -47,7 +47,7 @@ jobs: - name: Format run: | - poetry run poe format + poetry run poe format_check - name: Typecheck run: | diff --git a/pragma/core/assets.py b/pragma/core/assets.py index e1a01377..87656728 100644 --- a/pragma/core/assets.py +++ b/pragma/core/assets.py @@ -50,6 +50,7 @@ class PragmaOnchainAsset(TypedDict): {"type": "SPOT", "pair": ("USDT", "USD"), "decimals": 6}, {"type": "SPOT", "pair": ("USDC", "USD"), "decimals": 6}, {"type": "SPOT", "pair": ("MATIC", "USD"), "decimals": 8}, + {"type": "SPOT", "pair": ("NSTR", "USD"), "decimals": 8}, {"type": "SPOT", "pair": ("LORDS", "USD"), "decimals": 8}, {"type": "SPOT", "pair": ("ETH", "USDC"), "decimals": 6}, {"type": "SPOT", "pair": ("DAI", "USDC"), "decimals": 6}, diff --git a/pragma/publisher/fetchers/__init__.py b/pragma/publisher/fetchers/__init__.py index 8c8b7843..c8621089 100644 --- a/pragma/publisher/fetchers/__init__.py +++ b/pragma/publisher/fetchers/__init__.py @@ -16,6 +16,8 @@ from .propeller import PropellerFetcher from .starknetamm import StarknetAMMFetcher from .thegraph import TheGraphFetcher +from .mexc import MEXCFetcher +from .gateio import GateioFetcher __all__ = [ AscendexFetcher, @@ -36,4 +38,6 @@ PropellerFetcher, StarknetAMMFetcher, TheGraphFetcher, + MEXCFetcher, + GateioFetcher, ] diff --git a/pragma/publisher/fetchers/gateio.py b/pragma/publisher/fetchers/gateio.py new file mode 100644 index 00000000..fbb6f70e --- /dev/null +++ b/pragma/publisher/fetchers/gateio.py @@ -0,0 +1,120 @@ +import asyncio +import logging +import time +from typing import List, Union + +from aiohttp import ClientSession + +from pragma.core.assets import PragmaAsset, PragmaSpotAsset +from pragma.core.client import PragmaClient +from pragma.core.entry import SpotEntry +from pragma.core.utils import currency_pair_to_pair_id +from pragma.publisher.types import PublisherFetchError, PublisherInterfaceT + +logger = logging.getLogger(__name__) + + +class GateioFetcher(PublisherInterfaceT): + BASE_URL: str = "https://api.gateio.ws/api/v4/spot/tickers" + SOURCE: str = "GATEIO" + publisher: str + + def __init__(self, assets: List[PragmaAsset], publisher, client=None): + self.assets = assets + self.publisher = publisher + self.client = client or PragmaClient(network="mainnet") + + async def fetch_pair( + self, asset: PragmaSpotAsset, session: ClientSession, usdt_price=1 + ) -> Union[SpotEntry, PublisherFetchError]: + pair = asset["pair"] + + # For now still leaving this line, + if pair[1] == "USD": + pair = (pair[0], "USDT") + if pair[0] == "WETH": + pair = ("ETH", pair[1]) + else: + usdt_price = 1 + url = self.format_url(pair[0], pair[1]) + async with session.get(url) as resp: + if resp.status == 404: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from GATEIO" + ) + result = await resp.json() + if resp.status == 400: + return await self.operate_usdt_hop(asset, session) + return self._construct(asset=asset, result=result, usdt_price=usdt_price) + + async def fetch( + self, session: ClientSession + ) -> List[Union[SpotEntry, PublisherFetchError]]: + entries = [] + usdt_price = await self.get_stable_price("USDT") + for asset in self.assets: + if asset["type"] == "SPOT": + entries.append( + asyncio.ensure_future(self.fetch_pair(asset, session, usdt_price)) + ) + else: + logger.debug("Skipping Gate.io for non-spot asset %s", asset) + continue + return await asyncio.gather(*entries, return_exceptions=True) + + def format_url(self, quote_asset, base_asset): + url = f"{self.BASE_URL}?currency_pair={quote_asset}_{base_asset}" + return url + + async def operate_usdt_hop(self, asset, session) -> SpotEntry: + pair = asset["pair"] + url_pair1 = self.format_url(asset["pair"][0], "USDT") + async with session.get(url_pair1) as resp: + if resp.status == 404: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from Gate.io - hop failed for {pair[0]}" + ) + pair1_usdt = await resp.json() + if resp.status == 400: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from Gate.io - hop failed for {pair[0]}" + ) + url_pair2 = self.format_url(asset["pair"][1], "USDT") + async with session.get(url_pair2) as resp: + if resp.status == 404: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from Gate.io - hop failed for {pair[1]}" + ) + pair2_usdt = await resp.json() + if resp.status == 400: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from Gate.io - hop failed for {pair[1]}" + ) + return self._construct(asset=asset, result=pair2_usdt, hop_result=pair1_usdt) + + def _construct(self, asset, result, hop_result=None, usdt_price=1) -> SpotEntry: + pair = asset["pair"] + bid = float(result[0]["highest_bid"]) + ask = float(result[0]["lowest_ask"]) + price = (bid + ask) / (2 * usdt_price) + if hop_result is not None: + hop_bid = float(hop_result[0]["highest_bid"]) + hop_ask = float(hop_result[0]["lowest_ask"]) + hop_price = (hop_bid + hop_ask) / 2 + price = hop_price / price + timestamp = int(time.time()) + volume = float(result[0]["quote_volume"]) if hop_result is None else 0 + price_int = int(price * (10 ** asset["decimals"])) + pair_id = currency_pair_to_pair_id(*pair) + + logger.info("Fetched price %d for %s from Gate.io", price, "/".join(pair)) + + return SpotEntry( + pair_id=pair_id, + price=price_int, + timestamp=timestamp, + source=self.SOURCE, + publisher=self.publisher, + volume=volume, + autoscale_volume=False, + ) diff --git a/pragma/publisher/fetchers/mexc.py b/pragma/publisher/fetchers/mexc.py new file mode 100644 index 00000000..73999a84 --- /dev/null +++ b/pragma/publisher/fetchers/mexc.py @@ -0,0 +1,120 @@ +import asyncio +import logging +import time +from typing import List, Union + +from aiohttp import ClientSession + +from pragma.core.assets import PragmaAsset, PragmaSpotAsset +from pragma.core.client import PragmaClient +from pragma.core.entry import SpotEntry +from pragma.core.utils import currency_pair_to_pair_id +from pragma.publisher.types import PublisherFetchError, PublisherInterfaceT + +logger = logging.getLogger(__name__) + + +class MEXCFetcher(PublisherInterfaceT): + BASE_URL: str = "https://api.mexc.com/api/v3/ticker/24hr" + SOURCE: str = "MEXC" + publisher: str + + def __init__(self, assets: List[PragmaAsset], publisher, client=None): + self.assets = assets + self.publisher = publisher + self.client = client or PragmaClient(network="mainnet") + + async def fetch_pair( + self, asset: PragmaSpotAsset, session: ClientSession, usdt_price=1 + ) -> Union[SpotEntry, PublisherFetchError]: + pair = asset["pair"] + + # For now still leaving this line, + if pair[1] == "USD": + pair = (pair[0], "USDT") + if pair[0] == "WETH": + pair = ("ETH", pair[1]) + else: + usdt_price = 1 + url = self.format_url(pair[0], pair[1]) + async with session.get(url) as resp: + if resp.status == 400: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from MEXC" + ) + result = await resp.json() + if resp.status == 400: + return await self.operate_usdt_hop(asset, session) + return self._construct(asset=asset, result=result, usdt_price=usdt_price) + + async def fetch( + self, session: ClientSession + ) -> List[Union[SpotEntry, PublisherFetchError]]: + entries = [] + usdt_price = await self.get_stable_price("USDT") + for asset in self.assets: + if asset["type"] == "SPOT": + entries.append( + asyncio.ensure_future(self.fetch_pair(asset, session, usdt_price)) + ) + else: + logger.debug("Skipping MEXC for non-spot asset %s", asset) + continue + return await asyncio.gather(*entries, return_exceptions=True) + + def format_url(self, quote_asset, base_asset): + url = f"{self.BASE_URL}?symbol={quote_asset}{base_asset}" + return url + + async def operate_usdt_hop(self, asset, session) -> SpotEntry: + pair = asset["pair"] + url_pair1 = self.format_url(asset["pair"][0], "USDT") + async with session.get(url_pair1) as resp: + if resp.status == 400: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from MEXC - hop failed for {pair[0]}" + ) + pair1_usdt = await resp.json() + if resp.status == 400: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from MEXC - hop failed for {pair[0]}" + ) + url_pair2 = self.format_url(asset["pair"][1], "USDT") + async with session.get(url_pair2) as resp: + if resp.status == 400: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from MEXC - hop failed for {pair[1]}" + ) + pair2_usdt = await resp.json() + if resp.status == 400: + return PublisherFetchError( + f"No data found for {'/'.join(pair)} from MEXC - hop failed for {pair[1]}" + ) + return self._construct(asset=asset, result=pair2_usdt, hop_result=pair1_usdt) + + def _construct(self, asset, result, hop_result=None, usdt_price=1) -> SpotEntry: + pair = asset["pair"] + bid = float(result["bidPrice"]) + ask = float(result["askPrice"]) + price = (bid + ask) / (2 * usdt_price) + if hop_result is not None: + hop_bid = float(hop_result["bidPrice"]) + hop_ask = float(hop_result["askPrice"]) + hop_price = (hop_bid + hop_ask) / 2 + price = hop_price / price + timestamp = int(time.time()) + price_int = int(price * (10 ** asset["decimals"])) + pair_id = currency_pair_to_pair_id(*pair) + volume = float(result["quoteVolume"]) if hop_result is None else 0 + + logger.info("Fetched price %d for %s from MEXC", price, "/".join(pair)) + + return SpotEntry( + pair_id=pair_id, + price=price_int, + timestamp=timestamp, + source=self.SOURCE, + publisher=self.publisher, + volume=volume, + autoscale_volume=False, + ) diff --git a/pyproject.toml b/pyproject.toml index 97c2caff..eb6da37e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pragma-sdk" -version = "1.4.0" +version = "1.4.1" authors = ["0xevolve "] description = "Core package for rollup-native Pragma Oracle" readme = "README.md" @@ -48,6 +48,7 @@ types-deprecated = "^1.2.9" [tool.poe.tasks] format = "ruff format ." +format_check = "ruff format . --check" lint = "ruff check ." lint_fix.shell = "ruff check . --fix" typecheck = "mypy pragma" diff --git a/stagecoach/jobs/publishers/starknet_publisher/app.py b/stagecoach/jobs/publishers/starknet_publisher/app.py index 6c419438..a7c63193 100644 --- a/stagecoach/jobs/publishers/starknet_publisher/app.py +++ b/stagecoach/jobs/publishers/starknet_publisher/app.py @@ -23,6 +23,8 @@ OkxFetcher, PropellerFetcher, StarknetAMMFetcher, + MEXCFetcher, + GateioFetcher, ) from pragma.publisher.future_fetchers import ( BinanceFutureFetcher, @@ -134,6 +136,8 @@ async def _handler(assets): HuobiFetcher, OkxFetcher, BitstampFetcher, + MEXCFetcher, + GateioFetcher, StarknetAMMFetcher, BybitFetcher, BinanceFutureFetcher,