From 2baac660c21775ac5a7c9551ca3ef3520a8a52d9 Mon Sep 17 00:00:00 2001 From: djeck1432 Date: Mon, 18 Nov 2024 22:22:30 +0100 Subject: [PATCH 01/31] add new support functions --- apps/dashboard_app/charts/__init__.py | 2 +- apps/dashboard_app/helpers/load_data.py | 155 ++++++++++++ apps/dashboard_app/helpers/loans_table.py | 107 +++++++++ apps/dashboard_app/helpers/protocol_stats.py | 238 +++++++++++++++++++ 4 files changed, 501 insertions(+), 1 deletion(-) create mode 100644 apps/dashboard_app/helpers/load_data.py create mode 100644 apps/dashboard_app/helpers/loans_table.py create mode 100644 apps/dashboard_app/helpers/protocol_stats.py diff --git a/apps/dashboard_app/charts/__init__.py b/apps/dashboard_app/charts/__init__.py index a5f7e917..058ac177 100644 --- a/apps/dashboard_app/charts/__init__.py +++ b/apps/dashboard_app/charts/__init__.py @@ -1 +1 @@ -from dashboard_app.charts.main import Dashboard +from charts.main import Dashboard diff --git a/apps/dashboard_app/helpers/load_data.py b/apps/dashboard_app/helpers/load_data.py new file mode 100644 index 00000000..8e22c3b1 --- /dev/null +++ b/apps/dashboard_app/helpers/load_data.py @@ -0,0 +1,155 @@ +import asyncio +import itertools +import logging +import math +from time import monotonic + +from data_handler.handlers.loan_states.zklend.events import ZkLendState +from shared.amms import SwapAmm +from shared.constants import TOKEN_SETTINGS + +from dashboard_app.data_conector import DataConnector +from dashboard_app.helpers.loans_table import get_loans_table_data, get_protocol +from dashboard_app.helpers.tools import get_prices + +loger = logging.getLogger(__name__) +data_connector = DataConnector() + + +def init_zklend_state(): + zklend_state = ZkLendState() + start = monotonic() + zklend_data = data_connector.fetch_data(data_connector.ZKLEND_SQL_QUERY) + zklend_data_dict = zklend_data.to_dict(orient="records") + for loan_state in zklend_data_dict: + user_loan_state = zklend_state.loan_entities[loan_state["user"]] + user_loan_state.collateral_enabled.values = loan_state["collateral_enabled"] + user_loan_state.collateral.values = loan_state["collateral"] + user_loan_state.debt.values = loan_state["debt"] + + zklend_state.last_block_number = zklend_data["block"].max() + print(f"Initialized ZkLend state in {monotonic() - start:.2f}s") + + +if __name__ == "__main__": + init_zklend_state() + # TODO: Implement periodic updates + + +def update_data(zklend_state: ZkLendState): + # TODO: parallelize per protocol + # TODO: stream the data, don't wait until we get all events + # nostra_alpha_events = src.nostra_alpha.nostra_alpha_get_events() + # nostra_mainnet_events = src.nostra_mainnet.nostra_mainnet_get_events() + + # Iterate over ordered events to obtain the final state of each user. + + # nostra_alpha_state = src.nostra_alpha.NostraAlphaState() + # for _, nostra_alpha_event in nostra_alpha_events.iterrows(): + # nostra_alpha_state.process_event(event=nostra_alpha_event) + # + # nostra_mainnet_state = src.nostra_mainnet.NostraMainnetState() + # for _, nostra_mainnet_event in nostra_mainnet_events.iterrows(): + # nostra_mainnet_state.process_event(event=nostra_mainnet_event) + # logging.info(f"updated state in {time.time() - t1}s") + + # TODO: move this to state inits above? + # # Collect token parameters. + asyncio.run(zklend_state.collect_token_parameters()) + # TODO move it to separated function + # Get prices of the underlying tokens. + states = [ + zklend_state, + # nostra_alpha_state, + # nostra_mainnet_state, + ] + underlying_addresses_to_decimals = {} + for state in states: + underlying_addresses_to_decimals.update( + { + x.underlying_address: x.decimals + for x in state.token_parameters.collateral.values() + } + ) + underlying_addresses_to_decimals.update( + { + x.underlying_address: x.decimals + for x in state.token_parameters.debt.values() + } + ) + underlying_addresses_to_decimals.update( + {x.address: int(math.log10(x.decimal_factor)) for x in TOKEN_SETTINGS.values()} + ) + prices = get_prices(token_decimals=underlying_addresses_to_decimals) + # TODO: move it to separated function END + swap_amms = SwapAmm() + asyncio.run(swap_amms.init()) + + # for pair, state in itertools.product(PAIRS, states): + # protocol = src.protocol_parameters.get_protocol(state=state) + # logging.info( + # f"Preparing main chart data for protocol = {protocol} and pair = {pair}." + # ) + # # TODO: Decipher `pair` in a smarter way. + # collateral_token_underlying_symbol, debt_token_underlying_symbol = pair.split( + # "-" + # ) + # _ = src.main_chart.get_main_chart_data( + # state=state, + # prices=prices, + # swap_amms=swap_amms, + # collateral_token_underlying_symbol=collateral_token_underlying_symbol, + # debt_token_underlying_symbol=debt_token_underlying_symbol, + # save_data=True, + # ) + # logging.info( + # f"Main chart data for protocol = {protocol} and pair = {pair} prepared in {time.time() - t3}s" + # ) + + loan_stats = {} + for state in states: + protocol = get_protocol(state=state) + loan_stats[protocol] = get_loans_table_data( + state=state, prices=prices, save_data=True + ) + + general_stats = src.protocol_stats.get_general_stats( + states=states, loan_stats=loan_stats, save_data=True + ) + supply_stats = src.protocol_stats.get_supply_stats( + states=states, + prices=prices, + save_data=True, + ) + _ = src.protocol_stats.get_collateral_stats(states=states, save_data=True) + debt_stats = src.protocol_stats.get_debt_stats(states=states, save_data=True) + _ = src.protocol_stats.get_utilization_stats( + general_stats=general_stats, + supply_stats=supply_stats, + debt_stats=debt_stats, + save_data=True, + ) + + max_block_number = zklend_events["block_number"].max() + max_timestamp = zklend_events["timestamp"].max() + last_update = { + "timestamp": str(max_timestamp), + "block_number": str(max_block_number), + } + src.persistent_state.upload_object_as_pickle( + last_update, path=src.persistent_state.LAST_UPDATE_FILENAME + ) + zklend_state.save_loan_entities( + path=src.persistent_state.PERSISTENT_STATE_LOAN_ENTITIES_FILENAME + ) + zklend_state.clear_loan_entities() + src.persistent_state.upload_object_as_pickle( + zklend_state, path=src.persistent_state.PERSISTENT_STATE_FILENAME + ) + loan_entities = pandas.read_parquet( + f"gs://{src.helpers.GS_BUCKET_NAME}/{src.persistent_state.PERSISTENT_STATE_LOAN_ENTITIES_FILENAME}", + engine="fastparquet", + ) + zklend_state.set_loan_entities(loan_entities=loan_entities) + logging.info(f"Updated CSV data in {time.time() - t0}s") + return zklend_state diff --git a/apps/dashboard_app/helpers/loans_table.py b/apps/dashboard_app/helpers/loans_table.py new file mode 100644 index 00000000..dd425a52 --- /dev/null +++ b/apps/dashboard_app/helpers/loans_table.py @@ -0,0 +1,107 @@ +import pandas as pd +from data_handler.handlers.loan_states.nostra_alpha.events import NostraAlphaState +from data_handler.handlers.loan_states.nostra_mainnet.events import NostraMainnetState +from data_handler.handlers.loan_states.zklend.events import ZkLendState +from shared.state import State +from shared.types import Prices + + +def get_protocol(state: State) -> str: + # TODO: Improve the inference. + if isinstance(state, ZkLendState): + return "zkLend" + if isinstance(state, NostraAlphaState) and not isinstance( + state, NostraMainnetState + ): + return "Nostra Alpha" + if isinstance(state, NostraMainnetState): + return "Nostra Mainnet" + raise ValueError + + +def get_loans_table_data( + state: State, + prices: Prices, + save_data: bool = False, +) -> pd.DataFrame: + data = [] + for loan_entity_id, loan_entity in state.loan_entities.items(): + collateral_usd = loan_entity.compute_collateral_usd( + risk_adjusted=False, + collateral_token_parameters=state.token_parameters.collateral, + collateral_interest_rate_model=state.interest_rate_models.collateral, + prices=prices, + ) + risk_adjusted_collateral_usd = loan_entity.compute_collateral_usd( + risk_adjusted=True, + collateral_token_parameters=state.token_parameters.collateral, + collateral_interest_rate_model=state.interest_rate_models.collateral, + prices=prices, + ) + debt_usd = loan_entity.compute_debt_usd( + risk_adjusted=False, + debt_token_parameters=state.token_parameters.debt, + debt_interest_rate_model=state.interest_rate_models.debt, + prices=prices, + ) + if isinstance(state, NostraAlphaState) or isinstance(state, NostraMainnetState): + risk_adjusted_debt_usd = loan_entity.compute_debt_usd( + risk_adjusted=True, + debt_token_parameters=state.token_parameters.debt, + debt_interest_rate_model=state.interest_rate_models.debt, + prices=prices, + ) + health_factor = loan_entity.compute_health_factor( + standardized=False, + risk_adjusted_collateral_usd=risk_adjusted_collateral_usd, + risk_adjusted_debt_usd=risk_adjusted_debt_usd, + ) + standardized_health_factor = loan_entity.compute_health_factor( + standardized=True, + collateral_token_parameters=state.token_parameters.collateral, + risk_adjusted_collateral_usd=risk_adjusted_collateral_usd, + risk_adjusted_debt_usd=risk_adjusted_debt_usd, + ) + else: + health_factor = loan_entity.compute_health_factor( + standardized=False, + risk_adjusted_collateral_usd=risk_adjusted_collateral_usd, + debt_usd=debt_usd, + ) + standardized_health_factor = loan_entity.compute_health_factor( + standardized=True, + risk_adjusted_collateral_usd=risk_adjusted_collateral_usd, + debt_usd=debt_usd, + ) + + data.append( + { + "User": loan_entity_id, + "Protocol": get_protocol(state=state), + "Collateral (USD)": collateral_usd, + "Risk-adjusted collateral (USD)": risk_adjusted_collateral_usd, + "Debt (USD)": debt_usd, + "Health factor": health_factor, + "Standardized health factor": standardized_health_factor, + "Collateral": loan_entity.get_collateral_str( + collateral_token_parameters=state.token_parameters.collateral, + collateral_interest_rate_model=state.interest_rate_models.collateral, + ), + "Debt": loan_entity.get_debt_str( + debt_token_parameters=state.token_parameters.debt, + debt_interest_rate_model=state.interest_rate_models.debt, + ), + } + ) + return pd.DataFrame(data) + + +def get_supply_function_call_parameters( + protocol: str, + token_addresses: list[str], +) -> tuple[list[str], str]: + if protocol == "zkLend": + return token_addresses, "felt_total_supply" + if protocol in {"Nostra Alpha", "Nostra Mainnet"}: + return token_addresses, "totalSupply" + raise ValueError diff --git a/apps/dashboard_app/helpers/protocol_stats.py b/apps/dashboard_app/helpers/protocol_stats.py new file mode 100644 index 00000000..4abc6f8b --- /dev/null +++ b/apps/dashboard_app/helpers/protocol_stats.py @@ -0,0 +1,238 @@ +import asyncio +from collections import defaultdict + +import pandas as pd +from data_handler.handlers import blockchain_call +from shared.constants import TOKEN_SETTINGS +from shared.state import State +from shared.types import Prices + +from dashboard_app.helpers.loans_table import ( + get_protocol, + get_supply_function_call_parameters, +) +from dashboard_app.helpers.tools import get_addresses, get_underlying_address + + +def get_general_stats( + states: list[State], + loan_stats: dict[str, pd.DataFrame], + save_data: bool = False, +) -> pd.DataFrame: + data = [] + for state in states: + protocol = get_protocol(state=state) + number_of_active_users = state.compute_number_of_active_loan_entities() + number_of_active_borrowers = ( + state.compute_number_of_active_loan_entities_with_debt() + ) + data.append( + { + "Protocol": protocol, + "Number of active users": number_of_active_users, + # At the moment, Hashstack V0 and Hashstack V1 are the only protocols for which the number of active + # loans doesn't equal the number of active users. The reason is that Hashstack V0 and Hashstack V1 + # allow for liquidations on the loan level, whereas other protocols use user-level liquidations. + "Number of active loans": state.compute_number_of_active_loan_entities(), + "Number of active borrowers": number_of_active_borrowers, + "Total debt (USD)": round(loan_stats[protocol]["Debt (USD)"].sum(), 4), + "Total risk adjusted collateral (USD)": round( + loan_stats[protocol]["Risk-adjusted collateral (USD)"].sum(), 4 + ), + "Total Collateral (USD)": round( + loan_stats[protocol]["Collateral (USD)"].sum(), 4 + ), + } + ) + data = pd.DataFrame(data) + return data + + +def get_supply_stats( + states: list[State], + prices: Prices, + save_data: bool = False, +) -> pd.DataFrame: + data = [] + for state in states: + protocol = get_protocol(state=state) + token_supplies = {} + for token in TOKEN_SETTINGS: + ( + addresses, + selector, + ) = get_supply_function_call_parameters( + protocol=protocol, + token_addresses=get_addresses( + token_parameters=state.token_parameters.collateral, + underlying_symbol=token, + ), + ) + supply = 0 + for address in addresses: + supply += asyncio.run( + blockchain_call.func_call( + addr=int(address, base=16), + selector=selector, + calldata=[], + ) + )[0] + supply = supply / TOKEN_SETTINGS[token].decimal_factor + token_supplies[token] = round(supply, 4) + data.append( + { + "Protocol": protocol, + "ETH supply": token_supplies["ETH"], + "WBTC supply": token_supplies["WBTC"], + "USDC supply": token_supplies["USDC"], + "DAI supply": token_supplies["DAI"], + "USDT supply": token_supplies["USDT"], + "wstETH supply": token_supplies["wstETH"], + "LORDS supply": token_supplies["LORDS"], + "STRK supply": token_supplies["STRK"], + } + ) + data = pd.DataFrame(data) + data["Total supply (USD)"] = sum( + data[column] * prices[TOKEN_SETTINGS[column.replace(" supply", "")].address] + for column in data.columns + if "supply" in column + ).apply(lambda x: round(x, 4)) + return data + + +def get_collateral_stats( + states: list[State], + save_data: bool = False, +) -> pd.DataFrame: + data = [] + for state in states: + protocol = get_protocol(state=state) + token_collaterals = defaultdict(float) + for token in TOKEN_SETTINGS: + # TODO: save zkLend amounts under token_addresses? + if protocol == "zkLend": + token_addresses = [ + get_underlying_address( + token_parameters=state.token_parameters.collateral, + underlying_symbol=token, + ) + ] + elif protocol in {"Nostra Alpha", "Nostra Mainnet"}: + token_addresses = get_addresses( + token_parameters=state.token_parameters.collateral, + underlying_symbol=token, + ) + else: + raise ValueError + for token_address in token_addresses: + collateral = ( + sum( + float(loan_entity.collateral[token_address]) + for loan_entity in state.loan_entities.values() + ) + / TOKEN_SETTINGS[token].decimal_factor + * float(state.interest_rate_models.collateral[token_address]) + ) + token_collaterals[token] += round(collateral, 4) + + data.append( + { + "Protocol": protocol, + "ETH collateral": token_collaterals["ETH"], + "WBTC collateral": token_collaterals["WBTC"], + "USDC collateral": token_collaterals["USDC"], + "DAI collateral": token_collaterals["DAI"], + "USDT collateral": token_collaterals["USDT"], + "wstETH collateral": token_collaterals["wstETH"], + "LORDS collateral": token_collaterals["LORDS"], + "STRK collateral": token_collaterals["STRK"], + } + ) + return pd.DataFrame(data) + + +def get_debt_stats( + states: list[State], + save_data: bool = False, +) -> pd.DataFrame: + data = [] + for state in states: + protocol = get_protocol(state=state) + token_debts = defaultdict(float) + for token in TOKEN_SETTINGS: + # TODO: save zkLend amounts under token_addresses? + if protocol == "zkLend": + token_addresses = [ + get_underlying_address( + token_parameters=state.token_parameters.debt, + underlying_symbol=token, + ) + ] + elif protocol in {"Nostra Alpha", "Nostra Mainnet"}: + token_addresses = get_addresses( + token_parameters=state.token_parameters.debt, + underlying_symbol=token, + ) + else: + raise ValueError + for token_address in token_addresses: + debt = ( + sum( + float(loan_entity.debt[token_address]) + for loan_entity in state.loan_entities.values() + ) + / TOKEN_SETTINGS[token].decimal_factor + * float(state.interest_rate_models.debt[token_address]) + ) + token_debts[token] = round(debt, 4) + + data.append( + { + "Protocol": protocol, + "ETH debt": token_debts["ETH"], + "WBTC debt": token_debts["WBTC"], + "USDC debt": token_debts["USDC"], + "DAI debt": token_debts["DAI"], + "USDT debt": token_debts["USDT"], + "wstETH debt": token_debts["wstETH"], + "LORDS debt": token_debts["LORDS"], + "STRK debt": token_debts["STRK"], + } + ) + data = pd.DataFrame(data) + return data + + +def get_utilization_stats( + general_stats: pd.DataFrame, + supply_stats: pd.DataFrame, + debt_stats: pd.DataFrame, + save_data: bool = False, +) -> pd.DataFrame: + data = pd.DataFrame( + { + "Protocol": general_stats["Protocol"], + "Total utilization": general_stats["Total debt (USD)"] + / (general_stats["Total debt (USD)"] + supply_stats["Total supply (USD)"]), + "ETH utilization": debt_stats["ETH debt"] + / (supply_stats["ETH supply"] + debt_stats["ETH debt"]), + "WBTC utilization": debt_stats["WBTC debt"] + / (supply_stats["WBTC supply"] + debt_stats["WBTC debt"]), + "USDC utilization": debt_stats["USDC debt"] + / (supply_stats["USDC supply"] + debt_stats["USDC debt"]), + "DAI utilization": debt_stats["DAI debt"] + / (supply_stats["DAI supply"] + debt_stats["DAI debt"]), + "USDT utilization": debt_stats["USDT debt"] + / (supply_stats["USDT supply"] + debt_stats["USDT debt"]), + "wstETH utilization": debt_stats["wstETH debt"] + / (supply_stats["wstETH supply"] + debt_stats["wstETH debt"]), + "LORDS utilization": debt_stats["LORDS debt"] + / (supply_stats["LORDS supply"] + debt_stats["LORDS debt"]), + "STRK utilization": debt_stats["STRK debt"] + / (supply_stats["STRK supply"] + debt_stats["STRK debt"]), + }, + ) + utilization_columns = [x for x in data.columns if "utilization" in x] + data[utilization_columns] = data[utilization_columns].map(lambda x: round(x, 4)) + return data From 23f3cda2f7ce5acaa89ee0bbec245e62e398b275 Mon Sep 17 00:00:00 2001 From: djeck1432 Date: Mon, 18 Nov 2024 22:22:43 +0100 Subject: [PATCH 02/31] fix import --- apps/dashboard_app/charts/main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/dashboard_app/charts/main.py b/apps/dashboard_app/charts/main.py index 0595c187..f27feab8 100644 --- a/apps/dashboard_app/charts/main.py +++ b/apps/dashboard_app/charts/main.py @@ -1,8 +1,8 @@ import streamlit as st -from dashboard_app.charts.main_chart_figure import get_main_chart_figure -from dashboard_app.charts.utils import process_liquidity -from dashboard_app.helpers.settings import ( +from charts.main_chart_figure import get_main_chart_figure +from charts.utils import process_liquidity +from helpers.settings import ( COLLATERAL_TOKENS, DEBT_TOKENS, STABLECOIN_BUNDLE_NAME, From 1f66f037d071a3f33357f2e7de3d5c93dd94d83d Mon Sep 17 00:00:00 2001 From: djeck1432 Date: Mon, 18 Nov 2024 22:22:54 +0100 Subject: [PATCH 03/31] fix imports --- apps/dashboard_app/charts/main_chart_figure.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/apps/dashboard_app/charts/main_chart_figure.py b/apps/dashboard_app/charts/main_chart_figure.py index fdfe1153..08a24fe0 100644 --- a/apps/dashboard_app/charts/main_chart_figure.py +++ b/apps/dashboard_app/charts/main_chart_figure.py @@ -4,13 +4,12 @@ import plotly.express import plotly.graph_objs -from dashboard_app.helpers.settings import TOKEN_SETTINGS -from dashboard_app.helpers.tools import ( +from helpers.settings import TOKEN_SETTINGS +from helpers.tools import ( get_collateral_token_range, get_custom_data, get_prices, get_underlying_address, - save_dataframe, ) from shared.amms import SwapAmm from shared.state import State From 976129a73eb9a2ceda45c718761f32fe76fd0723 Mon Sep 17 00:00:00 2001 From: djeck1432 Date: Mon, 18 Nov 2024 22:25:04 +0100 Subject: [PATCH 04/31] add comments --- apps/dashboard_app/helpers/tools.py | 4 ++-- apps/data_handler/handlers/loan_states/zklend/utils.py | 1 + apps/legacy_app/update_data.py | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/apps/dashboard_app/helpers/tools.py b/apps/dashboard_app/helpers/tools.py index e576fc69..6afcd13c 100644 --- a/apps/dashboard_app/helpers/tools.py +++ b/apps/dashboard_app/helpers/tools.py @@ -6,14 +6,14 @@ import pandas as pd import requests from google.cloud.storage import Client +from shared.blockchain_call import func_call +from shared.types import TokenParameters from starknet_py.cairo.felt import decode_shortstring from dashboard_app.helpers.settings import ( PAIRS, UNDERLYING_SYMBOLS_TO_UNDERLYING_ADDRESSES, ) -from shared.blockchain_call import func_call -from shared.types import TokenParameters GS_BUCKET_NAME = "derisk-persistent-state/v3" diff --git a/apps/data_handler/handlers/loan_states/zklend/utils.py b/apps/data_handler/handlers/loan_states/zklend/utils.py index 357a47cf..943b24d8 100644 --- a/apps/data_handler/handlers/loan_states/zklend/utils.py +++ b/apps/data_handler/handlers/loan_states/zklend/utils.py @@ -65,6 +65,7 @@ def _set_loan_state_per_user(self, loan_state: ZkLendCollateralDebt) -> None: :param loan_state: The loan state data. """ + # FIXME fetch only result enabled/disabled for the user user_loan_state = self.zklend_state.loan_entities[loan_state.user_id] user_loan_state.collateral_enabled.values = loan_state.collateral_enabled user_loan_state.collateral.values = self._convert_float_to_decimal(loan_state.collateral) diff --git a/apps/legacy_app/update_data.py b/apps/legacy_app/update_data.py index e1d7b6bd..151d5b0e 100644 --- a/apps/legacy_app/update_data.py +++ b/apps/legacy_app/update_data.py @@ -71,7 +71,7 @@ def update_data(zklend_state: src.zklend.ZkLendState): # asyncio.run(hashstack_v0_state.collect_token_parameters()) # asyncio.run(hashstack_v1_state.collect_token_parameters()) logging.info(f"collected token parameters in {time.time() - t2}s") - + # TODO move it to separated function # Get prices of the underlying tokens. t_prices = time.time() states = [ @@ -103,7 +103,7 @@ def update_data(zklend_state: src.zklend.ZkLendState): ) prices = src.helpers.get_prices(token_decimals=underlying_addresses_to_decimals) logging.info(f"prices in {time.time() - t_prices}s") - + # TODO: move it to separated function END t_swap = time.time() swap_amms = src.swap_amm.SwapAmm() asyncio.run(swap_amms.init()) From b9dfcd2d4ba59456c11a8161575fe3739f916460 Mon Sep 17 00:00:00 2001 From: djeck1432 Date: Mon, 18 Nov 2024 22:30:47 +0100 Subject: [PATCH 05/31] adjust to new functions --- apps/dashboard_app/helpers/load_data.py | 54 +++++++------------------ 1 file changed, 15 insertions(+), 39 deletions(-) diff --git a/apps/dashboard_app/helpers/load_data.py b/apps/dashboard_app/helpers/load_data.py index 8e22c3b1..15cf247c 100644 --- a/apps/dashboard_app/helpers/load_data.py +++ b/apps/dashboard_app/helpers/load_data.py @@ -1,5 +1,4 @@ import asyncio -import itertools import logging import math from time import monotonic @@ -9,6 +8,13 @@ from shared.constants import TOKEN_SETTINGS from dashboard_app.data_conector import DataConnector +from dashboard_app.helpers.protocol_stats import ( + get_general_stats, + get_supply_stats, + get_collateral_stats, + get_debt_stats, + get_utilization_stats +) from dashboard_app.helpers.loans_table import get_loans_table_data, get_protocol from dashboard_app.helpers.tools import get_prices @@ -31,11 +37,6 @@ def init_zklend_state(): print(f"Initialized ZkLend state in {monotonic() - start:.2f}s") -if __name__ == "__main__": - init_zklend_state() - # TODO: Implement periodic updates - - def update_data(zklend_state: ZkLendState): # TODO: parallelize per protocol # TODO: stream the data, don't wait until we get all events @@ -110,46 +111,21 @@ def update_data(zklend_state: ZkLendState): for state in states: protocol = get_protocol(state=state) loan_stats[protocol] = get_loans_table_data( - state=state, prices=prices, save_data=True + state=state, prices=prices ) - general_stats = src.protocol_stats.get_general_stats( - states=states, loan_stats=loan_stats, save_data=True + general_stats = get_general_stats( + states=states, loan_stats=loan_stats ) - supply_stats = src.protocol_stats.get_supply_stats( + supply_stats = get_supply_stats( states=states, prices=prices, - save_data=True, ) - _ = src.protocol_stats.get_collateral_stats(states=states, save_data=True) - debt_stats = src.protocol_stats.get_debt_stats(states=states, save_data=True) - _ = src.protocol_stats.get_utilization_stats( + collateral_stats = get_collateral_stats(states=states) + debt_stats = get_debt_stats(states=states) + utilization_stats = get_utilization_stats( general_stats=general_stats, supply_stats=supply_stats, debt_stats=debt_stats, - save_data=True, - ) - - max_block_number = zklend_events["block_number"].max() - max_timestamp = zklend_events["timestamp"].max() - last_update = { - "timestamp": str(max_timestamp), - "block_number": str(max_block_number), - } - src.persistent_state.upload_object_as_pickle( - last_update, path=src.persistent_state.LAST_UPDATE_FILENAME - ) - zklend_state.save_loan_entities( - path=src.persistent_state.PERSISTENT_STATE_LOAN_ENTITIES_FILENAME - ) - zklend_state.clear_loan_entities() - src.persistent_state.upload_object_as_pickle( - zklend_state, path=src.persistent_state.PERSISTENT_STATE_FILENAME - ) - loan_entities = pandas.read_parquet( - f"gs://{src.helpers.GS_BUCKET_NAME}/{src.persistent_state.PERSISTENT_STATE_LOAN_ENTITIES_FILENAME}", - engine="fastparquet", ) - zklend_state.set_loan_entities(loan_entities=loan_entities) - logging.info(f"Updated CSV data in {time.time() - t0}s") - return zklend_state + return zklend_state, general_stats, supply_stats, collateral_stats, debt_stats, utilization_stats From eb6d2b6986919359a5ee1573b89afef4944fbc93 Mon Sep 17 00:00:00 2001 From: djeck1432 Date: Mon, 18 Nov 2024 22:31:00 +0100 Subject: [PATCH 06/31] removed unused params --- apps/dashboard_app/helpers/protocol_stats.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/apps/dashboard_app/helpers/protocol_stats.py b/apps/dashboard_app/helpers/protocol_stats.py index 4abc6f8b..2efb9c95 100644 --- a/apps/dashboard_app/helpers/protocol_stats.py +++ b/apps/dashboard_app/helpers/protocol_stats.py @@ -17,7 +17,6 @@ def get_general_stats( states: list[State], loan_stats: dict[str, pd.DataFrame], - save_data: bool = False, ) -> pd.DataFrame: data = [] for state in states: @@ -51,7 +50,6 @@ def get_general_stats( def get_supply_stats( states: list[State], prices: Prices, - save_data: bool = False, ) -> pd.DataFrame: data = [] for state in states: @@ -103,7 +101,6 @@ def get_supply_stats( def get_collateral_stats( states: list[State], - save_data: bool = False, ) -> pd.DataFrame: data = [] for state in states: @@ -154,7 +151,6 @@ def get_collateral_stats( def get_debt_stats( states: list[State], - save_data: bool = False, ) -> pd.DataFrame: data = [] for state in states: @@ -208,7 +204,6 @@ def get_utilization_stats( general_stats: pd.DataFrame, supply_stats: pd.DataFrame, debt_stats: pd.DataFrame, - save_data: bool = False, ) -> pd.DataFrame: data = pd.DataFrame( { From ef8da4d8076c76d92ba71b650031515c31b2fc88 Mon Sep 17 00:00:00 2001 From: djeck1432 Date: Mon, 18 Nov 2024 22:31:27 +0100 Subject: [PATCH 07/31] rework data connector --- apps/dashboard_app/data_conector.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/apps/dashboard_app/data_conector.py b/apps/dashboard_app/data_conector.py index 9c01496a..673fd6b3 100644 --- a/apps/dashboard_app/data_conector.py +++ b/apps/dashboard_app/data_conector.py @@ -9,7 +9,22 @@ class DataConnector: REQUIRED_VARS = ("DB_USER", "DB_PASSWORD", "DB_HOST", "DB_PORT", "DB_NAME") - SQL_QUERY = "SELECT * FROM %s WHERE protocol_id = 'zkLend'" + ZKLEND_SQL_QUERY = """ + SELECT + ls.block, + ls.user, + ls.collateral, + ls.debt, + zcd.collateral_enabled + FROM + loan_state AS ls + JOIN + zklend_collateral_debt AS zcd + ON + ls.user = zcd.user_id + WHERE + ls.protocol_id = 'zkLend'; + """ def __init__(self): """ @@ -23,15 +38,13 @@ def __init__(self): ) self.engine = sqlalchemy.create_engine(self.db_url) - def fetch_data(self, table_name: str, protocol_id: str) -> pd.DataFrame: + def fetch_data(self, query: str) -> pd.DataFrame: """ Fetch data from the database using a SQL query. - :param table_name: Name of the table to fetch data from. - :param protocol_id: ID of the protocol to fetch data for. + :param query: SQL query to execute. :return: DataFrame containing the query results """ - query = self.SQL_QUERY % (table_name,) with self.engine.connect() as connection: df = pd.read_sql(query, connection) return df From 98c3a14a9fd80dafc9bc514a292c1a431f15d3c5 Mon Sep 17 00:00:00 2001 From: djeck1432 Date: Wed, 20 Nov 2024 20:45:21 +0100 Subject: [PATCH 08/31] add new file --- .../handler_tools/data_parser/nostra.py | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 apps/data_handler/handler_tools/data_parser/nostra.py diff --git a/apps/data_handler/handler_tools/data_parser/nostra.py b/apps/data_handler/handler_tools/data_parser/nostra.py new file mode 100644 index 00000000..fb7a6cd1 --- /dev/null +++ b/apps/data_handler/handler_tools/data_parser/nostra.py @@ -0,0 +1,30 @@ + + + +class NostraDataParser: + """ + Parses the nostra data to human-readable format. + """ + def parse_interest_rate_model_event(self): + pass + + def parse_non_interest_bearing_collateral_mint_event(self): + pass + + def parse_non_interest_bearing_collateral_burn_event(self): + pass + + def parse_interest_bearing_collateral_mint_event(self): + pass + + def parse_interest_bearing_collateral_burn_event(self): + pass + + def parse_debt_transfer_event(self): + pass + + def parse_debt_mint_event(self): + pass + + def parse_debt_burn_event(self): + pass \ No newline at end of file From f97d29175328c5d4324d9dd7d7c9da58a43ae5f5 Mon Sep 17 00:00:00 2001 From: djeck1432 Date: Wed, 20 Nov 2024 21:22:02 +0100 Subject: [PATCH 09/31] add new file --- apps/data_handler/celery_app/event_tasks.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 apps/data_handler/celery_app/event_tasks.py diff --git a/apps/data_handler/celery_app/event_tasks.py b/apps/data_handler/celery_app/event_tasks.py new file mode 100644 index 00000000..e69de29b From f38b76076ba46fdd57384376789733099bed2775 Mon Sep 17 00:00:00 2001 From: djeck1432 Date: Thu, 21 Nov 2024 08:28:06 +0100 Subject: [PATCH 10/31] Update CONTRIBUTING.md --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7c13fe37..0762f48b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,7 +1,7 @@ # Contributor Guidelines 1. Task Claiming - - Comment: 'I would like to take this task, I love 🍍' including the estimated delivery timeline (start and completion dates) and a brief summary of relevant skills (required for complex tasks). Make sure, that you start with comment which is provided with secret phrase, otherwise your application will be ignored. + - Comment: 'I would like to take this task, I love orange' including the estimated delivery timeline (start and completion dates) and a brief summary of relevant skills (required for complex tasks). Make sure, that you start with comment which is provided with secret phrase, otherwise your application will be ignored. - Join the contributors' Telegram group for updates and discussions. 2. Task Assignment - Easy tasks: Assigned on a first-come, first-served basis. No further assignment is required. From bf3fb59a0a5a4cac2839efedd18a8284875dfad1 Mon Sep 17 00:00:00 2001 From: Benjtalkshow Date: Fri, 22 Nov 2024 07:52:56 +0100 Subject: [PATCH 11/31] Tasks for processing and storing ZkLend protocol events --- apps/data_handler/celery_app/event_tasks.py | 47 +++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/apps/data_handler/celery_app/event_tasks.py b/apps/data_handler/celery_app/event_tasks.py index e69de29b..3f0f8c77 100644 --- a/apps/data_handler/celery_app/event_tasks.py +++ b/apps/data_handler/celery_app/event_tasks.py @@ -0,0 +1,47 @@ +""" +Tasks for processing and storing ZkLend protocol events. +""" + +import logging +from datetime import datetime + +from data_handler.celery_app.celery_conf import app +from data_handler.handlers.events.zklend import ZklendTransformer + +logger = logging.getLogger(__name__) + + +@app.task(name="process_zklend_events") +def process_zklend_events(): + """ + Process and store ZkLend protocol events. + Fetches events from the blockchain, transforms them into the required format, + and saves them to the database. + """ + start_time = datetime.utcnow() + logger.info("Starting ZkLend event processing") + + try: + # Initialize and run transformer + transformer = ZklendTransformer() + transformer.run() + + # Log success metrics + execution_time = (datetime.utcnow() - start_time).total_seconds() + logger.info( + f"Successfully processed ZkLend events in {execution_time:.2f}s. " + f"Blocks: {transformer.last_block - transformer.PAGINATION_SIZE} " + f"to {transformer.last_block}" + ) + + except Exception as exc: + execution_time = (datetime.utcnow() - start_time).total_seconds() + logger.error( + f"Error processing ZkLend events after {execution_time:.2f}s: {exc}", + exc_info=True + ) + raise + + +if __name__ == "__main__": + process_zklend_events() \ No newline at end of file From 566aae89ce590e15453101d869147aac1d5c4500 Mon Sep 17 00:00:00 2001 From: Benjtalkshow Date: Fri, 22 Nov 2024 21:03:10 +0100 Subject: [PATCH 12/31] add executing task in celery_confy.py, catching more specific exceptions --- apps/data_handler/celery_app/celery_conf.py | 4 +++ apps/data_handler/celery_app/event_tasks.py | 33 +++++++++++---------- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/apps/data_handler/celery_app/celery_conf.py b/apps/data_handler/celery_app/celery_conf.py index 07f25f18..4697a5a9 100644 --- a/apps/data_handler/celery_app/celery_conf.py +++ b/apps/data_handler/celery_app/celery_conf.py @@ -68,6 +68,10 @@ "task": "ekubo_order_book", "schedule": ORDER_BOOK_TIME_INTERVAL, }, + f"process_zklend_events_{CRONTAB_TIME}_mins": { + "task": "process_zklend_events", + "schedule": crontab(minute=f"*/{CRONTAB_TIME}"), + }, } from data_handler.celery_app.order_books_tasks import ekubo_order_book diff --git a/apps/data_handler/celery_app/event_tasks.py b/apps/data_handler/celery_app/event_tasks.py index 3f0f8c77..f9d8ceb1 100644 --- a/apps/data_handler/celery_app/event_tasks.py +++ b/apps/data_handler/celery_app/event_tasks.py @@ -6,10 +6,10 @@ from datetime import datetime from data_handler.celery_app.celery_conf import app -from data_handler.handlers.events.zklend import ZklendTransformer +from data_handler.handlers.events.zklend.transform_events import ZklendTransformer -logger = logging.getLogger(__name__) +logger = logging.getLogger(__name__) @app.task(name="process_zklend_events") def process_zklend_events(): @@ -20,28 +20,31 @@ def process_zklend_events(): """ start_time = datetime.utcnow() logger.info("Starting ZkLend event processing") - try: # Initialize and run transformer transformer = ZklendTransformer() transformer.run() - # Log success metrics execution_time = (datetime.utcnow() - start_time).total_seconds() logger.info( - f"Successfully processed ZkLend events in {execution_time:.2f}s. " - f"Blocks: {transformer.last_block - transformer.PAGINATION_SIZE} " - f"to {transformer.last_block}" + "Successfully processed ZkLend events in %.2fs. Blocks: %d to %d", + execution_time, + transformer.last_block - transformer.PAGINATION_SIZE, + transformer.last_block ) - - except Exception as exc: + except (ValueError, TypeError, RuntimeError) as exc: # Catching more specific exceptions execution_time = (datetime.utcnow() - start_time).total_seconds() logger.error( - f"Error processing ZkLend events after {execution_time:.2f}s: {exc}", + "Error processing ZkLend events after %.2fs: %s", + execution_time, + exc, + exc_info=True + ) + except Exception as exc: # Still keeping a general exception catch as a fallback + execution_time = (datetime.utcnow() - start_time).total_seconds() + logger.error( + "Unexpected error processing ZkLend events after %.2fs: %s", + execution_time, + exc, exc_info=True ) - raise - - -if __name__ == "__main__": - process_zklend_events() \ No newline at end of file From 06a54633fc6fe281128d2b204e7de5e3fedbd496 Mon Sep 17 00:00:00 2001 From: 0xElectrifier <101304307+0xElectrifier@users.noreply.github.com> Date: Fri, 22 Nov 2024 04:43:24 +0000 Subject: [PATCH 13/31] Added `DebtMintEventData` and `DebtBurnEventData` serializers for formatting event data in `nostra_alpha` --- .../handler_tools/data_parser/nostra.py | 21 +++- .../handler_tools/data_parser/serializers.py | 112 ++++++++++++++++++ .../loan_states/nostra_alpha/events.py | 5 + 3 files changed, 132 insertions(+), 6 deletions(-) diff --git a/apps/data_handler/handler_tools/data_parser/nostra.py b/apps/data_handler/handler_tools/data_parser/nostra.py index fb7a6cd1..c0d93772 100644 --- a/apps/data_handler/handler_tools/data_parser/nostra.py +++ b/apps/data_handler/handler_tools/data_parser/nostra.py @@ -1,4 +1,7 @@ - +from data_handler.handler_tools.data_parser.serializers import ( + DebtMintEventData, + DebtBurnEventData +) class NostraDataParser: @@ -23,8 +26,14 @@ def parse_interest_bearing_collateral_burn_event(self): def parse_debt_transfer_event(self): pass - def parse_debt_mint_event(self): - pass - - def parse_debt_burn_event(self): - pass \ No newline at end of file + def parse_debt_mint_event(self, event_data: list[Any]) -> DebtMintEventData: + return DebtMintEventData( + user=event_data[0], + token=event_data[1] + ) + + def parse_debt_burn_event(self, event_data: list[Any]) -> DebtBurnEventData: + return DebtBurnEventData( + user=event_data[0], + amount=event_data[1] + ) \ No newline at end of file diff --git a/apps/data_handler/handler_tools/data_parser/serializers.py b/apps/data_handler/handler_tools/data_parser/serializers.py index 593f220d..9cb212f3 100644 --- a/apps/data_handler/handler_tools/data_parser/serializers.py +++ b/apps/data_handler/handler_tools/data_parser/serializers.py @@ -333,3 +333,115 @@ def validate_valid_addresses(cls, value: str, info: ValidationInfo) -> str: if not value.startswith("0x"): raise ValueError("Invalid address provided for %s" % info.field_name) return add_leading_zeros(value) + + +class DebtMintEventData(BaseModel): + """ + Class for representing debt mint event data. + + Attributes: + user (str): The address of the user associated with the debt mint event. + amount (str): The amount minted in the debt mint event. + + Returns: + DebtMintEventData: A Pydantic model with the parsed and validated event data in a human-readable format. + """ + + user: str + amount: str + + @field_validator("user") + def validate_address(cls, value: str, info: ValidationInfo) -> str: + """ + Validates that the provided address starts with '0x' and + formats it with leading zeros. + + Args: + value (str): The address string to validate. + + Returns: + str: The validated and formatted address. + + Raises: + ValueError: If the provided address does not start with '0x'. + """ + if not value.startswith("0x"): + raise ValueError(f"Invalid address provided for {info.field_name}") + return add_leading_zeros(value) + + @field_validator("amount") + def validate_numeric_string(cls, value: str, info: ValidationInfo) -> Decimal: + """ + Validates that the provided amount is numeric and converts it to a Decimal. + + Args: + value (str): The amount string to validate. + + Returns: + Decimal: The validated and converted amount as a Decimal. + + Raises: + ValueError: If the provided amount is not numeric. + """ + try: + return Decimal(int(value, 16)) + except ValueError: + raise ValueError( + f"{info.field_name} field is not a valid hexadecimal number" + ) + + +class DebtBurnEventData(BaseModel): + """ + Class for representing debt burn event data. + + Attributes: + user (str): The address of the user associated with the debt burn event. + amount (str): The amount burned in the debt burn event. + + Returns: + DebtBurnEventData: A Pydantic model with the parsed and validated event data in a human-readable format. + """ + + user: str + amount: str + + @field_validator("user") + def validate_address(cls, value: str, info: ValidationInfo) -> str: + """ + Validates that the provided address starts with '0x' and + formats it with leading zeros. + + Args: + value (str): The address string to validate. + + Returns: + str: The validated and formatted address. + + Raises: + ValueError: If the provided address does not start with '0x'. + """ + if not value.startswith("0x"): + raise ValueError(f"Invalid address provided for {info.field_name}") + return add_leading_zeros(value) + + @field_validator("amount") + def validate_numeric_string(cls, value: str, info: ValidationInfo) -> Decimal: + """ + Validates that the provided amount is numeric and converts it to a Decimal. + + Args: + value (str): The amount string to validate. + + Returns: + Decimal: The validated and converted amount as a Decimal. + + Raises: + ValueError: If the provided amount is not numeric. + """ + try: + return Decimal(int(value, 16)) + except ValueError: + raise ValueError( + f"{info.field_name} field is not a valid hexadecimal number" + ) \ No newline at end of file diff --git a/apps/data_handler/handlers/loan_states/nostra_alpha/events.py b/apps/data_handler/handlers/loan_states/nostra_alpha/events.py index 69b2e495..fd68a849 100644 --- a/apps/data_handler/handlers/loan_states/nostra_alpha/events.py +++ b/apps/data_handler/handlers/loan_states/nostra_alpha/events.py @@ -19,6 +19,7 @@ from data_handler.handlers.helpers import blockchain_call, get_addresses, get_symbol from data_handler.handlers.settings import TokenSettings from data_handler.handlers.state import NOSTRA_ALPHA_SPECIFIC_TOKEN_SETTINGS +from data_handler.handler_tools.data_parser.nostra import NostraDataParser from shared.constants import ProtocolIDs from shared.helpers import add_leading_zeros @@ -602,6 +603,8 @@ def process_debt_mint_event(self, event: pd.Series) -> None: Processes the `Burn` event. :param event: Event data. """ + data = NostraDataParser.parse_debt_mint_event(event["data"]) + user, amount = data.user, data.amount if event["keys"] == [self.MINT_KEY]: # The order of the values in the `data` column is: `user`, `amount`, ``. # Example: @@ -633,6 +636,8 @@ def process_debt_burn_event(self, event: pd.Series) -> None: Processes the `Burn` event. :param event: Event data. """ + data = NostraDataParser.parse_debt_mint_event(event["data"]) + user, amount = data.user, data.amount if event["keys"] == [self.BURN_KEY]: # The order of the values in the `data` column is: `user`, `amount`, ``. # Example: From 80e6c3154424be2a1f2b97fef9975dbec4bdd5ef Mon Sep 17 00:00:00 2001 From: 0xElectrifier <101304307+0xElectrifier@users.noreply.github.com> Date: Fri, 22 Nov 2024 05:14:43 +0000 Subject: [PATCH 14/31] Updated `process_debt_mint_event` and `process_debt_burn_event` functions with the serialized data --- .../handlers/loan_states/nostra_alpha/events.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/apps/data_handler/handlers/loan_states/nostra_alpha/events.py b/apps/data_handler/handlers/loan_states/nostra_alpha/events.py index fd68a849..c557490b 100644 --- a/apps/data_handler/handlers/loan_states/nostra_alpha/events.py +++ b/apps/data_handler/handlers/loan_states/nostra_alpha/events.py @@ -603,14 +603,12 @@ def process_debt_mint_event(self, event: pd.Series) -> None: Processes the `Burn` event. :param event: Event data. """ - data = NostraDataParser.parse_debt_mint_event(event["data"]) - user, amount = data.user, data.amount if event["keys"] == [self.MINT_KEY]: # The order of the values in the `data` column is: `user`, `amount`, ``. # Example: # https://starkscan.co/event/0x030d23c4769917bc673875e107ebdea31711e2bdc45e658125dbc2e988945f69_4. - user = add_leading_zeros(event["data"][0]) - face_amount = decimal.Decimal(str(int(event["data"][1], base=16))) + data = NostraDataParser.parse_debt_mint_event(event["data"]) + user, face_amount = data.user, data.amount else: raise ValueError("Event = {} has an unexpected structure.".format(event)) if user == self.DEFERRED_BATCH_CALL_ADAPTER_ADDRESS: @@ -636,14 +634,12 @@ def process_debt_burn_event(self, event: pd.Series) -> None: Processes the `Burn` event. :param event: Event data. """ - data = NostraDataParser.parse_debt_mint_event(event["data"]) - user, amount = data.user, data.amount if event["keys"] == [self.BURN_KEY]: # The order of the values in the `data` column is: `user`, `amount`, ``. # Example: # https://starkscan.co/event/0x002e4ee376785f687f32715d8bbed787b6d0fa9775dc9329ca2185155a139ca3_5. - user = add_leading_zeros(event["data"][0]) - face_amount = decimal.Decimal(str(int(event["data"][1], base=16))) + data = NostraDataParser.parse_debt_mint_event(event["data"]) + user, face_amount = data.user, data.amount else: raise ValueError("Event = {} has an unexpected structure.".format(event)) if user == self.DEFERRED_BATCH_CALL_ADAPTER_ADDRESS: From cda249dc45d3dc3276586976c1225a345db43efd Mon Sep 17 00:00:00 2001 From: 0xElectrifier <101304307+0xElectrifier@users.noreply.github.com> Date: Fri, 22 Nov 2024 12:13:54 +0000 Subject: [PATCH 15/31] Fixed pylint docs related errors --- .../handler_tools/data_parser/nostra.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/apps/data_handler/handler_tools/data_parser/nostra.py b/apps/data_handler/handler_tools/data_parser/nostra.py index c0d93772..4b5f4b94 100644 --- a/apps/data_handler/handler_tools/data_parser/nostra.py +++ b/apps/data_handler/handler_tools/data_parser/nostra.py @@ -27,12 +27,34 @@ def parse_debt_transfer_event(self): pass def parse_debt_mint_event(self, event_data: list[Any]) -> DebtMintEventData: + """ + Parses the debt mint event data into a human-readable format using the + DebtMintEventData serializer. + + Args: + event_data (List[Any]): A list containing the raw debt mint event data, + typically with 2 elements: user and amount. + + Returns: + DebtMintEventData: A Pydantic model with the parsed and validated event data in a human-readable format. + """ return DebtMintEventData( user=event_data[0], token=event_data[1] ) def parse_debt_burn_event(self, event_data: list[Any]) -> DebtBurnEventData: + """ + Parses the debt burn event data into a human-readable format using the + DebtBurnEventData serializer. + + Args: + event_data (List[Any]): A list containing the raw debt burn event data, + typically with 2 elements: user and amount. + + Returns: + DebtBurnEventData: A Pydantic model with the parsed and validated event data in a human-readable format. + """ return DebtBurnEventData( user=event_data[0], amount=event_data[1] From b0c582f8bc10589cdbbadb1e96804aac5e763c3b Mon Sep 17 00:00:00 2001 From: 0xElectrifier <101304307+0xElectrifier@users.noreply.github.com> Date: Fri, 22 Nov 2024 12:25:26 +0000 Subject: [PATCH 16/31] Fixed module-level pylint docstring related errors --- apps/data_handler/handler_tools/data_parser/nostra.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apps/data_handler/handler_tools/data_parser/nostra.py b/apps/data_handler/handler_tools/data_parser/nostra.py index 4b5f4b94..a49262c9 100644 --- a/apps/data_handler/handler_tools/data_parser/nostra.py +++ b/apps/data_handler/handler_tools/data_parser/nostra.py @@ -1,3 +1,7 @@ +""" +This module contains the logic to parse the nostra data to human-readable format. +""" + from data_handler.handler_tools.data_parser.serializers import ( DebtMintEventData, DebtBurnEventData From 68b9a6dbb76a4cdf8c33540388a4b6735ffd0a2c Mon Sep 17 00:00:00 2001 From: djeck1432 Date: Fri, 22 Nov 2024 14:18:28 +0100 Subject: [PATCH 17/31] fix typo --- apps/data_handler/handler_tools/data_parser/nostra.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/data_handler/handler_tools/data_parser/nostra.py b/apps/data_handler/handler_tools/data_parser/nostra.py index a49262c9..7d2370fa 100644 --- a/apps/data_handler/handler_tools/data_parser/nostra.py +++ b/apps/data_handler/handler_tools/data_parser/nostra.py @@ -1,7 +1,7 @@ """ This module contains the logic to parse the nostra data to human-readable format. """ - +from typing import Any from data_handler.handler_tools.data_parser.serializers import ( DebtMintEventData, DebtBurnEventData From 0ea16d900704ace24b1c77332ffba7a6ae3e3e00 Mon Sep 17 00:00:00 2001 From: djeck1432 Date: Fri, 22 Nov 2024 18:03:20 +0100 Subject: [PATCH 18/31] disable telegram bot if TELEGRAM_TOKEN is not set --- apps/web_app/telegram/bot.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/apps/web_app/telegram/bot.py b/apps/web_app/telegram/bot.py index 80d78646..70ddb7c4 100644 --- a/apps/web_app/telegram/bot.py +++ b/apps/web_app/telegram/bot.py @@ -3,7 +3,11 @@ from .config import TELEGRAM_TOKEN from .handlers import index_router -bot = Bot(token=TELEGRAM_TOKEN) -dp = Dispatcher() +if TELEGRAM_TOKEN: + bot = Bot(token=TELEGRAM_TOKEN) + dp = Dispatcher() -dp.include_router(index_router) + dp.include_router(index_router) +else: + bot = None + dp = None From 139789b8d6b712b75b641cb55d4ce4ece442e61c Mon Sep 17 00:00:00 2001 From: djeck1432 Date: Sat, 23 Nov 2024 12:04:58 +0100 Subject: [PATCH 19/31] Update CONTRIBUTING.md --- CONTRIBUTING.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0762f48b..345bb09b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -7,12 +7,14 @@ - Easy tasks: Assigned on a first-come, first-served basis. No further assignment is required. - Medium and Complex tasks: Prospective assignees must outline their approach to the task. Assignments will be based on these proposals to ensure optimal match and prioritization. The higher the complexity of the task, the more detailed description of the approach is needed. 4. Initial Commit Requirement - - If no commits are made or the assignee is unreachable within `12` hours post-assignment, we reserve the right to reassign the task. + - If no commits are made or the assignee is unreachable within `10` hours post-assignment, we reserve the right to reassign the task. - We also reserve the right to reassign tasks if it becomes clear they cannot be completed by the hackathon's end. 5. Submission Guidelines - Submit a pull request (PR) from the forked repository. - Ensure to rebase on the current master branch before creating the PR. - +6. Resolve issue in your PR + - after review, you have 12 hours to fix it, exception, if you have a lot you can push just partially if you have `medium` or `complex` task + - If you couldn't do that, you will be unassign. ### Communication * For questions, contact us via this GitHub (response might be slower) or Telegram ( https://t.me/derisk_dev for a faster response). From df099f2252caa4c87c54b4d64ae855bf2651d885 Mon Sep 17 00:00:00 2001 From: djeck1432 Date: Sat, 23 Nov 2024 21:46:06 +0100 Subject: [PATCH 20/31] add new sql query --- apps/dashboard_app/data_conector.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/apps/dashboard_app/data_conector.py b/apps/dashboard_app/data_conector.py index 673fd6b3..55cedde2 100644 --- a/apps/dashboard_app/data_conector.py +++ b/apps/dashboard_app/data_conector.py @@ -25,6 +25,16 @@ class DataConnector: WHERE ls.protocol_id = 'zkLend'; """ + ZKLEND_INTEREST_RATE_SQL_QUERY = """ + WITH max_block AS ( + SELECT MAX(block) AS max_block + FROM interest_rate + WHERE protocol_id = 'zkLend' + ) + SELECT collateral, debt, block + FROM interest_rate + WHERE protocol_id = 'zkLend' AND block = (SELECT max_block FROM max_block); + """ def __init__(self): """ From 533300fceeb047146ddf5aecadba729590231d4a Mon Sep 17 00:00:00 2001 From: djeck1432 Date: Sat, 23 Nov 2024 21:46:18 +0100 Subject: [PATCH 21/31] handle errors --- apps/shared/loan_entity.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/apps/shared/loan_entity.py b/apps/shared/loan_entity.py index 1fb3f7ae..7fdd3c06 100644 --- a/apps/shared/loan_entity.py +++ b/apps/shared/loan_entity.py @@ -126,11 +126,16 @@ def has_collateral(self) -> bool: Check if the entity has any collateral. :return: bool """ - return any(token_amount for token_amount in self.collateral.values()) + try: + collateral_tokens = self.collateral.values() + except TypeError: + collateral_tokens = self.collateral.values + + return any(token_amount for token_amount in collateral_tokens) def has_debt(self) -> bool: """ Check if the entity has any debt. :return: bool """ - return any(token_amount for token_amount in self.debt.values()) + return any(token_amount for token_amount in self.debt.values) From a84e427afc1920b90102ef0633241eaa88bf35fc Mon Sep 17 00:00:00 2001 From: djeck1432 Date: Sat, 23 Nov 2024 21:46:29 +0100 Subject: [PATCH 22/31] comment broken address --- apps/shared/constants.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/apps/shared/constants.py b/apps/shared/constants.py index 769a307d..0703ca3e 100644 --- a/apps/shared/constants.py +++ b/apps/shared/constants.py @@ -27,11 +27,12 @@ decimal_factor=Decimal("1e6"), address="0x053c91253bc9682c04929ca02ed00b3e423f6710d2ee7e0d5ebb06f3ecf368a8", ), - "DAI": TokenSettings( - symbol="DAI", - decimal_factor=Decimal("1e18"), - address="0x00da114221cb83fa859dbdb4c44beeaa0bb37c7537ad5ae66fe5e0efd20e6eb3", - ), + # FIXME Uncomment when DAI is added correct address + # "DAI": TokenSettings( + # symbol="DAI", + # decimal_factor=Decimal("1e18"), + # address="0x00da114221cb83fa859dbdb4c44beeaa0bb37c7537ad5ae66fe5e0efd20e6eb3", + # ), "USDT": TokenSettings( symbol="USDT", decimal_factor=Decimal("1e6"), From 77cbd0845f8be900de1485f8f0ff1b5ceb7c0783 Mon Sep 17 00:00:00 2001 From: djeck1432 Date: Sat, 23 Nov 2024 21:46:59 +0100 Subject: [PATCH 23/31] fix import --- apps/dashboard_app/charts/__init__.py | 2 +- apps/dashboard_app/charts/main.py | 13 +++++++++---- apps/dashboard_app/charts/main_chart_figure.py | 4 ++-- apps/dashboard_app/dashboard.py | 13 ++++++++++--- 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/apps/dashboard_app/charts/__init__.py b/apps/dashboard_app/charts/__init__.py index 058ac177..a5f7e917 100644 --- a/apps/dashboard_app/charts/__init__.py +++ b/apps/dashboard_app/charts/__init__.py @@ -1 +1 @@ -from charts.main import Dashboard +from dashboard_app.charts.main import Dashboard diff --git a/apps/dashboard_app/charts/main.py b/apps/dashboard_app/charts/main.py index f27feab8..d0beeba5 100644 --- a/apps/dashboard_app/charts/main.py +++ b/apps/dashboard_app/charts/main.py @@ -1,8 +1,8 @@ import streamlit as st -from charts.main_chart_figure import get_main_chart_figure -from charts.utils import process_liquidity -from helpers.settings import ( +from dashboard_app.charts.main_chart_figure import get_main_chart_figure +from dashboard_app.charts.utils import process_liquidity +from dashboard_app.helpers.settings import ( COLLATERAL_TOKENS, DEBT_TOKENS, STABLECOIN_BUNDLE_NAME, @@ -13,6 +13,7 @@ transform_loans_data, transform_main_chart_data, ) +from dashboard_app.helpers.load_data import set_data class Dashboard: @@ -22,7 +23,11 @@ class Dashboard: # "Nostra Mainnet", ] - def __init__(self): + def __init__(self, zklend_state): + """ + Initialize the dashboard. + :param zklend_state: ZkLendState + """ # Set the page configuration st.set_page_config( layout="wide", diff --git a/apps/dashboard_app/charts/main_chart_figure.py b/apps/dashboard_app/charts/main_chart_figure.py index 08a24fe0..da6963f0 100644 --- a/apps/dashboard_app/charts/main_chart_figure.py +++ b/apps/dashboard_app/charts/main_chart_figure.py @@ -4,8 +4,8 @@ import plotly.express import plotly.graph_objs -from helpers.settings import TOKEN_SETTINGS -from helpers.tools import ( +from dashboard_app.helpers.settings import TOKEN_SETTINGS +from dashboard_app.helpers.tools import ( get_collateral_token_range, get_custom_data, get_prices, diff --git a/apps/dashboard_app/dashboard.py b/apps/dashboard_app/dashboard.py index e86f1e25..668e0576 100644 --- a/apps/dashboard_app/dashboard.py +++ b/apps/dashboard_app/dashboard.py @@ -1,5 +1,12 @@ -from charts import Dashboard +import logging +from dashboard_app.charts import Dashboard +from dashboard_app.helpers.load_data import DashboardDataHandler + if __name__ == "__main__": - dashboard = Dashboard() - dashboard.run() + logging.basicConfig(level=logging.INFO) + dashboard_data_handler = DashboardDataHandler() + dashboard_data_handler.load_data() + + # dashboard = Dashboard() + # dashboard.run() From 891396ddf47d1665a9aa04c636a179810559f397 Mon Sep 17 00:00:00 2001 From: djeck1432 Date: Sat, 23 Nov 2024 21:47:25 +0100 Subject: [PATCH 24/31] add logs --- apps/data_handler/handlers/loan_states/zklend/events.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/apps/data_handler/handlers/loan_states/zklend/events.py b/apps/data_handler/handlers/loan_states/zklend/events.py index efddcfa6..3c473a8a 100644 --- a/apps/data_handler/handlers/loan_states/zklend/events.py +++ b/apps/data_handler/handlers/loan_states/zklend/events.py @@ -12,7 +12,6 @@ - collect_token_parameters: Fetches token parameters. - process_*_event: Updates loan states based on events. """ -import asyncio import copy import decimal import logging @@ -31,6 +30,8 @@ ZKLEND_SPECIFIC_TOKEN_SETTINGS, ) +logger = logging.getLogger(__name__) + from data_handler.handlers import blockchain_call from shared.types import ( @@ -474,8 +475,9 @@ async def collect_token_parameters(self) -> None: # Get the sets of unique collateral and debt tokens. collateral_tokens = {y for x in self.loan_entities.values() for y in x.collateral.keys()} debt_tokens = {y for x in self.loan_entities.values() for y in x.debt.keys()} - - # Get parameters for each collateral and debt token. Under zkLend, + logging.info(f"Collecting token parameters for collateral tokens: {collateral_tokens}") + logging.info(f"Collecting token parameters for debt tokens: {debt_tokens}") + # Get parameters for each collateral and debt token. Under zkLend, # the collateral token in the events data is # the underlying token directly. for underlying_collateral_token_address in collateral_tokens: From ba38d1752ca24b3b465396d28bfffde8cfdf7951 Mon Sep 17 00:00:00 2001 From: djeck1432 Date: Sat, 23 Nov 2024 22:37:15 +0100 Subject: [PATCH 25/31] fix bug --- apps/data_handler/handlers/loan_states/zklend/events.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/data_handler/handlers/loan_states/zklend/events.py b/apps/data_handler/handlers/loan_states/zklend/events.py index 3c473a8a..008fdb17 100644 --- a/apps/data_handler/handlers/loan_states/zklend/events.py +++ b/apps/data_handler/handlers/loan_states/zklend/events.py @@ -473,8 +473,8 @@ async def collect_token_parameters(self) -> None: """Collects and sets token parameters for collateral and debt tokens under zkLend, including collateral factors, liquidation bonuses, and debt factors.""" # Get the sets of unique collateral and debt tokens. - collateral_tokens = {y for x in self.loan_entities.values() for y in x.collateral.keys()} - debt_tokens = {y for x in self.loan_entities.values() for y in x.debt.keys()} + collateral_tokens = {y for x in self.loan_entities.values() for y in x.collateral.values.keys()} + debt_tokens = {y for x in self.loan_entities.values() for y in x.debt.values.keys()} logging.info(f"Collecting token parameters for collateral tokens: {collateral_tokens}") logging.info(f"Collecting token parameters for debt tokens: {debt_tokens}") # Get parameters for each collateral and debt token. Under zkLend, From f9edf4855df2bc5c5cc483345b0598e4c2b878b0 Mon Sep 17 00:00:00 2001 From: djeck1432 Date: Sat, 23 Nov 2024 22:37:24 +0100 Subject: [PATCH 26/31] fix imports --- apps/web_app/utils/state.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/apps/web_app/utils/state.py b/apps/web_app/utils/state.py index 064df53d..4c1249f8 100644 --- a/apps/web_app/utils/state.py +++ b/apps/web_app/utils/state.py @@ -1,12 +1,9 @@ -from abc import ABC, abstractmethod +from abc import ABC from collections import defaultdict from dataclasses import dataclass from decimal import Decimal -from typing import Any -import pandas as pd - -from .helpers import Portfolio, TokenValues +from shared.types import Portfolio, TokenValues from .settings import TOKEN_SETTINGS as BASE_TOKEN_SETTINGS from .settings import TokenSettings as BaseTokenSettings From a264caf44656881a212cf92cd414ba855bbad096 Mon Sep 17 00:00:00 2001 From: djeck1432 Date: Sat, 23 Nov 2024 22:38:21 +0100 Subject: [PATCH 27/31] DashboardDataHandler add --- apps/dashboard_app/helpers/load_data.py | 316 ++++++++++++++++-------- 1 file changed, 209 insertions(+), 107 deletions(-) diff --git a/apps/dashboard_app/helpers/load_data.py b/apps/dashboard_app/helpers/load_data.py index 15cf247c..75c37b00 100644 --- a/apps/dashboard_app/helpers/load_data.py +++ b/apps/dashboard_app/helpers/load_data.py @@ -1,131 +1,233 @@ import asyncio import logging import math +from collections import defaultdict from time import monotonic from data_handler.handlers.loan_states.zklend.events import ZkLendState -from shared.amms import SwapAmm from shared.constants import TOKEN_SETTINGS from dashboard_app.data_conector import DataConnector +from dashboard_app.helpers.loans_table import get_loans_table_data, get_protocol from dashboard_app.helpers.protocol_stats import ( - get_general_stats, - get_supply_stats, get_collateral_stats, get_debt_stats, - get_utilization_stats + get_general_stats, + get_supply_stats, + get_utilization_stats, ) -from dashboard_app.helpers.loans_table import get_loans_table_data, get_protocol from dashboard_app.helpers.tools import get_prices -loger = logging.getLogger(__name__) +logger = logging.getLogger(__name__) data_connector = DataConnector() -def init_zklend_state(): - zklend_state = ZkLendState() - start = monotonic() - zklend_data = data_connector.fetch_data(data_connector.ZKLEND_SQL_QUERY) - zklend_data_dict = zklend_data.to_dict(orient="records") - for loan_state in zklend_data_dict: - user_loan_state = zklend_state.loan_entities[loan_state["user"]] - user_loan_state.collateral_enabled.values = loan_state["collateral_enabled"] - user_loan_state.collateral.values = loan_state["collateral"] - user_loan_state.debt.values = loan_state["debt"] - - zklend_state.last_block_number = zklend_data["block"].max() - print(f"Initialized ZkLend state in {monotonic() - start:.2f}s") - - -def update_data(zklend_state: ZkLendState): - # TODO: parallelize per protocol - # TODO: stream the data, don't wait until we get all events - # nostra_alpha_events = src.nostra_alpha.nostra_alpha_get_events() - # nostra_mainnet_events = src.nostra_mainnet.nostra_mainnet_get_events() - - # Iterate over ordered events to obtain the final state of each user. - - # nostra_alpha_state = src.nostra_alpha.NostraAlphaState() - # for _, nostra_alpha_event in nostra_alpha_events.iterrows(): - # nostra_alpha_state.process_event(event=nostra_alpha_event) - # - # nostra_mainnet_state = src.nostra_mainnet.NostraMainnetState() - # for _, nostra_mainnet_event in nostra_mainnet_events.iterrows(): - # nostra_mainnet_state.process_event(event=nostra_mainnet_event) - # logging.info(f"updated state in {time.time() - t1}s") - - # TODO: move this to state inits above? - # # Collect token parameters. - asyncio.run(zklend_state.collect_token_parameters()) - # TODO move it to separated function - # Get prices of the underlying tokens. - states = [ - zklend_state, - # nostra_alpha_state, - # nostra_mainnet_state, - ] - underlying_addresses_to_decimals = {} - for state in states: - underlying_addresses_to_decimals.update( - { - x.underlying_address: x.decimals - for x in state.token_parameters.collateral.values() - } +class DashboardDataHandler: + """ + Class responsible to handle the data for the dashboard. + """ + + def __init__(self): + """ + Initialize the data handler. + """ + self.underlying_addresses_to_decimals = defaultdict(dict) + self.zklend_state = self._init_zklend_state() + self.prices = None + # TODO add also nostra states + self.states = [ + self.zklend_state, + # nostra_alpha_state, + # nostra_mainnet_state, + ] + + @staticmethod + def _init_zklend_state() -> ZkLendState: + """ + Initialize ZkLend state. + Fetch data from the database and initialize the state. + :return: Initialized ZkLend state. + """ + logger.info("Initializing ZkLend state.") + zklend_state = ZkLendState() + start = monotonic() + zklend_data = data_connector.fetch_data(data_connector.ZKLEND_SQL_QUERY) + zklend_interest_rate_data = data_connector.fetch_data( + data_connector.ZKLEND_INTEREST_RATE_SQL_QUERY ) - underlying_addresses_to_decimals.update( + + zklend_data_dict = zklend_data.to_dict(orient="records") + for loan_state in zklend_data_dict: + user_loan_state = zklend_state.loan_entities[loan_state["user"]] + user_loan_state.collateral_enabled.values = loan_state["collateral_enabled"] + user_loan_state.collateral.values = loan_state["collateral"] + user_loan_state.debt.values = loan_state["debt"] + + zklend_state.last_block_number = zklend_data["block"].max() + zklend_state.interest_rate_models.collateral = zklend_interest_rate_data[ + "collateral" + ].iloc[0] + zklend_state.interest_rate_models.debt = zklend_interest_rate_data["debt"].iloc[ + 0 + ] + logger.info(f"Initialized ZkLend state in {monotonic() - start:.2f}s") + + return zklend_state + + def _set_prices(self) -> None: + """ + Set the prices of the underlying tokens. + """ + logger.info("Setting prices.") + self.prices = get_prices(token_decimals=self.underlying_addresses_to_decimals) + logger.info("Prices set.") + + def _collect_token_parameters(self): + """ + Collect token parameters. + :return: + """ + logger.info("Collecting token parameters.") + asyncio.run(self.zklend_state.collect_token_parameters()) + logger.info("Token parameters collected.") + + def _set_underlying_addresses_to_decimals(self): + """ + Set the underlying addresses to decimals. + """ + logger.info("Setting underlying addresses to decimals.") + for state in self.states: + self.underlying_addresses_to_decimals.update( + { + x.underlying_address: x.decimals + for x in state.token_parameters.collateral.values() + } + ) + self.underlying_addresses_to_decimals.update( + { + x.underlying_address: x.decimals + for x in state.token_parameters.debt.values() + } + ) + self.underlying_addresses_to_decimals.update( { - x.underlying_address: x.decimals - for x in state.token_parameters.debt.values() + x.address: int(math.log10(x.decimal_factor)) + for x in TOKEN_SETTINGS.values() } ) - underlying_addresses_to_decimals.update( - {x.address: int(math.log10(x.decimal_factor)) for x in TOKEN_SETTINGS.values()} - ) - prices = get_prices(token_decimals=underlying_addresses_to_decimals) - # TODO: move it to separated function END - swap_amms = SwapAmm() - asyncio.run(swap_amms.init()) - - # for pair, state in itertools.product(PAIRS, states): - # protocol = src.protocol_parameters.get_protocol(state=state) - # logging.info( - # f"Preparing main chart data for protocol = {protocol} and pair = {pair}." - # ) - # # TODO: Decipher `pair` in a smarter way. - # collateral_token_underlying_symbol, debt_token_underlying_symbol = pair.split( - # "-" - # ) - # _ = src.main_chart.get_main_chart_data( - # state=state, - # prices=prices, - # swap_amms=swap_amms, - # collateral_token_underlying_symbol=collateral_token_underlying_symbol, - # debt_token_underlying_symbol=debt_token_underlying_symbol, - # save_data=True, - # ) - # logging.info( - # f"Main chart data for protocol = {protocol} and pair = {pair} prepared in {time.time() - t3}s" - # ) - - loan_stats = {} - for state in states: - protocol = get_protocol(state=state) - loan_stats[protocol] = get_loans_table_data( - state=state, prices=prices + logger.info("Underlying addresses to decimals set.") + + def _get_collateral_stats(self) -> dict: + """ + Get the collateral stats. + :return: dict + """ + logger.info("Getting collateral stats.") + collateral_stats = get_collateral_stats(states=self.states) + logger.info("Collateral stats collected.") + return collateral_stats + + def _get_supply_stats(self) -> dict: + """ + Get the supply stats. + :return: dict + """ + logger.info("Getting supply stats.") + supply_stats = get_supply_stats( + states=self.states, + prices=self.prices, + ) + logger.info("Supply stats collected.") + return supply_stats + + def _get_general_stats(self, loan_stats) -> dict: + """ + Get the general stats. + :return: dict + """ + logger.info("Getting general stats.") + general_stats = get_general_stats(states=self.states, loan_stats=loan_stats) + logger.info("General stats collected.") + return general_stats + + @staticmethod + def _get_utilization_stats( + general_stats: dict, supply_stats: dict, debt_stats: dict + ) -> dict: + """ + Get the utilization stats. + :param general_stats: general_stats dict + :param supply_stats: supply_stats dict + :param debt_stats: debt_stats dict + :return: dict + """ + logger.info("Getting utilization stats.") + utilization_stats = get_utilization_stats( + general_stats=general_stats, + supply_stats=supply_stats, + debt_stats=debt_stats, ) + logger.info("Utilization stats collected.") + return utilization_stats - general_stats = get_general_stats( - states=states, loan_stats=loan_stats - ) - supply_stats = get_supply_stats( - states=states, - prices=prices, - ) - collateral_stats = get_collateral_stats(states=states) - debt_stats = get_debt_stats(states=states) - utilization_stats = get_utilization_stats( - general_stats=general_stats, - supply_stats=supply_stats, - debt_stats=debt_stats, - ) - return zklend_state, general_stats, supply_stats, collateral_stats, debt_stats, utilization_stats + def _get_debt_stats(self) -> dict: + """ + Get the debt stats. + :return: dict + """ + logger.info("Getting debt stats.") + debt_stats = get_debt_stats(states=self.states) + logger.info("Debt stats collected.") + return debt_stats + + def _get_loan_stats(self) -> dict: + """ + Get the loan stats. + :return: dict + """ + logger.info("Getting loan stats.") + loan_stats = {} + for state in self.states: + protocol = get_protocol(state=state) + loan_stats[protocol] = get_loans_table_data(state=state, prices=self.prices) + logger.info("Loan stats collected.") + return loan_stats + + def load_data(self) -> dict: + """ + Get the dashboard data. + :return: dict - The dashboard data. + """ + logger.info("Getting dashboard data.") + # Get token parameters. + self._collect_token_parameters() + # Set the underlying addresses to decimals. + self._set_underlying_addresses_to_decimals() + # Set the prices. + self._set_prices() + + # Get the loan stats. + loan_stats = self._get_loan_stats() + + # Get the general stats. + general_stats = self._get_general_stats(loan_stats=loan_stats) + # Get the supply stats. + supply_stats = self._get_supply_stats() + # Get the collateral stats. + collateral_stats = self._get_collateral_stats() + # Get the debt stats. + debt_stats = self._get_debt_stats() + # Get the utilization stats. + utilization_stats = self._get_utilization_stats( + general_stats=general_stats, + supply_stats=supply_stats, + debt_stats=debt_stats, + ) + return ( + self.zklend_state, + general_stats, + supply_stats, + collateral_stats, + debt_stats, + utilization_stats, + ) From b075f2f59748647ea890592883e6f62cafad0ea3 Mon Sep 17 00:00:00 2001 From: djeck1432 Date: Sat, 23 Nov 2024 22:38:34 +0100 Subject: [PATCH 28/31] refactoring --- apps/dashboard_app/helpers/loans_table.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/apps/dashboard_app/helpers/loans_table.py b/apps/dashboard_app/helpers/loans_table.py index dd425a52..c71a0178 100644 --- a/apps/dashboard_app/helpers/loans_table.py +++ b/apps/dashboard_app/helpers/loans_table.py @@ -22,8 +22,13 @@ def get_protocol(state: State) -> str: def get_loans_table_data( state: State, prices: Prices, - save_data: bool = False, ) -> pd.DataFrame: + """ + Get the loans table data. + :param state: ZkLendState | NostraAlphaState | NostraMainnetState + :param prices: Prices + :return: DataFrame + """ data = [] for loan_entity_id, loan_entity in state.loan_entities.items(): collateral_usd = loan_entity.compute_collateral_usd( From c7bda51824bdf4e3d92f320ff3fb60fd480cfc6e Mon Sep 17 00:00:00 2001 From: djeck1432 Date: Sat, 23 Nov 2024 22:38:49 +0100 Subject: [PATCH 29/31] rework get price --- apps/dashboard_app/helpers/tools.py | 58 +++++++++++++++++++---------- 1 file changed, 39 insertions(+), 19 deletions(-) diff --git a/apps/dashboard_app/helpers/tools.py b/apps/dashboard_app/helpers/tools.py index 6afcd13c..92062443 100644 --- a/apps/dashboard_app/helpers/tools.py +++ b/apps/dashboard_app/helpers/tools.py @@ -7,6 +7,7 @@ import requests from google.cloud.storage import Client from shared.blockchain_call import func_call +from shared.constants import TOKEN_SETTINGS from shared.types import TokenParameters from starknet_py.cairo.felt import decode_shortstring @@ -106,32 +107,51 @@ async def get_symbol(token_address: str) -> str: def get_prices(token_decimals: dict[str, int]) -> dict[str, float]: + """ + Get the prices of the tokens. + :param token_decimals: Token decimals. + :return: Dict with token addresses as keys and token prices as values. + """ URL = "https://starknet.impulse.avnu.fi/v1/tokens/short" response = requests.get(URL) - if response.ok: - # Fetch information about tokens. - tokens_info = response.json() + if not response.ok: + response.raise_for_status() - prices = {} - for token, decimals in token_decimals.items(): - token_info = list( - filter( - lambda x: (add_leading_zeros(x["address"]) == token), tokens_info - ) - ) + tokens_info = response.json() - # Remove duplicates. - token_info = [dict(y) for y in {tuple(x.items()) for x in token_info}] + # Define the addresses for which you do not want to apply add_leading_zeros + skip_leading_zeros_addresses = { + TOKEN_SETTINGS["STRK"].address, + } - # Perform sanity checks. - assert len(token_info) == 1 - assert decimals == token_info[0]["decimals"] + # Create a map of token addresses to token information, applying add_leading_zeros conditionally + token_info_map = { + ( + token["address"] + if token["address"] in skip_leading_zeros_addresses + else add_leading_zeros(token["address"]) + ): token + for token in tokens_info + } - prices[token] = token_info[0]["currentPrice"] - return prices - else: - response.raise_for_status() + prices = {} + for token, decimals in token_decimals.items(): + token_info = token_info_map.get(token) + + if not token_info: + logging.error(f"Token {token} not found in response.") + continue + + if decimals != token_info.get("decimals"): + logging.error( + f"Decimal mismatch for token {token}: expected {decimals}, got {token_info.get('decimals')}" + ) + continue + + prices[token] = token_info.get("currentPrice") + + return prices def upload_file_to_bucket(source_path: str, target_path: str): From 7bc66790f4e733e4f1248a20b368000c82b18f2b Mon Sep 17 00:00:00 2001 From: djeck1432 Date: Sat, 23 Nov 2024 22:56:07 +0100 Subject: [PATCH 30/31] remove unused import --- apps/dashboard_app/charts/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/dashboard_app/charts/main.py b/apps/dashboard_app/charts/main.py index d0beeba5..48995625 100644 --- a/apps/dashboard_app/charts/main.py +++ b/apps/dashboard_app/charts/main.py @@ -13,7 +13,6 @@ transform_loans_data, transform_main_chart_data, ) -from dashboard_app.helpers.load_data import set_data class Dashboard: From 5bf8ca57333d77ebc4c8dd386e267e15de82f414 Mon Sep 17 00:00:00 2001 From: djeck1432 Date: Sat, 23 Nov 2024 22:56:36 +0100 Subject: [PATCH 31/31] refactoring --- apps/dashboard_app/helpers/protocol_stats.py | 88 +++++++++++++------- 1 file changed, 56 insertions(+), 32 deletions(-) diff --git a/apps/dashboard_app/helpers/protocol_stats.py b/apps/dashboard_app/helpers/protocol_stats.py index 2efb9c95..a1ab6d70 100644 --- a/apps/dashboard_app/helpers/protocol_stats.py +++ b/apps/dashboard_app/helpers/protocol_stats.py @@ -1,5 +1,6 @@ import asyncio from collections import defaultdict +from decimal import Decimal import pandas as pd from data_handler.handlers import blockchain_call @@ -18,6 +19,12 @@ def get_general_stats( states: list[State], loan_stats: dict[str, pd.DataFrame], ) -> pd.DataFrame: + """ + Get general stats for the dashboard. + :param states: States zklend, nostra_alpha, nostra_mainnet + :param loan_stats: Loan stats data + :return: DataFrame with general stats + """ data = [] for state in states: protocol = get_protocol(state=state) @@ -51,6 +58,12 @@ def get_supply_stats( states: list[State], prices: Prices, ) -> pd.DataFrame: + """ + Get supply stats for the dashboard. + :param states: States zklend, nostra_alpha, nostra_mainnet + :param prices: Prices dict + :return: DataFrame with supply stats + """ data = [] for state in states: protocol = get_protocol(state=state) @@ -77,26 +90,30 @@ def get_supply_stats( )[0] supply = supply / TOKEN_SETTINGS[token].decimal_factor token_supplies[token] = round(supply, 4) + + default_value = Decimal(0.0) data.append( { "Protocol": protocol, - "ETH supply": token_supplies["ETH"], - "WBTC supply": token_supplies["WBTC"], - "USDC supply": token_supplies["USDC"], - "DAI supply": token_supplies["DAI"], - "USDT supply": token_supplies["USDT"], - "wstETH supply": token_supplies["wstETH"], - "LORDS supply": token_supplies["LORDS"], - "STRK supply": token_supplies["STRK"], + "ETH supply": token_supplies.get("ETH", default_value), + "wBTC supply": token_supplies.get("wBTC", default_value), + "USDC supply": token_supplies.get("USDC", default_value), + # FIXME Uncomment when wBTC is added correct address + # "DAI supply": token_supplies.get("DAI", default_value), + "USDT supply": token_supplies.get("USDT", default_value), + "wstETH supply": token_supplies.get("wstETH", default_value), + "LORDS supply": token_supplies.get("LORDS", default_value), + "STRK supply": token_supplies.get("STRK", default_value), } ) - data = pd.DataFrame(data) - data["Total supply (USD)"] = sum( - data[column] * prices[TOKEN_SETTINGS[column.replace(" supply", "")].address] - for column in data.columns + df = pd.DataFrame(data) + df["Total supply (USD)"] = sum( + df[column] + * Decimal(prices[TOKEN_SETTINGS[column.replace(" supply", "")].address]) + for column in df.columns if "supply" in column ).apply(lambda x: round(x, 4)) - return data + return df def get_collateral_stats( @@ -123,21 +140,24 @@ def get_collateral_stats( else: raise ValueError for token_address in token_addresses: - collateral = ( - sum( - float(loan_entity.collateral[token_address]) - for loan_entity in state.loan_entities.values() + try: + collateral = ( + sum( + float(loan_entity.collateral.values.get(token_address, 0.0)) + for loan_entity in state.loan_entities.values() + ) + / TOKEN_SETTINGS[token].decimal_factor + * float(state.interest_rate_models.collateral[token_address]) ) - / TOKEN_SETTINGS[token].decimal_factor - * float(state.interest_rate_models.collateral[token_address]) - ) - token_collaterals[token] += round(collateral, 4) - + token_collaterals[token] += round(collateral, 4) + except TypeError: + # FIXME Remove when all tokens are added + token_collaterals[token] = Decimal(0.0) data.append( { "Protocol": protocol, "ETH collateral": token_collaterals["ETH"], - "WBTC collateral": token_collaterals["WBTC"], + "wBTC collateral": token_collaterals["wBTC"], "USDC collateral": token_collaterals["USDC"], "DAI collateral": token_collaterals["DAI"], "USDT collateral": token_collaterals["USDT"], @@ -173,15 +193,19 @@ def get_debt_stats( else: raise ValueError for token_address in token_addresses: - debt = ( - sum( - float(loan_entity.debt[token_address]) - for loan_entity in state.loan_entities.values() + try: + debt = ( + sum( + float(loan_entity.debt[token_address]) + for loan_entity in state.loan_entities.values() + ) + / TOKEN_SETTINGS[token].decimal_factor + * float(state.interest_rate_models.debt[token_address]) ) - / TOKEN_SETTINGS[token].decimal_factor - * float(state.interest_rate_models.debt[token_address]) - ) - token_debts[token] = round(debt, 4) + token_debts[token] = round(debt, 4) + except TypeError: + # FIXME Remove when all tokens are added + token_debts[token] = Decimal(0.0) data.append( { @@ -213,7 +237,7 @@ def get_utilization_stats( "ETH utilization": debt_stats["ETH debt"] / (supply_stats["ETH supply"] + debt_stats["ETH debt"]), "WBTC utilization": debt_stats["WBTC debt"] - / (supply_stats["WBTC supply"] + debt_stats["WBTC debt"]), + / (supply_stats.get("WBTC supply") + debt_stats.get("WBTC debt")), "USDC utilization": debt_stats["USDC debt"] / (supply_stats["USDC supply"] + debt_stats["USDC debt"]), "DAI utilization": debt_stats["DAI debt"]