From ab1bb299e502f3b9066a163d236c20554ff21c46 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Sat, 21 Dec 2024 02:13:40 -0400 Subject: [PATCH 1/4] feat: reduce default transaction workers 10k -> 1k --- eth_portfolio/address.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth_portfolio/address.py b/eth_portfolio/address.py index e12e6463..9987e68d 100644 --- a/eth_portfolio/address.py +++ b/eth_portfolio/address.py @@ -55,7 +55,7 @@ def __init__( address: Address, start_block: Block, load_prices: bool, - num_workers_transactions: int = 10_000, + num_workers_transactions: int = 1000, asynchronous: bool = False, ) -> None: # type: ignore """ From bcc6c6924ce2f0ea170c81579834eb68284f5a87 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 21 Dec 2024 06:14:07 +0000 Subject: [PATCH 2/4] chore: `black .` --- eth_portfolio/_cache.py | 6 ++++-- eth_portfolio/_db/utils.py | 2 +- eth_portfolio/_ledgers/address.py | 5 ++++- eth_portfolio/_loaders/token_transfer.py | 5 ++--- eth_portfolio/_loaders/transaction.py | 14 ++++++++------ 5 files changed, 19 insertions(+), 13 deletions(-) diff --git a/eth_portfolio/_cache.py b/eth_portfolio/_cache.py index 01fe4edb..35b55d12 100644 --- a/eth_portfolio/_cache.py +++ b/eth_portfolio/_cache.py @@ -38,8 +38,10 @@ def get_cache_file_path(args, kwargs): makedirs(cache_path_for_fn, exist_ok=True) if inspect.iscoroutinefunction(fn): - read_executor = PruningThreadPoolExecutor(8, f"{_THREAD_NAME_PREFIX}-{fn.__qualname__}-read") - + read_executor = PruningThreadPoolExecutor( + 8, f"{_THREAD_NAME_PREFIX}-{fn.__qualname__}-read" + ) + queue = PriorityQueue() @log_broken diff --git a/eth_portfolio/_db/utils.py b/eth_portfolio/_db/utils.py index 9efde3d2..c841b3c1 100644 --- a/eth_portfolio/_db/utils.py +++ b/eth_portfolio/_db/utils.py @@ -473,7 +473,7 @@ def token_transfers_known_at_startup() -> Dict[_TokenTransferPK, bytes]: tx_index: int log_index: int raw: bytes - + transfers = {} for chainid, block, tx_index, log_index, raw in select( (t.block.chain.id, t.block.number, t.transaction_index, t.log_index, t.raw) diff --git a/eth_portfolio/_ledgers/address.py b/eth_portfolio/_ledgers/address.py index 4b5321bb..93dce50c 100644 --- a/eth_portfolio/_ledgers/address.py +++ b/eth_portfolio/_ledgers/address.py @@ -592,6 +592,7 @@ async def get_transaction_status(txhash: str) -> Status: _trace_semaphores = defaultdict(lambda: a_sync.Semaphore(16, __name__ + ".trace_semaphore")) + @cache_to_disk @eth_retry.auto_retry async def get_traces(filter_params: TraceFilterParams) -> List[FilterTrace]: @@ -607,7 +608,9 @@ async def get_traces(filter_params: TraceFilterParams) -> List[FilterTrace]: The list of traces. """ if chain.id == Network.Polygon: - logger.warning("polygon doesnt support trace_filter method, must develop alternate solution") + logger.warning( + "polygon doesnt support trace_filter method, must develop alternate solution" + ) return [] semaphore_key = tuple( sorted(tuple(filter_params.get(x, ("",)) for x in ("toAddress", "fromAddress"))) diff --git a/eth_portfolio/_loaders/token_transfer.py b/eth_portfolio/_loaders/token_transfer.py index 68786af2..3ba81081 100644 --- a/eth_portfolio/_loaders/token_transfer.py +++ b/eth_portfolio/_loaders/token_transfer.py @@ -29,7 +29,7 @@ logger = getLogger(__name__) token_transfer_semaphore = BlockSemaphore( - 20_000, # Some arbitrary number + 20_000, # Some arbitrary number name="eth_portfolio.token_transfers", ) """A semaphore that regulates the concurrent processing of token transfers by processing lower blocks first.""" @@ -104,7 +104,7 @@ async def load_token_transfer( ) else: coro_results["transaction_index"] = await tx_index_coro - + except Exception as e: logger.error( f"%s %s for %s %s at block %s:", @@ -208,7 +208,6 @@ async def get_transaction_index(hash: str) -> int: new = TypeError(e, receipt_bytes, decode(receipt_bytes)) logger.exception(new) raise new from e - class HasTxIndex(Struct): diff --git a/eth_portfolio/_loaders/transaction.py b/eth_portfolio/_loaders/transaction.py index 1a608afc..9a09f20f 100644 --- a/eth_portfolio/_loaders/transaction.py +++ b/eth_portfolio/_loaders/transaction.py @@ -109,7 +109,9 @@ async def get_block_for_nonce(address: Address, nonce: Nonce) -> int: lowest_known_nonce_greater_than_query = None # it is impossible for n to == nonce - for less_than, ns in itertools.groupby(filter(lambda n: n != nonce, nonces[address]), lambda n: n < nonce): + for less_than, ns in itertools.groupby( + filter(lambda n: n != nonce, nonces[address]), lambda n: n < nonce + ): if less_than: max_value = max(ns) if ( @@ -172,8 +174,8 @@ async def get_block_for_nonce(address: Address, nonce: Nonce) -> int: lo += int((hi - lo) / 2) or 1 if debug_logs_enabled: logger._log( - DEBUG, - "Nonce for %s at %s is %s, checking higher block %s", + DEBUG, + "Nonce for %s at %s is %s, checking higher block %s", (address, old_lo, _nonce, lo), ) continue @@ -184,15 +186,15 @@ async def get_block_for_nonce(address: Address, nonce: Nonce) -> int: lo = int(lo / 2) if debug_logs_enabled: logger._log( - DEBUG, - "Nonce for %s at %s is %s, checking lower block %s", + DEBUG, + "Nonce for %s at %s is %s, checking lower block %s", (address, hi, _nonce, lo), ) continue if debug_logs_enabled: logger._log(DEBUG, "Found nonce %s for %s at block %s", (nonce, address, lo)) - + return lo From 6c013ad63532f028abb7a0c467f2a0a6a60f5b52 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Sat, 21 Dec 2024 02:14:47 -0400 Subject: [PATCH 3/4] Update portfolio.py --- eth_portfolio/portfolio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth_portfolio/portfolio.py b/eth_portfolio/portfolio.py index 624cf559..ed3a24ec 100644 --- a/eth_portfolio/portfolio.py +++ b/eth_portfolio/portfolio.py @@ -206,7 +206,7 @@ def __init__( start_block: int = 0, label: str = _DEFAULT_LABEL, load_prices: bool = True, - num_workers_transactions: int = 10_000, + num_workers_transactions: int = 1000, asynchronous: bool = False, ) -> None: """ From a04b1b62adf4323777705e085a85325a7add24cc Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Sat, 21 Dec 2024 02:14:57 -0400 Subject: [PATCH 4/4] Update address.py (#168) --- eth_portfolio/_ledgers/address.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth_portfolio/_ledgers/address.py b/eth_portfolio/_ledgers/address.py index 93dce50c..abf519da 100644 --- a/eth_portfolio/_ledgers/address.py +++ b/eth_portfolio/_ledgers/address.py @@ -408,7 +408,7 @@ class AddressTransactionsLedger(AddressLedgerBase[TransactionsList, Transaction] _list_type = TransactionsList __slots__ = ("cached_thru_nonce", "_queue", "_ready", "_num_workers", "_workers") - def __init__(self, portfolio_address: "PortfolioAddress", num_workers: int = 10_000): + def __init__(self, portfolio_address: "PortfolioAddress", num_workers: int = 1000): """ Initializes the AddressTransactionsLedger instance.