From 314aa8f86979b806c12ab0d095761918bbe28da2 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Thu, 7 Sep 2023 18:51:20 +0200 Subject: [PATCH 001/124] add rewards --- .../cardano_account_pandas_dumper.py | 124 +++++++++++------- 1 file changed, 78 insertions(+), 46 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index f894bce..dd078c6 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -14,6 +14,7 @@ class AccountData: LOVELACE_ASSET = "lovelace" LOVELACE_DECIMALS = 6 + REWARD_PREFIX = "_REWARD_" # Fake tx hash for rewards starts with this. def __init__( self, @@ -51,39 +52,45 @@ def _load_transaction_hashes(self, api: BlockFrostApi) -> List[str]: t.tx_hash for t in sorted( itertools.chain( - *[ - api.address_transactions( - a, - to_block=self.to_block, - gather_pages=True, - ) - for a in self.own_addresses - ], + *( + [ + api.address_transactions( + a, + to_block=self.to_block, + gather_pages=True, + ) + for a in self.own_addresses + ] + + [self._reward_transactions(api=api)] + ), ), - key=lambda x: (x.block_height, x.tx_index), + key=lambda x: (x.block_time, x.tx_index), ) ], ) ) - @staticmethod def _load_transaction_data( - api: BlockFrostApi, tx_hashes: List[str] + self, api: BlockFrostApi, tx_hashes: List[str] ) -> OrderedDict[str, Namespace]: result = OrderedDict() for tx_hash in tx_hashes: - transaction = api.transaction(tx_hash) - transaction.utxos = api.transaction_utxos(tx_hash) - transaction.metadata = api.transaction_metadata(tx_hash) - transaction.redeemers = ( - api.transaction_redeemers(tx_hash) if transaction.redeemer_count else [] - ) - transaction.withdrawals = ( - api.transaction_withdrawals(tx_hash) - if transaction.withdrawal_count - else [] - ) - result[tx_hash] = transaction + if not tx_hash.startswith(self.REWARD_PREFIX): + transaction = api.transaction(tx_hash) + transaction.utxos = api.transaction_utxos(tx_hash) + transaction.metadata = api.transaction_metadata(tx_hash) + transaction.redeemers = ( + api.transaction_redeemers(tx_hash) + if transaction.redeemer_count + else [] + ) + transaction.withdrawals = ( + api.transaction_withdrawals(tx_hash) + if transaction.withdrawal_count + else [] + ) + transaction.reward_amount = None + result[tx_hash] = transaction return result def _collect_from_transactions(self, api: BlockFrostApi) -> None: @@ -93,18 +100,40 @@ def _collect_from_transactions(self, api: BlockFrostApi) -> None: self.addresses: Dict[str, Namespace] = {} all_addresses: Set[str] = set(self.own_addresses) for tx_obj in self.transactions.values(): - all_asset_ids.update( - [a.unit for i in tx_obj.utxos.inputs for a in i.amount] - + [a.unit for i in tx_obj.utxos.outputs for a in i.amount] - ) - all_addresses.update( - [i.address for i in tx_obj.utxos.inputs] - + [i.address for i in tx_obj.utxos.outputs] - ) + if not tx_obj.reward_amount: + all_asset_ids.update( + [a.unit for i in tx_obj.utxos.inputs for a in i.amount] + + [a.unit for i in tx_obj.utxos.outputs for a in i.amount] + ) + all_addresses.update( + [i.address for i in tx_obj.utxos.inputs] + + [i.address for i in tx_obj.utxos.outputs] + ) all_asset_ids.remove(self.LOVELACE_ASSET) for asset in all_asset_ids: self.assets[asset] = api.asset(asset) + def _reward_transaction(self, api: BlockFrostApi, reward: Namespace) -> Namespace: + result = Namespace() + result.tx_hash = result.hash = self.REWARD_PREFIX + str(reward.epoch) + result.Metadata = Namespace() + result.Metadata.message = f"Reward: {reward.type}" + result.reward_amount = reward.amount + epoch = api.epoch(reward.epoch) + result.block_time = epoch.end_time + result.fees = "0" + result.deposit = "0" + result.tx_index = 0 + result.redeemers = [] + return result + + def _reward_transactions(self, api: BlockFrostApi) -> List[Namespace]: + return [ + self._reward_transaction(api=api, reward=r) + for s in self.staking_addresses + for r in api.account_rewards(s, gather_pages=True) + ] + class AccountPandasDumper: """Hold logic to convert an instance of AccountData to a Pandas dataframe.""" @@ -304,30 +333,33 @@ def transaction_dict(self, transaction: Namespace) -> Optional[Dict]: (" fees", int(transaction.fees), self.data.LOVELACE_DECIMALS), ("deposit", int(transaction.deposit), self.data.LOVELACE_DECIMALS), ( - "withdrawal_sum", - sum( + "rewards", + -sum( [int(w.amount) for w in transaction.withdrawals], - ), + ) + if not transaction.reward_amount + else transaction.reward_amount, self.data.LOVELACE_DECIMALS, ), ] ] ) balance_result: Dict = defaultdict(lambda: Decimal(0)) + if not transaction.reward_amount: + # Reward pseudo transactions have no utxos + for i in transaction.utxos.nonref_inputs: + if not i.collateral or not transaction.valid_contract: + for amount in i.amount: + key = self._utxo_amount_key(i, amount) + if key is not None: + balance_result[key] -= Decimal(amount.quantity) - for i in transaction.utxos.nonref_inputs: - if not i.collateral or not transaction.valid_contract: - for amount in i.amount: - key = self._utxo_amount_key(i, amount) + for out in transaction.utxos.outputs: + for amount in out.amount: + key = self._utxo_amount_key(out, amount) if key is not None: - balance_result[key] -= Decimal(amount.quantity) - - for out in transaction.utxos.outputs: - for amount in out.amount: - key = self._utxo_amount_key(out, amount) - if key is not None: - balance_result[key] += Decimal(amount.quantity) - result.update({k: v for k, v in balance_result.items() if v != 0}) + balance_result[key] += Decimal(amount.quantity) + result.update({k: v for k, v in balance_result.items() if v != 0}) return result def make_transaction_array(self, to_block: int) -> pandas.DataFrame: From c268d7fad6adc37b3c664874a3e987f75f36edb0 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Fri, 8 Sep 2023 12:34:52 +0200 Subject: [PATCH 002/124] add rewards arg --- src/cardano_account_pandas_dumper/__main__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/cardano_account_pandas_dumper/__main__.py b/src/cardano_account_pandas_dumper/__main__.py index ac867d3..0118b86 100644 --- a/src/cardano_account_pandas_dumper/__main__.py +++ b/src/cardano_account_pandas_dumper/__main__.py @@ -86,6 +86,11 @@ def _create_arg_parser(): help="Add header row with concatenation of policy_id and hex-encoded asset_name.", action="store_true", ) + result.add_argument( + "--no_rewards", + help="Do not add reward transactions.", + action="store_true", + ) return result @@ -132,6 +137,7 @@ def main(): api=api_instance, staking_addresses=staking_addresses_set, to_block=args.to_block, + rewards=not args.no_rewards, ) except (ApiError, JSONDecodeError, OSError) as exception: parser.exit( @@ -157,6 +163,7 @@ def main(): unmute=args.unmute, truncate_length=None if args.no_truncate else args.truncate_length, raw_asset=args.raw_asset or False, + rewards=data_from_api.rewards, ) dataframe = reporter.make_transaction_array(args.to_block or data_from_api.to_block) if args.pandas_output: From 52ab8a82feac054402cc173cf1193e5ae24a9905 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Fri, 8 Sep 2023 15:01:54 +0200 Subject: [PATCH 003/124] pandaize, S1E1 --- src/cardano_account_pandas_dumper/__main__.py | 9 +- .../cardano_account_pandas_dumper.py | 260 +++++++++--------- 2 files changed, 131 insertions(+), 138 deletions(-) diff --git a/src/cardano_account_pandas_dumper/__main__.py b/src/cardano_account_pandas_dumper/__main__.py index 0118b86..a9dee1d 100644 --- a/src/cardano_account_pandas_dumper/__main__.py +++ b/src/cardano_account_pandas_dumper/__main__.py @@ -122,11 +122,10 @@ def main(): ), status=1, ) - if args.to_block is not None and args.to_block > data_from_api.to_block: + if args.to_block is not None: parser.exit( message=( - f"Specified to_block {args.to_block} for report, " - + f"available data only to block {data_from_api.to_block}" + "--to_block not allowed with --from_checkpoint (to_block always taken from checkpoint)." ), status=1, ) @@ -165,7 +164,7 @@ def main(): raw_asset=args.raw_asset or False, rewards=data_from_api.rewards, ) - dataframe = reporter.make_transaction_array(args.to_block or data_from_api.to_block) + dataframe = reporter.make_transaction_array() if args.pandas_output: try: dataframe.to_pickle(args.pandas_output) @@ -180,7 +179,7 @@ def main(): if args.csv_output: try: - dataframe.to_csv(args.csv_output, index=False) + dataframe.to_csv(args.csv_output, index=True) except OSError as exception: warnings.warn(f"Failed to write CSV file: {exception}") print("Done.") diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index dd078c6..b576c6c 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -1,10 +1,11 @@ """ Cardano Account To Pandas Dumper.""" import datetime import itertools -from collections import OrderedDict, defaultdict +from collections import defaultdict from decimal import Context, Decimal -from typing import Any, Dict, FrozenSet, List, Mapping, Optional, Set, Tuple -import pandas +from typing import Any, Dict, FrozenSet, Iterable, List, Mapping, Optional, Set, Tuple +import pandas as pd +import numpy as np from blockfrost import BlockFrostApi from blockfrost.utils import Namespace @@ -14,25 +15,29 @@ class AccountData: LOVELACE_ASSET = "lovelace" LOVELACE_DECIMALS = 6 - REWARD_PREFIX = "_REWARD_" # Fake tx hash for rewards starts with this. + TRANSACTION_OFFSET = np.timedelta64( + 1000, "ns" + ) # Time for a stransaction is block_time + index * TRANSACTION_OFFSET def __init__( self, api: BlockFrostApi, staking_addresses: FrozenSet[str], to_block: Optional[int], + rewards: bool, ) -> None: self.staking_addresses = staking_addresses if to_block is None: to_block = int(api.block_latest().height) self.to_block = to_block - self.own_addresses = self._load_own_addresses(api) - self.transactions = self._load_transaction_data( - api, self._load_transaction_hashes(api) - ) - self._collect_from_transactions(api) + self.own_addresses: FrozenSet[str] = self._own_addresses(api) + self.rewards = rewards + if self.rewards: + self.reward_transactions: pd.Series = self._reward_transactions(api) + self.transactions: pd.Series = self._transaction_data(api) + self.assets: pd.Series = self._assets_from_transactions(api) - def _load_own_addresses(self, api: BlockFrostApi) -> FrozenSet[str]: + def _own_addresses(self, api: BlockFrostApi) -> FrozenSet[str]: return frozenset( [ a.address @@ -45,94 +50,93 @@ def _load_own_addresses(self, api: BlockFrostApi) -> FrozenSet[str]: ] ) - def _load_transaction_hashes(self, api: BlockFrostApi) -> List[str]: - return list( - OrderedDict.fromkeys( - [ - t.tx_hash - for t in sorted( - itertools.chain( - *( - [ - api.address_transactions( - a, - to_block=self.to_block, - gather_pages=True, - ) - for a in self.own_addresses - ] - + [self._reward_transactions(api=api)] - ), - ), - key=lambda x: (x.block_time, x.tx_index), - ) - ], - ) - ) - - def _load_transaction_data( - self, api: BlockFrostApi, tx_hashes: List[str] - ) -> OrderedDict[str, Namespace]: - result = OrderedDict() - for tx_hash in tx_hashes: - if not tx_hash.startswith(self.REWARD_PREFIX): - transaction = api.transaction(tx_hash) - transaction.utxos = api.transaction_utxos(tx_hash) - transaction.metadata = api.transaction_metadata(tx_hash) + def _transaction_data( + self, + api: BlockFrostApi, + ) -> pd.Series: + result_list = [] + for addr in self.own_addresses: + for outer_tx in api.address_transactions( + addr, + to_block=self.to_block, + gather_pages=True, + ): + transaction = api.transaction(outer_tx.tx_hash) + transaction.utxos = api.transaction_utxos(outer_tx.tx_hash) + transaction.utxos.nonref_inputs = [ + i for i in transaction.utxos.inputs if not i.reference + ] + transaction.metadata = api.transaction_metadata(outer_tx.tx_hash) transaction.redeemers = ( - api.transaction_redeemers(tx_hash) + api.transaction_redeemers(outer_tx.tx_hash) if transaction.redeemer_count else [] ) transaction.withdrawals = ( - api.transaction_withdrawals(tx_hash) + api.transaction_withdrawals(outer_tx.tx_hash) if transaction.withdrawal_count else [] ) transaction.reward_amount = None - result[tx_hash] = transaction - return result - def _collect_from_transactions(self, api: BlockFrostApi) -> None: - self.assets: Dict[str, Namespace] = {} + result_list.append(transaction) + index = pd.DatetimeIndex( + [ + np.datetime64(datetime.datetime.fromtimestamp(t.block_time)) + + (int(t.index) * self.TRANSACTION_OFFSET) + for t in result_list + ], + ) + return pd.Series(name="Transactions", data=result_list, index=index) + def _assets_from_transactions(self, api: BlockFrostApi) -> pd.Series: all_asset_ids: Set[str] = set() - self.addresses: Dict[str, Namespace] = {} - all_addresses: Set[str] = set(self.own_addresses) - for tx_obj in self.transactions.values(): - if not tx_obj.reward_amount: + for tx_obj in self.transactions: + if hasattr(tx_obj, "utxos"): all_asset_ids.update( [a.unit for i in tx_obj.utxos.inputs for a in i.amount] + [a.unit for i in tx_obj.utxos.outputs for a in i.amount] ) - all_addresses.update( - [i.address for i in tx_obj.utxos.inputs] - + [i.address for i in tx_obj.utxos.outputs] - ) - all_asset_ids.remove(self.LOVELACE_ASSET) - for asset in all_asset_ids: - self.assets[asset] = api.asset(asset) + all_asset_ids.remove(AccountData.LOVELACE_ASSET) + return pd.Series( + name="Assets", data={asset: api.asset(asset) for asset in all_asset_ids} + ) - def _reward_transaction(self, api: BlockFrostApi, reward: Namespace) -> Namespace: + @staticmethod + def _reward_transaction(api: BlockFrostApi, reward: Namespace) -> Namespace: result = Namespace() - result.tx_hash = result.hash = self.REWARD_PREFIX + str(reward.epoch) + result.tx_hash = None result.Metadata = Namespace() - result.Metadata.message = f"Reward: {reward.type}" + result.Metadata.message = f"Reward: {reward.type} - {reward.epoch}" result.reward_amount = reward.amount - epoch = api.epoch(reward.epoch) - result.block_time = epoch.end_time + epoch = api.epoch(reward.epoch + 1) # Time is right before start of next epoch. + result.block_time = epoch.start_time + result.index = -1 result.fees = "0" result.deposit = "0" - result.tx_index = 0 result.redeemers = [] + result.hash = None + result.withdrawals = [] + result.utxos = Namespace() + result.utxos.inputs = [] + result.utxos.outputs = [] + result.utxos.nonref_inputs = [] return result - def _reward_transactions(self, api: BlockFrostApi) -> List[Namespace]: - return [ - self._reward_transaction(api=api, reward=r) - for s in self.staking_addresses - for r in api.account_rewards(s, gather_pages=True) + def _reward_transactions(self, api: BlockFrostApi) -> pd.Series: + result_list = [ + self._reward_transaction(api=api, reward=a_r) + for s_a in self.staking_addresses + for a_r in api.account_rewards(s_a, gather_pages=True) ] + index = pd.DatetimeIndex( + [ + np.datetime64(datetime.datetime.fromtimestamp(t.block_time)) + + (t.index * self.TRANSACTION_OFFSET) + for t in result_list + ], + ) + return pd.Series(name="Rewards", data=result_list, index=index) class AccountPandasDumper: @@ -153,6 +157,7 @@ def __init__( unmute: bool, truncate_length: Optional[int], raw_asset: bool, + rewards: bool, ): self.known_dict = known_dict self.data = data @@ -160,11 +165,8 @@ def __init__( self.detail_level = detail_level self.unmute = unmute self.raw_asset = raw_asset + self.rewards = rewards self.decimal_context = Context() - for tx_obj in self.data.transactions.values(): - tx_obj.utxos.nonref_inputs = [ - i for i in tx_obj.utxos.inputs if not i.reference - ] def _format_asset(self, asset: str) -> Optional[str]: if asset == AccountData.LOVELACE_ASSET: @@ -309,75 +311,67 @@ def _utxo_amount_key(self, utxo: Namespace, amount: Namespace) -> Optional[Tuple else (asset, amount.unit, addr, self._decimals_for_asset(amount.unit)) ) - def transaction_dict(self, transaction: Namespace) -> Optional[Dict]: + def transaction_dict(self) -> Iterable[Dict]: """Return a dict holding a Pandas row for transaction tx""" - result: Dict = OrderedDict( - [ - ( - (" metadata", k, d) - if not self.raw_asset - else (" metadata", k, "", d), - v, - ) - for k, v, d in [ + result_list = [] + for transaction in self.data.transactions: + result: Dict = dict( + [ ( - " block_time", - # Make sure time monotonically increases by adding tx index to block time - datetime.datetime.fromtimestamp( - int(transaction.block_time) + int(transaction.index) + (" metadata", k, d) + if not self.raw_asset + else (" metadata", k, "", d), + v, + ) + for k, v, d in [ + (" hash", transaction.hash, 0), + (" message", self._format_message(transaction), 0), + (" fees", int(transaction.fees), self.data.LOVELACE_DECIMALS), + ( + "deposit", + int(transaction.deposit), + self.data.LOVELACE_DECIMALS, ), - 0, - ), - (" hash", transaction.hash, 0), - (" message", self._format_message(transaction), 0), - (" fees", int(transaction.fees), self.data.LOVELACE_DECIMALS), - ("deposit", int(transaction.deposit), self.data.LOVELACE_DECIMALS), - ( - "rewards", - -sum( - [int(w.amount) for w in transaction.withdrawals], - ) - if not transaction.reward_amount - else transaction.reward_amount, - self.data.LOVELACE_DECIMALS, - ), + ( + "rewards", + -sum( + [int(w.amount) for w in transaction.withdrawals], + ) + if not transaction.reward_amount + else transaction.reward_amount, + self.data.LOVELACE_DECIMALS, + ), + ] ] - ] - ) - balance_result: Dict = defaultdict(lambda: Decimal(0)) - if not transaction.reward_amount: - # Reward pseudo transactions have no utxos - for i in transaction.utxos.nonref_inputs: - if not i.collateral or not transaction.valid_contract: - for amount in i.amount: - key = self._utxo_amount_key(i, amount) - if key is not None: - balance_result[key] -= Decimal(amount.quantity) + ) + balance_result: Dict = defaultdict(lambda: Decimal(0)) + if not transaction.reward_amount: + # Reward pseudo transactions have no utxos + for i in transaction.utxos.nonref_inputs: + if not i.collateral or not transaction.valid_contract: + for amount in i.amount: + key = self._utxo_amount_key(i, amount) + if key is not None: + balance_result[key] -= Decimal(amount.quantity) - for out in transaction.utxos.outputs: - for amount in out.amount: - key = self._utxo_amount_key(out, amount) - if key is not None: - balance_result[key] += Decimal(amount.quantity) - result.update({k: v for k, v in balance_result.items() if v != 0}) - return result + for out in transaction.utxos.outputs: + for amount in out.amount: + key = self._utxo_amount_key(out, amount) + if key is not None: + balance_result[key] += Decimal(amount.quantity) + result.update({k: v for k, v in balance_result.items() if v != 0}) + result_list.append(result) + return result_list - def make_transaction_array(self, to_block: int) -> pandas.DataFrame: + def make_transaction_array(self) -> pd.DataFrame: """Return a dataframe with each transaction until the specified block, included.""" - data = [] - for transaction in self.data.transactions.values(): - if to_block is not None and int(transaction.block_height) > to_block: - break - data.append(self.transaction_dict(transaction)) - frame = pandas.DataFrame( - data=data, - ) + frame = pd.DataFrame(data=self.transaction_dict()) # Scale according to decimal index row, and drop that row for col in frame.columns: if col[-1]: scale = self.decimal_context.power(10, -Decimal(col[-1])) frame[col] *= scale # type: ignore - frame.columns = pandas.MultiIndex.from_tuples([c[:-1] for c in frame.columns]) + frame.columns = pd.MultiIndex.from_tuples([c[:-1] for c in frame.columns]) frame.sort_index(axis=1, level=0, sort_remaining=True, inplace=True) return frame From f5321114070f2289634fdae08316cb723a5b6387 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Fri, 8 Sep 2023 15:06:44 +0200 Subject: [PATCH 004/124] all transactions now have utxos --- .../cardano_account_pandas_dumper.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index b576c6c..ef6a148 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -345,8 +345,6 @@ def transaction_dict(self) -> Iterable[Dict]: ] ) balance_result: Dict = defaultdict(lambda: Decimal(0)) - if not transaction.reward_amount: - # Reward pseudo transactions have no utxos for i in transaction.utxos.nonref_inputs: if not i.collateral or not transaction.valid_contract: for amount in i.amount: From 48b4c2dfd8080527fd96453c91f6a5902b62d9fb Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Fri, 8 Sep 2023 15:35:52 +0200 Subject: [PATCH 005/124] fix long line --- src/cardano_account_pandas_dumper/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cardano_account_pandas_dumper/__main__.py b/src/cardano_account_pandas_dumper/__main__.py index a9dee1d..7f1c0af 100644 --- a/src/cardano_account_pandas_dumper/__main__.py +++ b/src/cardano_account_pandas_dumper/__main__.py @@ -125,7 +125,7 @@ def main(): if args.to_block is not None: parser.exit( message=( - "--to_block not allowed with --from_checkpoint (to_block always taken from checkpoint)." + "--to_block not allowed with --from_checkpoint (always taken from checkpoint)." ), status=1, ) From e6ed419db990215b09b8005d2cfa08999fa290d2 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Fri, 8 Sep 2023 15:46:29 +0200 Subject: [PATCH 006/124] pandaize S1E2 --- .../cardano_account_pandas_dumper.py | 124 ++++++++++-------- 1 file changed, 67 insertions(+), 57 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index ef6a148..4d87688 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -3,7 +3,18 @@ import itertools from collections import defaultdict from decimal import Context, Decimal -from typing import Any, Dict, FrozenSet, Iterable, List, Mapping, Optional, Set, Tuple +from typing import ( + Any, + Dict, + FrozenSet, + Iterable, + List, + Mapping, + Optional, + OrderedDict, + Set, + Tuple, +) import pandas as pd import numpy as np from blockfrost import BlockFrostApi @@ -300,70 +311,69 @@ def _decimals_for_asset(self, asset: str) -> int: def _utxo_amount_key(self, utxo: Namespace, amount: Namespace) -> Optional[Tuple]: if self.detail_level < 2 and utxo.address not in self.data.own_addresses: return None - asset = self._format_asset(amount.unit) - if asset is None: - return None - addr = self._format_address(utxo.address) + return (amount.unit, utxo.address) - return ( - (asset, addr, self._decimals_for_asset(amount.unit)) - if not self.raw_asset - else (asset, amount.unit, addr, self._decimals_for_asset(amount.unit)) - ) - - def transaction_dict(self) -> Iterable[Dict]: + def transaction_metadata(self, transaction: Namespace) -> dict: """Return a dict holding a Pandas row for transaction tx""" - result_list = [] - for transaction in self.data.transactions: - result: Dict = dict( - [ + result: Dict = OrderedDict( + [ + ( + (" metadata", k, d) + if not self.raw_asset + else (" metadata", k, "", d), + v, + ) + for k, v, d in [ + (" hash", transaction.hash, 0), + (" message", self._format_message(transaction), 0), + (" fees", int(transaction.fees), self.data.LOVELACE_DECIMALS), ( - (" metadata", k, d) - if not self.raw_asset - else (" metadata", k, "", d), - v, - ) - for k, v, d in [ - (" hash", transaction.hash, 0), - (" message", self._format_message(transaction), 0), - (" fees", int(transaction.fees), self.data.LOVELACE_DECIMALS), - ( - "deposit", - int(transaction.deposit), - self.data.LOVELACE_DECIMALS, - ), - ( - "rewards", - -sum( - [int(w.amount) for w in transaction.withdrawals], - ) - if not transaction.reward_amount - else transaction.reward_amount, - self.data.LOVELACE_DECIMALS, - ), - ] + "deposit", + int(transaction.deposit), + self.data.LOVELACE_DECIMALS, + ), + ( + "rewards", + -sum( + [int(w.amount) for w in transaction.withdrawals], + ) + if not transaction.reward_amount + else transaction.reward_amount, + self.data.LOVELACE_DECIMALS, + ), ] - ) - balance_result: Dict = defaultdict(lambda: Decimal(0)) - for i in transaction.utxos.nonref_inputs: - if not i.collateral or not transaction.valid_contract: - for amount in i.amount: - key = self._utxo_amount_key(i, amount) - if key is not None: - balance_result[key] -= Decimal(amount.quantity) + ] + ) + return result + + def transaction_outputs(self, transaction: Namespace) -> dict: + result = {} + balance_result: Dict = OrderedDict() + for i in transaction.utxos.nonref_inputs: + if not i.collateral or not transaction.valid_contract: + for amount in i.amount: + key = self._utxo_amount_key(i, amount) + if key is not None: + if key not in balance_result: + balance_result[key] = Decimal(0) + balance_result[key] -= Decimal(amount.quantity) - for out in transaction.utxos.outputs: - for amount in out.amount: - key = self._utxo_amount_key(out, amount) - if key is not None: - balance_result[key] += Decimal(amount.quantity) - result.update({k: v for k, v in balance_result.items() if v != 0}) - result_list.append(result) - return result_list + for out in transaction.utxos.outputs: + for amount in out.amount: + key = self._utxo_amount_key(out, amount) + if key is not None: + if key not in balance_result: + balance_result[key] = Decimal(0) + balance_result[key] += Decimal(amount.quantity) + result.update({k: v for k, v in balance_result.items() if v != 0}) + return result def make_transaction_array(self) -> pd.DataFrame: """Return a dataframe with each transaction until the specified block, included.""" - frame = pd.DataFrame(data=self.transaction_dict()) + + metadata = self.data.transactions.map(self.transaction_metadata) + outputs = self.data.transactions.map(self.transaction_outputs) + frame = pd.DataFrame(data=[metadata, outputs]) # Scale according to decimal index row, and drop that row for col in frame.columns: From 9899f37595808c9b4d94fa7736907732f0f9bece Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Fri, 8 Sep 2023 20:07:31 +0200 Subject: [PATCH 007/124] pandaize S1E3 --- src/cardano_account_pandas_dumper/__main__.py | 7 -- .../cardano_account_pandas_dumper.py | 118 ++++++++---------- 2 files changed, 54 insertions(+), 71 deletions(-) diff --git a/src/cardano_account_pandas_dumper/__main__.py b/src/cardano_account_pandas_dumper/__main__.py index 7f1c0af..0483e5b 100644 --- a/src/cardano_account_pandas_dumper/__main__.py +++ b/src/cardano_account_pandas_dumper/__main__.py @@ -170,13 +170,6 @@ def main(): dataframe.to_pickle(args.pandas_output) except (pickle.PicklingError, OSError) as exception: warnings.warn(f"Failed to write pandas file: {exception}") - # Add total line at the bottom for csv output. - total = ["", "Total", ""] - for column in dataframe.columns[3:]: - # Only NaN is float in the column - total.append(sum([a for a in dataframe[column] if not isinstance(a, float)])) - dataframe.loc["Total"] = total - if args.csv_output: try: dataframe.to_csv(args.csv_output, index=True) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 4d87688..4742428 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -1,5 +1,6 @@ """ Cardano Account To Pandas Dumper.""" import datetime +import functools import itertools from collections import defaultdict from decimal import Context, Decimal @@ -7,13 +8,11 @@ Any, Dict, FrozenSet, - Iterable, List, Mapping, Optional, OrderedDict, Set, - Tuple, ) import pandas as pd import numpy as np @@ -308,78 +307,69 @@ def _decimals_for_asset(self, asset: str) -> int: return self.data.assets[asset].metadata.decimals return 0 - def _utxo_amount_key(self, utxo: Namespace, amount: Namespace) -> Optional[Tuple]: - if self.detail_level < 2 and utxo.address not in self.data.own_addresses: - return None - return (amount.unit, utxo.address) - - def transaction_metadata(self, transaction: Namespace) -> dict: + def _transaction_metadata(self, transaction: Namespace) -> Any: """Return a dict holding a Pandas row for transaction tx""" result: Dict = OrderedDict( - [ - ( - (" metadata", k, d) - if not self.raw_asset - else (" metadata", k, "", d), - v, - ) - for k, v, d in [ - (" hash", transaction.hash, 0), - (" message", self._format_message(transaction), 0), - (" fees", int(transaction.fees), self.data.LOVELACE_DECIMALS), - ( - "deposit", - int(transaction.deposit), - self.data.LOVELACE_DECIMALS, - ), - ( - "rewards", - -sum( - [int(w.amount) for w in transaction.withdrawals], - ) - if not transaction.reward_amount - else transaction.reward_amount, - self.data.LOVELACE_DECIMALS, - ), - ] - ] + { + "hash": transaction.hash, + "message": self._format_message(transaction), + } ) - return result + return pd.DataFrame([result]) - def transaction_outputs(self, transaction: Namespace) -> dict: - result = {} + def _transaction_outputs(self, transaction: Namespace) -> Any: + result = OrderedDict() + result[(self.data.LOVELACE_ASSET, "fees")] = Decimal(transaction.fees) + result[(self.data.LOVELACE_ASSET, "deposit")] = Decimal(transaction.deposit) + result[(self.data.LOVELACE_ASSET, "rewards")] = ( + self.decimal_context.minus( + functools.reduce( + self.decimal_context.add, + [Decimal(w.amount) for w in transaction.withdrawals], + Decimal(0), + ) + ) + if not transaction.reward_amount + else Decimal(transaction.reward_amount) + ) balance_result: Dict = OrderedDict() - for i in transaction.utxos.nonref_inputs: - if not i.collateral or not transaction.valid_contract: - for amount in i.amount: - key = self._utxo_amount_key(i, amount) - if key is not None: - if key not in balance_result: - balance_result[key] = Decimal(0) - balance_result[key] -= Decimal(amount.quantity) - - for out in transaction.utxos.outputs: - for amount in out.amount: - key = self._utxo_amount_key(out, amount) - if key is not None: + for utxo in transaction.utxos.nonref_inputs: + if not utxo.collateral or not transaction.valid_contract: + for amount in utxo.amount: + key = (amount.unit, utxo.address) if key not in balance_result: balance_result[key] = Decimal(0) - balance_result[key] += Decimal(amount.quantity) + balance_result[key] -= Decimal(amount.quantity) + + for utxo in transaction.utxos.outputs: + for amount in utxo.amount: + key = (amount.unit, utxo.address) + if key not in balance_result: + balance_result[key] = Decimal(0) + balance_result[key] += Decimal(amount.quantity) result.update({k: v for k, v in balance_result.items() if v != 0}) - return result + return pd.DataFrame([result]) def make_transaction_array(self) -> pd.DataFrame: """Return a dataframe with each transaction until the specified block, included.""" + metadata = pd.DataFrame( + self.data.transactions.map(self._transaction_metadata).rename("metadata") + ) + outputs = pd.DataFrame( + self.data.transactions.map(self._transaction_outputs).rename("balance") + ) + # Add total line at the bottom + # total = [] + # for column in outputs.columns: + # # Only NaN is float in the column + # total.append( + # functools.reduce( + # self.decimal_context.add, + # [a for a in outputs[column] if type(a) is type(Decimal(0))], + # Decimal(0), + # ) + # ) + # outputs.loc["Total"] = total + frame = pd.DataFrame(metadata).join(outputs, how="outer") - metadata = self.data.transactions.map(self.transaction_metadata) - outputs = self.data.transactions.map(self.transaction_outputs) - frame = pd.DataFrame(data=[metadata, outputs]) - - # Scale according to decimal index row, and drop that row - for col in frame.columns: - if col[-1]: - scale = self.decimal_context.power(10, -Decimal(col[-1])) - frame[col] *= scale # type: ignore - frame.columns = pd.MultiIndex.from_tuples([c[:-1] for c in frame.columns]) - frame.sort_index(axis=1, level=0, sort_remaining=True, inplace=True) return frame From 3dde035a35241a97623044a767010e7f934a0f69 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Sat, 9 Sep 2023 12:13:03 +0200 Subject: [PATCH 008/124] pandaize S1E4 --- .../cardano_account_pandas_dumper.py | 62 ++++++++----------- 1 file changed, 27 insertions(+), 35 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 4742428..12ac8c3 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -3,7 +3,6 @@ import functools import itertools from collections import defaultdict -from decimal import Context, Decimal from typing import ( Any, Dict, @@ -176,7 +175,6 @@ def __init__( self.unmute = unmute self.raw_asset = raw_asset self.rewards = rewards - self.decimal_context = Context() def _format_asset(self, asset: str) -> Optional[str]: if asset == AccountData.LOVELACE_ASSET: @@ -307,30 +305,20 @@ def _decimals_for_asset(self, asset: str) -> int: return self.data.assets[asset].metadata.decimals return 0 - def _transaction_metadata(self, transaction: Namespace) -> Any: - """Return a dict holding a Pandas row for transaction tx""" - result: Dict = OrderedDict( - { - "hash": transaction.hash, - "message": self._format_message(transaction), - } - ) - return pd.DataFrame([result]) - - def _transaction_outputs(self, transaction: Namespace) -> Any: - result = OrderedDict() - result[(self.data.LOVELACE_ASSET, "fees")] = Decimal(transaction.fees) - result[(self.data.LOVELACE_ASSET, "deposit")] = Decimal(transaction.deposit) + def _transaction_balance(self, transaction: Namespace) -> Any: + result = {} + result[(self.data.LOVELACE_ASSET, "fees")] = np.longlong(transaction.fees) + result[(self.data.LOVELACE_ASSET, "deposit")] = np.longlong(transaction.deposit) result[(self.data.LOVELACE_ASSET, "rewards")] = ( - self.decimal_context.minus( + np.negative( functools.reduce( - self.decimal_context.add, - [Decimal(w.amount) for w in transaction.withdrawals], - Decimal(0), + np.add, + [np.longlong(w.amount) for w in transaction.withdrawals], + np.longlong(0), ) ) if not transaction.reward_amount - else Decimal(transaction.reward_amount) + else np.longlong(transaction.reward_amount) ) balance_result: Dict = OrderedDict() for utxo in transaction.utxos.nonref_inputs: @@ -338,38 +326,42 @@ def _transaction_outputs(self, transaction: Namespace) -> Any: for amount in utxo.amount: key = (amount.unit, utxo.address) if key not in balance_result: - balance_result[key] = Decimal(0) - balance_result[key] -= Decimal(amount.quantity) + balance_result[key] = np.longlong(0) + balance_result[key] -= np.longlong(amount.quantity) for utxo in transaction.utxos.outputs: for amount in utxo.amount: key = (amount.unit, utxo.address) if key not in balance_result: - balance_result[key] = Decimal(0) - balance_result[key] += Decimal(amount.quantity) + balance_result[key] = np.longlong(0) + balance_result[key] += np.longlong(amount.quantity) result.update({k: v for k, v in balance_result.items() if v != 0}) - return pd.DataFrame([result]) + return result def make_transaction_array(self) -> pd.DataFrame: """Return a dataframe with each transaction until the specified block, included.""" - metadata = pd.DataFrame( - self.data.transactions.map(self._transaction_metadata).rename("metadata") - ) - outputs = pd.DataFrame( - self.data.transactions.map(self._transaction_outputs).rename("balance") + tx_hash = self.data.transactions.rename("tx_hash").map(lambda x: x.hash) + message = self.data.transactions.rename("message").map(self._format_message) + balance_objects = [self._transaction_balance(x) for x in self.data.transactions] + balance = pd.Series( + name="balance", data=balance_objects, index=self.data.transactions.index ) + # index = pd.MultiIndex.from_tuples(balance.columns) + # Add total line at the bottom # total = [] # for column in outputs.columns: # # Only NaN is float in the column # total.append( # functools.reduce( - # self.decimal_context.add, - # [a for a in outputs[column] if type(a) is type(Decimal(0))], - # Decimal(0), + # self.np.longlong_context.add, + # [a for a in outputs[column] if type(a) is type(np.longlong(0))], + # np.longlong(0), # ) # ) # outputs.loc["Total"] = total - frame = pd.DataFrame(metadata).join(outputs, how="outer") + frame = ( + pd.DataFrame(tx_hash).join(message, how="inner").join(balance, how="inner") + ) return frame From 1027a13f701e6d7a8a6cf3a64033e4b2978ebfe9 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Sat, 9 Sep 2023 14:39:54 +0200 Subject: [PATCH 009/124] pandaization phase 1 complete --- src/cardano_account_pandas_dumper/__main__.py | 2 +- .../cardano_account_pandas_dumper.py | 99 ++++++++++++------- 2 files changed, 67 insertions(+), 34 deletions(-) diff --git a/src/cardano_account_pandas_dumper/__main__.py b/src/cardano_account_pandas_dumper/__main__.py index 0483e5b..e6918ef 100644 --- a/src/cardano_account_pandas_dumper/__main__.py +++ b/src/cardano_account_pandas_dumper/__main__.py @@ -164,7 +164,7 @@ def main(): raw_asset=args.raw_asset or False, rewards=data_from_api.rewards, ) - dataframe = reporter.make_transaction_array() + dataframe = reporter.make_transaction_frame() if args.pandas_output: try: dataframe.to_pickle(args.pandas_output) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 12ac8c3..cf4ce9a 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -10,7 +10,6 @@ List, Mapping, Optional, - OrderedDict, Set, ) import pandas as pd @@ -24,9 +23,6 @@ class AccountData: LOVELACE_ASSET = "lovelace" LOVELACE_DECIMALS = 6 - TRANSACTION_OFFSET = np.timedelta64( - 1000, "ns" - ) # Time for a stransaction is block_time + index * TRANSACTION_OFFSET def __init__( self, @@ -89,14 +85,7 @@ def _transaction_data( transaction.reward_amount = None result_list.append(transaction) - index = pd.DatetimeIndex( - [ - np.datetime64(datetime.datetime.fromtimestamp(t.block_time)) - + (int(t.index) * self.TRANSACTION_OFFSET) - for t in result_list - ], - ) - return pd.Series(name="Transactions", data=result_list, index=index) + return pd.Series(name="Transactions", data=result_list) def _assets_from_transactions(self, api: BlockFrostApi) -> pd.Series: all_asset_ids: Set[str] = set() @@ -138,14 +127,7 @@ def _reward_transactions(self, api: BlockFrostApi) -> pd.Series: for s_a in self.staking_addresses for a_r in api.account_rewards(s_a, gather_pages=True) ] - index = pd.DatetimeIndex( - [ - np.datetime64(datetime.datetime.fromtimestamp(t.block_time)) - + (t.index * self.TRANSACTION_OFFSET) - for t in result_list - ], - ) - return pd.Series(name="Rewards", data=result_list, index=index) + return pd.Series(name="Rewards", data=result_list) class AccountPandasDumper: @@ -157,6 +139,9 @@ class AccountPandasDumper: SCRIPTS_KEY = "scripts" LABELS_KEY = "labels" ASSETS_KEY = "assets" + TRANSACTION_OFFSET = np.timedelta64( + 1000, "ns" + ) # Time for a stransaction is block_time + index * TRANSACTION_OFFSET def __init__( self, @@ -320,7 +305,7 @@ def _transaction_balance(self, transaction: Namespace) -> Any: if not transaction.reward_amount else np.longlong(transaction.reward_amount) ) - balance_result: Dict = OrderedDict() + balance_result: Dict = {} for utxo in transaction.utxos.nonref_inputs: if not utxo.collateral or not transaction.valid_contract: for amount in utxo.amount: @@ -335,19 +320,48 @@ def _transaction_balance(self, transaction: Namespace) -> Any: if key not in balance_result: balance_result[key] = np.longlong(0) balance_result[key] += np.longlong(amount.quantity) - result.update({k: v for k, v in balance_result.items() if v != 0}) + result.update({k: v for k, v in balance_result.items() if v != np.longlong(0)}) return result - def make_transaction_array(self) -> pd.DataFrame: - """Return a dataframe with each transaction until the specified block, included.""" - tx_hash = self.data.transactions.rename("tx_hash").map(lambda x: x.hash) - message = self.data.transactions.rename("message").map(self._format_message) - balance_objects = [self._transaction_balance(x) for x in self.data.transactions] - balance = pd.Series( - name="balance", data=balance_objects, index=self.data.transactions.index + def _make_hash_frame(self) -> pd.DataFrame: + tx_hash = pd.DataFrame( + data=[x.hash for x in self.data.transactions], + columns=["hash"], ) - # index = pd.MultiIndex.from_tuples(balance.columns) + tx_hash.columns = pd.MultiIndex.from_tuples([("", c) for c in tx_hash.columns]) + return tx_hash + def _make_timestamp_frame(self) -> pd.DataFrame: + timestamp = pd.DataFrame( + data=[ + np.datetime64(datetime.datetime.fromtimestamp(x.block_time)) + + (int(x.index) * self.TRANSACTION_OFFSET) + for x in self.data.transactions + ], + columns=["timestamp"], + ) + timestamp.columns = pd.MultiIndex.from_tuples( + [("", c) for c in timestamp.columns] + ) + return timestamp + + def _make_message_frame(self) -> pd.DataFrame: + message = pd.DataFrame( + data=[self._format_message(x) for x in self.data.transactions], + columns=["message"], + ) + message.columns = pd.MultiIndex.from_tuples([("", c) for c in message.columns]) + return message + + def _make_balance_frame(self) -> pd.DataFrame: + balance = pd.DataFrame( + data=[self._transaction_balance(x) for x in self.data.transactions], + ) + balance.columns = pd.MultiIndex.from_tuples(balance.columns) + balance.sort_index(axis=1, level=0, sort_remaining=True, inplace=True) + return balance + + def make_transaction_frame(self) -> pd.DataFrame: # Add total line at the bottom # total = [] # for column in outputs.columns: @@ -360,8 +374,27 @@ def make_transaction_array(self) -> pd.DataFrame: # ) # ) # outputs.loc["Total"] = total - frame = ( - pd.DataFrame(tx_hash).join(message, how="inner").join(balance, how="inner") + frame = pd.merge( + left=self._make_timestamp_frame(), + right=self._make_hash_frame(), + left_index=True, + right_index=True, + how="left", ) - + frame = pd.merge( + left=frame, + right=self._make_message_frame(), + left_index=True, + right_index=True, + how="left", + ) + frame = pd.merge( + left=frame, + right=self._make_balance_frame(), + left_index=True, + right_index=True, + how="left", + ) + frame.drop_duplicates(inplace=True) + frame.sort_values(by=("", "timestamp"), inplace=True) return frame From bb533bc72f44ebafbb246988dce7d7fba24893a7 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Sat, 9 Sep 2023 15:08:07 +0200 Subject: [PATCH 010/124] add rewards --- .../cardano_account_pandas_dumper.py | 33 +++++++++++++++---- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index cf4ce9a..17b3148 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -104,8 +104,8 @@ def _assets_from_transactions(self, api: BlockFrostApi) -> pd.Series: def _reward_transaction(api: BlockFrostApi, reward: Namespace) -> Namespace: result = Namespace() result.tx_hash = None - result.Metadata = Namespace() - result.Metadata.message = f"Reward: {reward.type} - {reward.epoch}" + result.metadata = Namespace() + result.metadata.message = f"Reward: {reward.type} - {reward.epoch}" result.reward_amount = reward.amount epoch = api.epoch(reward.epoch + 1) # Time is right before start of next epoch. result.block_time = epoch.start_time @@ -325,7 +325,13 @@ def _transaction_balance(self, transaction: Namespace) -> Any: def _make_hash_frame(self) -> pd.DataFrame: tx_hash = pd.DataFrame( - data=[x.hash for x in self.data.transactions], + data=[ + x.hash + for x in itertools.chain( + self.data.transactions, + self.data.reward_transactions if self.rewards else pd.Series(), + ) + ], columns=["hash"], ) tx_hash.columns = pd.MultiIndex.from_tuples([("", c) for c in tx_hash.columns]) @@ -336,7 +342,10 @@ def _make_timestamp_frame(self) -> pd.DataFrame: data=[ np.datetime64(datetime.datetime.fromtimestamp(x.block_time)) + (int(x.index) * self.TRANSACTION_OFFSET) - for x in self.data.transactions + for x in itertools.chain( + self.data.transactions, + self.data.reward_transactions if self.rewards else pd.Series(), + ) ], columns=["timestamp"], ) @@ -347,7 +356,13 @@ def _make_timestamp_frame(self) -> pd.DataFrame: def _make_message_frame(self) -> pd.DataFrame: message = pd.DataFrame( - data=[self._format_message(x) for x in self.data.transactions], + data=[ + self._format_message(x) + for x in itertools.chain( + self.data.transactions, + self.data.reward_transactions if self.rewards else pd.Series(), + ) + ], columns=["message"], ) message.columns = pd.MultiIndex.from_tuples([("", c) for c in message.columns]) @@ -355,7 +370,13 @@ def _make_message_frame(self) -> pd.DataFrame: def _make_balance_frame(self) -> pd.DataFrame: balance = pd.DataFrame( - data=[self._transaction_balance(x) for x in self.data.transactions], + data=[ + self._transaction_balance(x) + for x in itertools.chain( + self.data.transactions, + self.data.reward_transactions if self.rewards else pd.Series(), + ) + ], ) balance.columns = pd.MultiIndex.from_tuples(balance.columns) balance.sort_index(axis=1, level=0, sort_remaining=True, inplace=True) From 3c0859bf3582aa6b75b1b04457a362fc7788289c Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Sat, 9 Sep 2023 16:57:50 +0200 Subject: [PATCH 011/124] allow to_block if consistent with checkpoint --- src/cardano_account_pandas_dumper/__main__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cardano_account_pandas_dumper/__main__.py b/src/cardano_account_pandas_dumper/__main__.py index e6918ef..0ac3edd 100644 --- a/src/cardano_account_pandas_dumper/__main__.py +++ b/src/cardano_account_pandas_dumper/__main__.py @@ -122,10 +122,10 @@ def main(): ), status=1, ) - if args.to_block is not None: + if args.to_block is not None and args.to_block != data_from_api.to_block: parser.exit( message=( - "--to_block not allowed with --from_checkpoint (always taken from checkpoint)." + f"--to_block {args.to_block} different from checkpoint's {data_from_api.to_block}." ), status=1, ) From 0f08d4b02a677c14be1303b090c00b267315e2ab Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Sat, 9 Sep 2023 17:04:52 +0200 Subject: [PATCH 012/124] fix lint --- src/cardano_account_pandas_dumper/__main__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/cardano_account_pandas_dumper/__main__.py b/src/cardano_account_pandas_dumper/__main__.py index 0ac3edd..d252525 100644 --- a/src/cardano_account_pandas_dumper/__main__.py +++ b/src/cardano_account_pandas_dumper/__main__.py @@ -125,7 +125,8 @@ def main(): if args.to_block is not None and args.to_block != data_from_api.to_block: parser.exit( message=( - f"--to_block {args.to_block} different from checkpoint's {data_from_api.to_block}." + f"--to_block {args.to_block} different " + + f"from checkpoint's {data_from_api.to_block}." ), status=1, ) From d950f34a6b9cee9f2c9283e4d79a773b38413805 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Sat, 9 Sep 2023 17:05:24 +0200 Subject: [PATCH 013/124] add rewards --- .../cardano_account_pandas_dumper.py | 64 +++++++++++++------ 1 file changed, 46 insertions(+), 18 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 17b3148..874b247 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -101,11 +101,20 @@ def _assets_from_transactions(self, api: BlockFrostApi) -> pd.Series: ) @staticmethod - def _reward_transaction(api: BlockFrostApi, reward: Namespace) -> Namespace: + def _reward_transaction( + api: BlockFrostApi, reward: Namespace, pools: Mapping[str, Namespace] + ) -> Namespace: result = Namespace() result.tx_hash = None - result.metadata = Namespace() - result.metadata.message = f"Reward: {reward.type} - {reward.epoch}" + pool_name = ( + pools[reward.pool_id].name if reward.pool_id in pools else reward.pool_id + ) + result.metadata = [ + Namespace( + label="674", + json_metadata=f"Reward: {reward.type} - {pool_name} - {reward.epoch}", + ) + ] result.reward_amount = reward.amount epoch = api.epoch(reward.epoch + 1) # Time is right before start of next epoch. result.block_time = epoch.start_time @@ -122,12 +131,21 @@ def _reward_transaction(api: BlockFrostApi, reward: Namespace) -> Namespace: return result def _reward_transactions(self, api: BlockFrostApi) -> pd.Series: - result_list = [ - self._reward_transaction(api=api, reward=a_r) + reward_list = [ + a_r for s_a in self.staking_addresses for a_r in api.account_rewards(s_a, gather_pages=True) ] - return pd.Series(name="Rewards", data=result_list) + + pool_result_list = { + pool: api.pool_metadata(pool) + for pool in frozenset([r.pool_id for r in reward_list]) + } + reward_result_list = [ + self._reward_transaction(api=api, reward=a_r, pools=pool_result_list) + for a_r in reward_list + ] + return pd.Series(name="Rewards", data=reward_result_list) class AccountPandasDumper: @@ -327,9 +345,11 @@ def _make_hash_frame(self) -> pd.DataFrame: tx_hash = pd.DataFrame( data=[ x.hash - for x in itertools.chain( - self.data.transactions, - self.data.reward_transactions if self.rewards else pd.Series(), + for x in pd.concat( + objs=[ + self.data.transactions, + self.data.reward_transactions if self.rewards else pd.Series(), + ] ) ], columns=["hash"], @@ -342,9 +362,11 @@ def _make_timestamp_frame(self) -> pd.DataFrame: data=[ np.datetime64(datetime.datetime.fromtimestamp(x.block_time)) + (int(x.index) * self.TRANSACTION_OFFSET) - for x in itertools.chain( - self.data.transactions, - self.data.reward_transactions if self.rewards else pd.Series(), + for x in pd.concat( + objs=[ + self.data.transactions, + self.data.reward_transactions if self.rewards else pd.Series(), + ], ) ], columns=["timestamp"], @@ -358,9 +380,11 @@ def _make_message_frame(self) -> pd.DataFrame: message = pd.DataFrame( data=[ self._format_message(x) - for x in itertools.chain( - self.data.transactions, - self.data.reward_transactions if self.rewards else pd.Series(), + for x in pd.concat( + objs=[ + self.data.transactions, + self.data.reward_transactions if self.rewards else pd.Series(), + ], ) ], columns=["message"], @@ -372,9 +396,11 @@ def _make_balance_frame(self) -> pd.DataFrame: balance = pd.DataFrame( data=[ self._transaction_balance(x) - for x in itertools.chain( - self.data.transactions, - self.data.reward_transactions if self.rewards else pd.Series(), + for x in pd.concat( + objs=[ + self.data.transactions, + self.data.reward_transactions if self.rewards else pd.Series(), + ], ) ], ) @@ -383,6 +409,8 @@ def _make_balance_frame(self) -> pd.DataFrame: return balance def make_transaction_frame(self) -> pd.DataFrame: + """Build a transaction spreadsheet.""" + # Add total line at the bottom # total = [] # for column in outputs.columns: From 9b5f088a405088d42a100f92a57aefc858acf83e Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Sat, 9 Sep 2023 17:20:53 +0200 Subject: [PATCH 014/124] fix index --- .../cardano_account_pandas_dumper.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 874b247..1040b28 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -446,4 +446,6 @@ def make_transaction_frame(self) -> pd.DataFrame: ) frame.drop_duplicates(inplace=True) frame.sort_values(by=("", "timestamp"), inplace=True) + frame.reset_index(inplace=True) + frame.drop([("index", "")], axis=1, inplace=True) return frame From 196eec11fb21c730108dd46cf7c211aade6d629c Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Sun, 10 Sep 2023 09:59:33 +0200 Subject: [PATCH 015/124] drop index --- src/cardano_account_pandas_dumper/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cardano_account_pandas_dumper/__main__.py b/src/cardano_account_pandas_dumper/__main__.py index d252525..72e7705 100644 --- a/src/cardano_account_pandas_dumper/__main__.py +++ b/src/cardano_account_pandas_dumper/__main__.py @@ -173,7 +173,7 @@ def main(): warnings.warn(f"Failed to write pandas file: {exception}") if args.csv_output: try: - dataframe.to_csv(args.csv_output, index=True) + dataframe.to_csv(args.csv_output, index=False) except OSError as exception: warnings.warn(f"Failed to write CSV file: {exception}") print("Done.") From be92861c2947af62e9ad99c486647254ad3d5d4f Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Sun, 10 Sep 2023 10:00:21 +0200 Subject: [PATCH 016/124] limit rewards --- .../cardano_account_pandas_dumper.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 1040b28..1e6216d 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -33,8 +33,12 @@ def __init__( ) -> None: self.staking_addresses = staking_addresses if to_block is None: - to_block = int(api.block_latest().height) + to_block = int(api.block_latest().height - 1) self.to_block = to_block + block_last = api.block(self.to_block) + block_after_last = api.block(self.to_block + 1) + self.end_time = block_after_last.time + self.end_epoch = block_last.epoch + 1 self.own_addresses: FrozenSet[str] = self._own_addresses(api) self.rewards = rewards if self.rewards: @@ -135,6 +139,7 @@ def _reward_transactions(self, api: BlockFrostApi) -> pd.Series: a_r for s_a in self.staking_addresses for a_r in api.account_rewards(s_a, gather_pages=True) + if a_r.epoch < self.end_epoch ] pool_result_list = { @@ -406,6 +411,7 @@ def _make_balance_frame(self) -> pd.DataFrame: ) balance.columns = pd.MultiIndex.from_tuples(balance.columns) balance.sort_index(axis=1, level=0, sort_remaining=True, inplace=True) + return balance def make_transaction_frame(self) -> pd.DataFrame: From 2c256d160f611f914259cc7c14a2a35349f9014e Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Sun, 10 Sep 2023 10:45:47 +0200 Subject: [PATCH 017/124] simplify _transaction_balance --- .../cardano_account_pandas_dumper.py | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 1e6216d..f21b50c 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -9,8 +9,10 @@ FrozenSet, List, Mapping, + MutableMapping, Optional, Set, + Tuple, ) import pandas as pd import numpy as np @@ -290,7 +292,7 @@ def _format_message(self, tx_obj: Namespace) -> str: ] ): result = ["(internal)"] - return " ".join(result) + return " ".join(result).removeprefix("Message : ") def _format_script(self, script: str) -> str: return self.known_dict[self.SCRIPTS_KEY].get( @@ -314,7 +316,9 @@ def _decimals_for_asset(self, asset: str) -> int: return 0 def _transaction_balance(self, transaction: Namespace) -> Any: - result = {} + result: MutableMapping[Tuple[str, str], np.longlong] = defaultdict( + lambda: np.longlong(0) + ) result[(self.data.LOVELACE_ASSET, "fees")] = np.longlong(transaction.fees) result[(self.data.LOVELACE_ASSET, "deposit")] = np.longlong(transaction.deposit) result[(self.data.LOVELACE_ASSET, "rewards")] = ( @@ -328,23 +332,16 @@ def _transaction_balance(self, transaction: Namespace) -> Any: if not transaction.reward_amount else np.longlong(transaction.reward_amount) ) - balance_result: Dict = {} for utxo in transaction.utxos.nonref_inputs: if not utxo.collateral or not transaction.valid_contract: for amount in utxo.amount: - key = (amount.unit, utxo.address) - if key not in balance_result: - balance_result[key] = np.longlong(0) - balance_result[key] -= np.longlong(amount.quantity) + result[(amount.unit, utxo.address)] -= np.longlong(amount.quantity) for utxo in transaction.utxos.outputs: for amount in utxo.amount: - key = (amount.unit, utxo.address) - if key not in balance_result: - balance_result[key] = np.longlong(0) - balance_result[key] += np.longlong(amount.quantity) - result.update({k: v for k, v in balance_result.items() if v != np.longlong(0)}) - return result + result[(amount.unit, utxo.address)] += np.longlong(amount.quantity) + + return dict([i for i in result.items() if i[1] != np.longlong(0)]) def _make_hash_frame(self) -> pd.DataFrame: tx_hash = pd.DataFrame( From bb0dc4a376767bafedb0dee1095c8765715e4ead Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Sun, 10 Sep 2023 13:40:08 +0200 Subject: [PATCH 018/124] add own to index --- .../cardano_account_pandas_dumper.py | 163 +++++++----------- 1 file changed, 67 insertions(+), 96 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index f21b50c..d1edd84 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -155,6 +155,9 @@ def _reward_transactions(self, api: BlockFrostApi) -> pd.Series: return pd.Series(name="Rewards", data=reward_result_list) +TRANSACTION_OFFSET = np.timedelta64(1000, "ns") + + class AccountPandasDumper: """Hold logic to convert an instance of AccountData to a Pandas dataframe.""" @@ -164,9 +167,7 @@ class AccountPandasDumper: SCRIPTS_KEY = "scripts" LABELS_KEY = "labels" ASSETS_KEY = "assets" - TRANSACTION_OFFSET = np.timedelta64( - 1000, "ns" - ) # Time for a stransaction is block_time + index * TRANSACTION_OFFSET + # Time for a stransaction is block_time + index * TRANSACTION_OFFSET def __init__( self, @@ -316,12 +317,14 @@ def _decimals_for_asset(self, asset: str) -> int: return 0 def _transaction_balance(self, transaction: Namespace) -> Any: - result: MutableMapping[Tuple[str, str], np.longlong] = defaultdict( + result: MutableMapping[Tuple[str, str, bool], np.longlong] = defaultdict( lambda: np.longlong(0) ) - result[(self.data.LOVELACE_ASSET, "fees")] = np.longlong(transaction.fees) - result[(self.data.LOVELACE_ASSET, "deposit")] = np.longlong(transaction.deposit) - result[(self.data.LOVELACE_ASSET, "rewards")] = ( + result[(self.data.LOVELACE_ASSET, "fees", True)] = np.longlong(transaction.fees) + result[(self.data.LOVELACE_ASSET, "deposit", True)] = np.longlong( + transaction.deposit + ) + result[(self.data.LOVELACE_ASSET, "rewards", True)] = ( np.negative( functools.reduce( np.add, @@ -335,82 +338,40 @@ def _transaction_balance(self, transaction: Namespace) -> Any: for utxo in transaction.utxos.nonref_inputs: if not utxo.collateral or not transaction.valid_contract: for amount in utxo.amount: - result[(amount.unit, utxo.address)] -= np.longlong(amount.quantity) + result[ + ( + amount.unit, + utxo.address, + utxo.address in self.data.own_addresses, + ) + ] -= np.longlong(amount.quantity) for utxo in transaction.utxos.outputs: for amount in utxo.amount: - result[(amount.unit, utxo.address)] += np.longlong(amount.quantity) - - return dict([i for i in result.items() if i[1] != np.longlong(0)]) - - def _make_hash_frame(self) -> pd.DataFrame: - tx_hash = pd.DataFrame( - data=[ - x.hash - for x in pd.concat( - objs=[ - self.data.transactions, - self.data.reward_transactions if self.rewards else pd.Series(), - ] - ) - ], - columns=["hash"], - ) - tx_hash.columns = pd.MultiIndex.from_tuples([("", c) for c in tx_hash.columns]) - return tx_hash - - def _make_timestamp_frame(self) -> pd.DataFrame: - timestamp = pd.DataFrame( - data=[ - np.datetime64(datetime.datetime.fromtimestamp(x.block_time)) - + (int(x.index) * self.TRANSACTION_OFFSET) - for x in pd.concat( - objs=[ - self.data.transactions, - self.data.reward_transactions if self.rewards else pd.Series(), - ], - ) - ], - columns=["timestamp"], - ) - timestamp.columns = pd.MultiIndex.from_tuples( - [("", c) for c in timestamp.columns] - ) - return timestamp - - def _make_message_frame(self) -> pd.DataFrame: - message = pd.DataFrame( - data=[ - self._format_message(x) - for x in pd.concat( - objs=[ - self.data.transactions, - self.data.reward_transactions if self.rewards else pd.Series(), - ], - ) - ], - columns=["message"], - ) - message.columns = pd.MultiIndex.from_tuples([("", c) for c in message.columns]) - return message + result[ + ( + amount.unit, + utxo.address, + utxo.address in self.data.own_addresses, + ) + ] += np.longlong(amount.quantity) - def _make_balance_frame(self) -> pd.DataFrame: + return result + + def _make_balance_frame(self, transactions: pd.Series) -> pd.DataFrame: balance = pd.DataFrame( - data=[ - self._transaction_balance(x) - for x in pd.concat( - objs=[ - self.data.transactions, - self.data.reward_transactions if self.rewards else pd.Series(), - ], - ) - ], + data=[self._transaction_balance(x) for x in transactions], ) balance.columns = pd.MultiIndex.from_tuples(balance.columns) balance.sort_index(axis=1, level=0, sort_remaining=True, inplace=True) - return balance + @staticmethod + def _extract_timestamp(transaction: Namespace) -> Any: + return np.datetime64( + datetime.datetime.fromtimestamp(transaction.block_time) + ) + (int(transaction.index) * TRANSACTION_OFFSET) + def make_transaction_frame(self) -> pd.DataFrame: """Build a transaction spreadsheet.""" @@ -426,29 +387,39 @@ def make_transaction_frame(self) -> pd.DataFrame: # ) # ) # outputs.loc["Total"] = total - frame = pd.merge( - left=self._make_timestamp_frame(), - right=self._make_hash_frame(), - left_index=True, - right_index=True, - how="left", - ) - frame = pd.merge( - left=frame, - right=self._make_message_frame(), - left_index=True, - right_index=True, - how="left", - ) - frame = pd.merge( - left=frame, - right=self._make_balance_frame(), - left_index=True, - right_index=True, - how="left", + transactions = pd.concat( + objs=[ + self.data.transactions, + self.data.reward_transactions if self.rewards else pd.Series(), + ], + ).rename("transactions") + timestamp = transactions.rename("timestamp").map(self._extract_timestamp) + tx_hash = transactions.rename("hash").map(lambda x: x.hash) + message = transactions.rename("message").map(self._format_message) + balance = self._make_balance_frame(transactions) + # Drop assets that only touch foreign addresses + assets_to_drop = [ + (x,) + for x in ( + frozenset( + # Assets that touch other addresses + x[0] + for x in balance.xs(False, level=2, axis=1).columns + ) + - frozenset( + # Assets that touch own addresses + x[0] + for x in balance.xs(True, level=2, axis=1).columns + ) + ) + ] + balance.drop(assets_to_drop, axis=1, inplace=True) + + frame = pd.concat([timestamp, tx_hash, message], axis=1) + frame.columns = pd.MultiIndex.from_tuples( + [("metadata", c, "") for c in frame.columns] ) + frame = frame.merge(balance, left_index=True, right_index=True) frame.drop_duplicates(inplace=True) - frame.sort_values(by=("", "timestamp"), inplace=True) - frame.reset_index(inplace=True) - frame.drop([("index", "")], axis=1, inplace=True) + frame.sort_values(by=("metadata", "timestamp", ""), inplace=True) return frame From dad3b86e37d10ff9bb4e5f65fb8fa35658fae67a Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Sun, 10 Sep 2023 15:45:02 +0200 Subject: [PATCH 019/124] dynamic index size --- .../cardano_account_pandas_dumper.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index d1edd84..a15dbf2 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -361,6 +361,7 @@ def _transaction_balance(self, transaction: Namespace) -> Any: def _make_balance_frame(self, transactions: pd.Series) -> pd.DataFrame: balance = pd.DataFrame( data=[self._transaction_balance(x) for x in transactions], + dtype=pd.Int64Dtype, ) balance.columns = pd.MultiIndex.from_tuples(balance.columns) balance.sort_index(axis=1, level=0, sort_remaining=True, inplace=True) @@ -414,12 +415,15 @@ def make_transaction_frame(self) -> pd.DataFrame: ) ] balance.drop(assets_to_drop, axis=1, inplace=True) - + balance_column_index_length = len(balance.columns[0]) frame = pd.concat([timestamp, tx_hash, message], axis=1) frame.columns = pd.MultiIndex.from_tuples( - [("metadata", c, "") for c in frame.columns] + [ + ("metadata", c) + (balance_column_index_length - 2) * ("",) + for c in frame.columns + ] ) frame = frame.merge(balance, left_index=True, right_index=True) frame.drop_duplicates(inplace=True) - frame.sort_values(by=("metadata", "timestamp", ""), inplace=True) + frame.sort_values(by=frame.columns[1], inplace=True) return frame From 2b0a079fd47699a712f7b920a3aea11bde024f43 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Sun, 10 Sep 2023 19:22:21 +0200 Subject: [PATCH 020/124] muted_policies now a list --- src/cardano_account_pandas_dumper/known.jsonc | 52 +++++++++---------- 1 file changed, 25 insertions(+), 27 deletions(-) diff --git a/src/cardano_account_pandas_dumper/known.jsonc b/src/cardano_account_pandas_dumper/known.jsonc index 7d99f8d..71d6fc6 100644 --- a/src/cardano_account_pandas_dumper/known.jsonc +++ b/src/cardano_account_pandas_dumper/known.jsonc @@ -70,31 +70,31 @@ "fd0d72fafee1d230a74c31ac503a192abd5b71888ae3f94128c1e634": "INDY staking NFT" }, // Policies to mute - "muted_policies": { - "0029cb7c88c7567b63d1a512c0ed626aa169688ec980730c0473b913": true, - "026a18d04a0c642759bb3d83b12e3344894e5c1c7b2aeb1a2113a570": true, - "0be55d262b29f564998ff81efe21bdc0022621c12f15af08d0f2ddb1": true, - "13aa2accf2e1561723aa26871e071fdf32c867cff7e7d50ad470d62f": true, - "24b458412c2a7f9acb9c53c7ec4325b36806912ed56d2f91bfcf4d26": true, - "2f2e0404310c106e2a260e8eb5a7e43f00cff42c667489d30e179816": true, - "2fccae8bc1c8553a2185b2e77ccdea22f2e1d6e87beb80ef4eaf8cce": true, - "408f13b240f57c3473b5727a68a88c50c1b6bb15c0c10912008dc59e": true, - "5817c34e5702473304f3cf676299176d3824e55b8c0bfa94830429fd": true, - "636c06ed152e70e9a1240da585a2841c15e4c3174044ca9a62cadab2": true, - "72c8ee38fc1adfc5db49f7384bd64d41e9b9ce46ec11ba9a292e5d02": true, - "8a7b0a43a169218fb66996c3945f53d7712eae9e31db193adb55663f": true, - "909133088303c49f3a30f1cc8ed553a73857a29779f6c6561cd8093f": true, - "97da12de04a6b527cc3b3469c5e5485cf258dfd1021f12e728f2e714": true, - "af3d70acf4bd5b3abb319a7d75c89fb3e56eafcdd46b2e9b57a2557f": true, - "d195ca7db29f0f13a00cac7fca70426ff60bad4e1e87d3757fae8484": true, - "de9b756719341e79785aa13c164e7fe68c189ed04d61c9876b2fe53f": true, - "e3455f2715338b454fb853442f72dc03b98396854f97510027fe22ff": true, - "e4214b7cce62ac6fbba385d164df48e157eae5863521b4b67ca71d86": true, - "e8a447d4e19016ca2aa74d20b4c4de87adb1f21dfb5493bf2d7281a6": true, - "f66d78b4a3cb3d37afa0ec36461e51ecbde00f26c8f0a68f94b69880": true, - "21ca2902ad0406bc2ce361ad91157b3542544d86d4e3d3e3c43f1763": true, - "2960fe7a50e78948eb7219079c39b8169e34a4464a3f1aea9328d87c": true - }, + "muted_policies": [ + "0029cb7c88c7567b63d1a512c0ed626aa169688ec980730c0473b913", + "026a18d04a0c642759bb3d83b12e3344894e5c1c7b2aeb1a2113a570", + "0be55d262b29f564998ff81efe21bdc0022621c12f15af08d0f2ddb1", + "13aa2accf2e1561723aa26871e071fdf32c867cff7e7d50ad470d62f", + "24b458412c2a7f9acb9c53c7ec4325b36806912ed56d2f91bfcf4d26", + "2f2e0404310c106e2a260e8eb5a7e43f00cff42c667489d30e179816", + "2fccae8bc1c8553a2185b2e77ccdea22f2e1d6e87beb80ef4eaf8cce", + "408f13b240f57c3473b5727a68a88c50c1b6bb15c0c10912008dc59e", + "5817c34e5702473304f3cf676299176d3824e55b8c0bfa94830429fd", + "636c06ed152e70e9a1240da585a2841c15e4c3174044ca9a62cadab2", + "72c8ee38fc1adfc5db49f7384bd64d41e9b9ce46ec11ba9a292e5d02", + "8a7b0a43a169218fb66996c3945f53d7712eae9e31db193adb55663f", + "909133088303c49f3a30f1cc8ed553a73857a29779f6c6561cd8093f", + "97da12de04a6b527cc3b3469c5e5485cf258dfd1021f12e728f2e714", + "af3d70acf4bd5b3abb319a7d75c89fb3e56eafcdd46b2e9b57a2557f", + "d195ca7db29f0f13a00cac7fca70426ff60bad4e1e87d3757fae8484", + "de9b756719341e79785aa13c164e7fe68c189ed04d61c9876b2fe53f", + "e3455f2715338b454fb853442f72dc03b98396854f97510027fe22ff", + "e4214b7cce62ac6fbba385d164df48e157eae5863521b4b67ca71d86", + "e8a447d4e19016ca2aa74d20b4c4de87adb1f21dfb5493bf2d7281a6", + "f66d78b4a3cb3d37afa0ec36461e51ecbde00f26c8f0a68f94b69880", + "21ca2902ad0406bc2ce361ad91157b3542544d86d4e3d3e3c43f1763", + "2960fe7a50e78948eb7219079c39b8169e34a4464a3f1aea9328d87c" + ], // Known script hashes "scripts": { "00fb107bfbd51b3a5638867d3688e986ba38ff34fb738f5bd42b20d5": "MuesliSwap (Partial Match Order)", @@ -125,8 +125,6 @@ "e6c90a5923713af5786963dee0fdffd830ca7e0c86a041d9e5833e91": "Wingriders (Pool payment credential)", "ea5358d9fe82cc7ad8de0e76b4eabd851526408e51daa9d8bb4b137d": "Indigo Protocol (CDP creator)" }, - // Known assets - "assets": {}, // Known metadata labels "labels": { "674": "Message", From ab487f00365761c4e5f3794feaefefde5831fa55 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Sun, 10 Sep 2023 19:23:12 +0200 Subject: [PATCH 021/124] more Pandas refactoring --- .../cardano_account_pandas_dumper.py | 157 +++++++++++------- 1 file changed, 95 insertions(+), 62 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index a15dbf2..2fa7729 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -93,6 +93,31 @@ def _transaction_data( result_list.append(transaction) return pd.Series(name="Transactions", data=result_list) + @staticmethod + def _decode_asset_name(name: str, policy: str) -> str: + if isinstance(name, str): + name_bytes = bytearray( + [ + b if b in range(32, 127) else 127 + for b in bytes.fromhex(name.removeprefix(policy)) + ] + ) + return name_bytes.decode(encoding="ascii", errors="replace") + else: + return "???" + + @classmethod + def _fix_api_asset(cls, asset: Namespace) -> Namespace: + if not hasattr(asset, "metadata") or asset.metadata is None: + asset.metadata = Namespace() + if not hasattr(asset.metadata, "name"): + asset.metadata.name = cls._decode_asset_name( + asset.asset_name, asset.policy_id + ) + if not hasattr(asset.metadata, "decimals"): + asset.metadata.decimals = 0 + return asset + def _assets_from_transactions(self, api: BlockFrostApi) -> pd.Series: all_asset_ids: Set[str] = set() for tx_obj in self.transactions: @@ -101,9 +126,23 @@ def _assets_from_transactions(self, api: BlockFrostApi) -> pd.Series: [a.unit for i in tx_obj.utxos.inputs for a in i.amount] + [a.unit for i in tx_obj.utxos.outputs for a in i.amount] ) - all_asset_ids.remove(AccountData.LOVELACE_ASSET) + lovelace_asset_obj = Namespace() + lovelace_asset_obj.metadata = Namespace() + lovelace_asset_obj.metadata.name = "ADA" + lovelace_asset_obj.metadata.decimals = self.LOVELACE_DECIMALS + lovelace_asset_obj.policy_id = "" + lovelace_asset_obj.asset_name = "ADA" + return pd.Series( - name="Assets", data={asset: api.asset(asset) for asset in all_asset_ids} + name="Assets", + data={ + asset: ( + self._fix_api_asset(api.asset(asset)) + if asset != self.LOVELACE_ASSET + else lovelace_asset_obj + ) + for asset in all_asset_ids + }, ) @staticmethod @@ -166,13 +205,11 @@ class AccountPandasDumper: MUTED_POLICIES_KEY = "muted_policies" SCRIPTS_KEY = "scripts" LABELS_KEY = "labels" - ASSETS_KEY = "assets" - # Time for a stransaction is block_time + index * TRANSACTION_OFFSET def __init__( self, data: AccountData, - known_dict: Mapping[str, Mapping[str, str]], + known_dict: Any, detail_level: int, unmute: bool, truncate_length: Optional[int], @@ -188,36 +225,14 @@ def __init__( self.rewards = rewards def _format_asset(self, asset: str) -> Optional[str]: - if asset == AccountData.LOVELACE_ASSET: - return " ADA" - if asset in self.known_dict[self.ASSETS_KEY]: - return self.known_dict[self.ASSETS_KEY][asset] - asset_obj = self.data.assets[asset] - if asset_obj.metadata and asset_obj.metadata.name: - return asset_obj.metadata.name - if isinstance(asset_obj.asset_name, str): - name_bytes = bytearray( - [ - b if b in range(32, 127) else 127 - for b in bytes.fromhex( - asset_obj.asset_name.removeprefix(asset_obj.policy_id) - ) - ] - ) - name_str = name_bytes.decode(encoding="ascii", errors="replace") - else: - name_str = "???" - policy = self._format_policy(asset_obj.policy_id, self.unmute) - return f"{policy}@{name_str}" if policy is not None else None + return self.data.assets[asset].metadata.name def _truncate(self, value: str) -> str: return ( (value[: self.truncate_length] + "...") if self.truncate_length else value ) - def _format_policy(self, policy: str, unmute: bool) -> Optional[str]: - if self.known_dict[self.MUTED_POLICIES_KEY].get(policy, False) and not unmute: - return None + def _format_policy(self, policy: str) -> Optional[str]: return self.known_dict[self.POLICIES_KEY].get(policy, self._truncate(policy)) @staticmethod @@ -281,7 +296,7 @@ def _format_message(self, tx_obj: Namespace) -> str: ) elif redeemer.purpose == "mint": redeemer_scripts["Mint:"].append( - self._format_policy(redeemer.script_hash, True) + self._format_policy(redeemer.script_hash) ) for k, redeemer_script in redeemer_scripts.items(): result.append(k) @@ -310,11 +325,7 @@ def _format_address(self, address: str) -> str: ) def _decimals_for_asset(self, asset: str) -> int: - if asset == self.data.LOVELACE_ASSET: - return self.data.LOVELACE_DECIMALS - if asset in self.data.assets and self.data.assets[asset].metadata: - return self.data.assets[asset].metadata.decimals - return 0 + return self.data.assets[asset].metadata.decimals def _transaction_balance(self, transaction: Namespace) -> Any: result: MutableMapping[Tuple[str, str, bool], np.longlong] = defaultdict( @@ -358,21 +369,50 @@ def _transaction_balance(self, transaction: Namespace) -> Any: return result - def _make_balance_frame(self, transactions: pd.Series) -> pd.DataFrame: - balance = pd.DataFrame( - data=[self._transaction_balance(x) for x in transactions], - dtype=pd.Int64Dtype, - ) - balance.columns = pd.MultiIndex.from_tuples(balance.columns) - balance.sort_index(axis=1, level=0, sort_remaining=True, inplace=True) - return balance - @staticmethod def _extract_timestamp(transaction: Namespace) -> Any: return np.datetime64( datetime.datetime.fromtimestamp(transaction.block_time) ) + (int(transaction.index) * TRANSACTION_OFFSET) + def _drop_foreign_assets(self, balance: pd.DataFrame) -> None: + # Drop assets that only touch foreign addresses + balance.columns = pd.MultiIndex.from_tuples(balance.columns) + assets_to_drop = frozenset( + # Assets that touch other addresses + x[0] + for x in balance.xs(False, level=-1, axis=1).columns + ) - frozenset( + # Assets that touch own addresses + x[0] + for x in balance.xs(True, level=-1, axis=1).columns + ) + + balance.drop(assets_to_drop, axis=1, inplace=True) + + def _drop_muted_policies(self, balance: pd.DataFrame) -> None: + if ( + (not self.unmute) + and self.MUTED_POLICIES_KEY in self.known_dict + and self.known_dict[self.MUTED_POLICIES_KEY] + ): + policies_to_mute = frozenset(self.known_dict[self.MUTED_POLICIES_KEY]) + policy_length = len(self.known_dict[self.MUTED_POLICIES_KEY][0]) + assets_to_drop = frozenset( + [ + x[0] + for x in balance.columns + if x[0][:policy_length] in policies_to_mute + ] + ) + balance.drop(assets_to_drop, axis=1, inplace=True) + + def _relabel_assets(self, balance: pd.DataFrame) -> None: + new_columns = [ + (self.data.assets[x[0]].metadata.name,) + x[1:] for x in balance.columns + ] + balance.columns = new_columns + def make_transaction_frame(self) -> pd.DataFrame: """Build a transaction spreadsheet.""" @@ -397,25 +437,18 @@ def make_transaction_frame(self) -> pd.DataFrame: timestamp = transactions.rename("timestamp").map(self._extract_timestamp) tx_hash = transactions.rename("hash").map(lambda x: x.hash) message = transactions.rename("message").map(self._format_message) - balance = self._make_balance_frame(transactions) - # Drop assets that only touch foreign addresses - assets_to_drop = [ - (x,) - for x in ( - frozenset( - # Assets that touch other addresses - x[0] - for x in balance.xs(False, level=2, axis=1).columns - ) - - frozenset( - # Assets that touch own addresses - x[0] - for x in balance.xs(True, level=2, axis=1).columns - ) - ) - ] - balance.drop(assets_to_drop, axis=1, inplace=True) + balance = pd.DataFrame( + data=[self._transaction_balance(x) for x in transactions], + dtype=pd.Int64Dtype, + ) + self._drop_foreign_assets(balance) + self._drop_muted_policies(balance) + self._relabel_assets(balance) + balance.columns = pd.MultiIndex.from_tuples(balance.columns) + + balance.sort_index(axis=1, level=0, sort_remaining=True, inplace=True) balance_column_index_length = len(balance.columns[0]) + frame = pd.concat([timestamp, tx_hash, message], axis=1) frame.columns = pd.MultiIndex.from_tuples( [ From d1d28a4e8bb8a9795df6ee5dd168748d3ad853f3 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Mon, 11 Sep 2023 16:35:33 +0200 Subject: [PATCH 022/124] fix assets and policies --- .../cardano_account_pandas_dumper.py | 106 +++++++++--------- 1 file changed, 50 insertions(+), 56 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 2fa7729..84f8482 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -46,7 +46,7 @@ def __init__( if self.rewards: self.reward_transactions: pd.Series = self._reward_transactions(api) self.transactions: pd.Series = self._transaction_data(api) - self.assets: pd.Series = self._assets_from_transactions(api) + self.assets: pd.DataFrame = self._assets_from_transactions(api) def _own_addresses(self, api: BlockFrostApi) -> FrozenSet[str]: return frozenset( @@ -93,32 +93,20 @@ def _transaction_data( result_list.append(transaction) return pd.Series(name="Transactions", data=result_list) - @staticmethod - def _decode_asset_name(name: str, policy: str) -> str: - if isinstance(name, str): - name_bytes = bytearray( - [ - b if b in range(32, 127) else 127 - for b in bytes.fromhex(name.removeprefix(policy)) - ] - ) - return name_bytes.decode(encoding="ascii", errors="replace") - else: - return "???" - @classmethod - def _fix_api_asset(cls, asset: Namespace) -> Namespace: + def _fix_api_asset(cls, asset_id: str, asset: Namespace) -> Namespace: + asset.asset_id = asset_id if not hasattr(asset, "metadata") or asset.metadata is None: asset.metadata = Namespace() - if not hasattr(asset.metadata, "name"): - asset.metadata.name = cls._decode_asset_name( - asset.asset_name, asset.policy_id - ) + if not (hasattr(asset.metadata, "name") and asset.metadata.name): + asset.raw_name = asset_id.removeprefix(asset.policy_id) + else: + asset.raw_name = str(bytes(asset.metadata.name, "utf-8").hex()) if not hasattr(asset.metadata, "decimals"): asset.metadata.decimals = 0 return asset - def _assets_from_transactions(self, api: BlockFrostApi) -> pd.Series: + def _assets_from_transactions(self, api: BlockFrostApi) -> pd.DataFrame: all_asset_ids: Set[str] = set() for tx_obj in self.transactions: if hasattr(tx_obj, "utxos"): @@ -132,18 +120,25 @@ def _assets_from_transactions(self, api: BlockFrostApi) -> pd.Series: lovelace_asset_obj.metadata.decimals = self.LOVELACE_DECIMALS lovelace_asset_obj.policy_id = "" lovelace_asset_obj.asset_name = "ADA" - - return pd.Series( - name="Assets", - data={ - asset: ( - self._fix_api_asset(api.asset(asset)) - if asset != self.LOVELACE_ASSET - else lovelace_asset_obj - ) - for asset in all_asset_ids - }, + asset_list = [ + self._fix_api_asset( + asset, + api.asset(asset) + if asset != self.LOVELACE_ASSET + else lovelace_asset_obj, + ) + for asset in all_asset_ids + ] + assets = pd.DataFrame( + data=asset_list, + index=pd.MultiIndex.from_tuples( + [ + (asset.asset_id, asset.policy_id, asset.raw_name) + for asset in asset_list + ] + ), ) + return assets @staticmethod def _reward_transaction( @@ -324,18 +319,22 @@ def _format_address(self, address: str) -> str: "other", ) - def _decimals_for_asset(self, asset: str) -> int: - return self.data.assets[asset].metadata.decimals + def _decimals_for_asset(self, asset: str) -> np.longlong: + return np.longlong(self.data.assets[asset].metadata.decimals) + + def _asset_tuple(self, asset_id: str) -> Tuple: + asset = self.data.assets[0][(asset_id,)][0] + return (asset.policy_id, asset.raw_name) def _transaction_balance(self, transaction: Namespace) -> Any: - result: MutableMapping[Tuple[str, str, bool], np.longlong] = defaultdict( - lambda: np.longlong(0) - ) - result[(self.data.LOVELACE_ASSET, "fees", True)] = np.longlong(transaction.fees) - result[(self.data.LOVELACE_ASSET, "deposit", True)] = np.longlong( - transaction.deposit - ) - result[(self.data.LOVELACE_ASSET, "rewards", True)] = ( + result: MutableMapping[Tuple, np.longlong] = defaultdict(lambda: np.longlong(0)) + result[ + self._asset_tuple(self.data.LOVELACE_ASSET) + ("fees", True) + ] = np.longlong(transaction.fees) + result[ + self._asset_tuple(self.data.LOVELACE_ASSET) + ("deposit", True) + ] = np.longlong(transaction.deposit) + result[self._asset_tuple(self.data.LOVELACE_ASSET) + ("rewards", True)] = ( np.negative( functools.reduce( np.add, @@ -350,8 +349,8 @@ def _transaction_balance(self, transaction: Namespace) -> Any: if not utxo.collateral or not transaction.valid_contract: for amount in utxo.amount: result[ - ( - amount.unit, + self._asset_tuple(amount.unit) + + ( utxo.address, utxo.address in self.data.own_addresses, ) @@ -360,8 +359,8 @@ def _transaction_balance(self, transaction: Namespace) -> Any: for utxo in transaction.utxos.outputs: for amount in utxo.amount: result[ - ( - amount.unit, + self._asset_tuple(amount.unit) + + ( utxo.address, utxo.address in self.data.own_addresses, ) @@ -380,11 +379,11 @@ def _drop_foreign_assets(self, balance: pd.DataFrame) -> None: balance.columns = pd.MultiIndex.from_tuples(balance.columns) assets_to_drop = frozenset( # Assets that touch other addresses - x[0] + x[:2] for x in balance.xs(False, level=-1, axis=1).columns ) - frozenset( # Assets that touch own addresses - x[0] + x[:2] for x in balance.xs(True, level=-1, axis=1).columns ) @@ -397,15 +396,10 @@ def _drop_muted_policies(self, balance: pd.DataFrame) -> None: and self.known_dict[self.MUTED_POLICIES_KEY] ): policies_to_mute = frozenset(self.known_dict[self.MUTED_POLICIES_KEY]) - policy_length = len(self.known_dict[self.MUTED_POLICIES_KEY][0]) - assets_to_drop = frozenset( - [ - x[0] - for x in balance.columns - if x[0][:policy_length] in policies_to_mute - ] + policies_to_drop = frozenset( + [x[0] for x in balance.columns if x[0] in policies_to_mute] ) - balance.drop(assets_to_drop, axis=1, inplace=True) + balance.drop(policies_to_drop, axis=1, inplace=True) def _relabel_assets(self, balance: pd.DataFrame) -> None: new_columns = [ @@ -443,7 +437,7 @@ def make_transaction_frame(self) -> pd.DataFrame: ) self._drop_foreign_assets(balance) self._drop_muted_policies(balance) - self._relabel_assets(balance) + # self._relabel_assets(balance) balance.columns = pd.MultiIndex.from_tuples(balance.columns) balance.sort_index(axis=1, level=0, sort_remaining=True, inplace=True) From 50486e14e37cbdb4c046a687cde9d99a359e2cef Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Mon, 11 Sep 2023 16:59:39 +0200 Subject: [PATCH 023/124] fix sorting --- .../cardano_account_pandas_dumper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 84f8482..0dcd57e 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -452,5 +452,5 @@ def make_transaction_frame(self) -> pd.DataFrame: ) frame = frame.merge(balance, left_index=True, right_index=True) frame.drop_duplicates(inplace=True) - frame.sort_values(by=frame.columns[1], inplace=True) + frame.sort_values(by=frame.columns[0], inplace=True) return frame From 5068dd902e53ef0c18216898c7ce87fe608729e7 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Tue, 12 Sep 2023 10:37:18 +0200 Subject: [PATCH 024/124] pandaize S2E1 --- .../cardano_account_pandas_dumper.py | 80 ++++++++++--------- 1 file changed, 43 insertions(+), 37 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 0dcd57e..733629d 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -17,7 +17,7 @@ import pandas as pd import numpy as np from blockfrost import BlockFrostApi -from blockfrost.utils import Namespace +import blockfrost.utils class AccountData: @@ -44,9 +44,9 @@ def __init__( self.own_addresses: FrozenSet[str] = self._own_addresses(api) self.rewards = rewards if self.rewards: - self.reward_transactions: pd.Series = self._reward_transactions(api) - self.transactions: pd.Series = self._transaction_data(api) - self.assets: pd.DataFrame = self._assets_from_transactions(api) + self.reward_transactions = self._reward_transactions(api) + self.transactions = self._transaction_data(api) + self.assets = self._assets_from_transactions(api) def _own_addresses(self, api: BlockFrostApi) -> FrozenSet[str]: return frozenset( @@ -91,13 +91,17 @@ def _transaction_data( transaction.reward_amount = None result_list.append(transaction) - return pd.Series(name="Transactions", data=result_list) + return pd.Series( + name="Transactions", data=result_list, index=[t.hash for t in result_list] + ).sort_index() @classmethod - def _fix_api_asset(cls, asset_id: str, asset: Namespace) -> Namespace: + def _fix_api_asset( + cls, asset_id: str, asset: blockfrost.utils.Namespace + ) -> blockfrost.utils.Namespace: asset.asset_id = asset_id if not hasattr(asset, "metadata") or asset.metadata is None: - asset.metadata = Namespace() + asset.metadata = blockfrost.utils.Namespace() if not (hasattr(asset.metadata, "name") and asset.metadata.name): asset.raw_name = asset_id.removeprefix(asset.policy_id) else: @@ -106,16 +110,16 @@ def _fix_api_asset(cls, asset_id: str, asset: Namespace) -> Namespace: asset.metadata.decimals = 0 return asset - def _assets_from_transactions(self, api: BlockFrostApi) -> pd.DataFrame: + def _assets_from_transactions(self, api: BlockFrostApi) -> pd.Series: all_asset_ids: Set[str] = set() - for tx_obj in self.transactions: + for tx_obj in self.transactions: # pylint: disable=not-an-iterable if hasattr(tx_obj, "utxos"): all_asset_ids.update( [a.unit for i in tx_obj.utxos.inputs for a in i.amount] + [a.unit for i in tx_obj.utxos.outputs for a in i.amount] ) - lovelace_asset_obj = Namespace() - lovelace_asset_obj.metadata = Namespace() + lovelace_asset_obj = blockfrost.utils.Namespace() + lovelace_asset_obj.metadata = blockfrost.utils.Namespace() lovelace_asset_obj.metadata.name = "ADA" lovelace_asset_obj.metadata.decimals = self.LOVELACE_DECIMALS lovelace_asset_obj.policy_id = "" @@ -129,7 +133,7 @@ def _assets_from_transactions(self, api: BlockFrostApi) -> pd.DataFrame: ) for asset in all_asset_ids ] - assets = pd.DataFrame( + return pd.Series( data=asset_list, index=pd.MultiIndex.from_tuples( [ @@ -137,20 +141,21 @@ def _assets_from_transactions(self, api: BlockFrostApi) -> pd.DataFrame: for asset in asset_list ] ), - ) - return assets + ).sort_index() @staticmethod def _reward_transaction( - api: BlockFrostApi, reward: Namespace, pools: Mapping[str, Namespace] - ) -> Namespace: - result = Namespace() + api: BlockFrostApi, + reward: blockfrost.utils.Namespace, + pools: Mapping[str, blockfrost.utils.Namespace], + ) -> blockfrost.utils.Namespace: + result = blockfrost.utils.Namespace() result.tx_hash = None pool_name = ( pools[reward.pool_id].name if reward.pool_id in pools else reward.pool_id ) result.metadata = [ - Namespace( + blockfrost.utils.Namespace( label="674", json_metadata=f"Reward: {reward.type} - {pool_name} - {reward.epoch}", ) @@ -164,7 +169,7 @@ def _reward_transaction( result.redeemers = [] result.hash = None result.withdrawals = [] - result.utxos = Namespace() + result.utxos = blockfrost.utils.Namespace() result.utxos.inputs = [] result.utxos.outputs = [] result.utxos.nonref_inputs = [] @@ -186,7 +191,11 @@ def _reward_transactions(self, api: BlockFrostApi) -> pd.Series: self._reward_transaction(api=api, reward=a_r, pools=pool_result_list) for a_r in reward_list ] - return pd.Series(name="Rewards", data=reward_result_list) + return pd.Series( + name="Rewards", + data=reward_result_list, + index=[t.hash for t in reward_result_list], + ).sort_index() TRANSACTION_OFFSET = np.timedelta64(1000, "ns") @@ -238,17 +247,17 @@ def _is_hex_number(num: Any) -> bool: ) ) - def _munge_metadata(self, namespace_obj) -> Any: - if isinstance(namespace_obj, Namespace): + def _munge_metadata(self, obj: blockfrost.utils.Namespace) -> Any: + if isinstance(obj, blockfrost.utils.Namespace): result = {} - for att in dir(namespace_obj): + for att in dir(obj): if att.startswith("_") or att in ( "to_json", "to_dict", ): continue hex_name = self._is_hex_number(att) - value = getattr(namespace_obj, att) + value = getattr(obj, att) hex_value = isinstance(value, str) and self._is_hex_number(value) if (hex_name and hex_value) and not self.unmute: continue @@ -257,16 +266,12 @@ def _munge_metadata(self, namespace_obj) -> Any: if value: result[out_att] = value return result - elif ( - isinstance(namespace_obj, str) - and self._is_hex_number(namespace_obj) - and not self.unmute - ): + elif isinstance(obj, str) and self._is_hex_number(obj) and not self.unmute: return {} else: - return namespace_obj + return obj - def _format_message(self, tx_obj: Namespace) -> str: + def _format_message(self, tx_obj: blockfrost.utils.Namespace) -> str: result: List[str] = [] for metadata_key in tx_obj.metadata: if metadata_key.label in self.known_dict[self.LABELS_KEY]: @@ -323,10 +328,10 @@ def _decimals_for_asset(self, asset: str) -> np.longlong: return np.longlong(self.data.assets[asset].metadata.decimals) def _asset_tuple(self, asset_id: str) -> Tuple: - asset = self.data.assets[0][(asset_id,)][0] + asset = self.data.assets[(asset_id,)].iloc[0] return (asset.policy_id, asset.raw_name) - def _transaction_balance(self, transaction: Namespace) -> Any: + def _transaction_balance(self, transaction: blockfrost.utils.Namespace) -> Any: result: MutableMapping[Tuple, np.longlong] = defaultdict(lambda: np.longlong(0)) result[ self._asset_tuple(self.data.LOVELACE_ASSET) + ("fees", True) @@ -369,7 +374,7 @@ def _transaction_balance(self, transaction: Namespace) -> Any: return result @staticmethod - def _extract_timestamp(transaction: Namespace) -> Any: + def _extract_timestamp(transaction: blockfrost.utils.Namespace) -> Any: return np.datetime64( datetime.datetime.fromtimestamp(transaction.block_time) ) + (int(transaction.index) * TRANSACTION_OFFSET) @@ -386,7 +391,7 @@ def _drop_foreign_assets(self, balance: pd.DataFrame) -> None: x[:2] for x in balance.xs(True, level=-1, axis=1).columns ) - + balance.sort_index(inplace=True, axis=1) balance.drop(assets_to_drop, axis=1, inplace=True) def _drop_muted_policies(self, balance: pd.DataFrame) -> None: @@ -399,6 +404,7 @@ def _drop_muted_policies(self, balance: pd.DataFrame) -> None: policies_to_drop = frozenset( [x[0] for x in balance.columns if x[0] in policies_to_mute] ) + balance.sort_index(inplace=True, axis=1) balance.drop(policies_to_drop, axis=1, inplace=True) def _relabel_assets(self, balance: pd.DataFrame) -> None: @@ -442,15 +448,15 @@ def make_transaction_frame(self) -> pd.DataFrame: balance.sort_index(axis=1, level=0, sort_remaining=True, inplace=True) balance_column_index_length = len(balance.columns[0]) - frame = pd.concat([timestamp, tx_hash, message], axis=1) + frame.reset_index(drop=True, inplace=True) frame.columns = pd.MultiIndex.from_tuples( [ ("metadata", c) + (balance_column_index_length - 2) * ("",) for c in frame.columns ] ) - frame = frame.merge(balance, left_index=True, right_index=True) + frame = frame.join(balance) frame.drop_duplicates(inplace=True) frame.sort_values(by=frame.columns[0], inplace=True) return frame From a87078df42292fda4f3d90d8c9729a75fc7ce6c2 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Tue, 12 Sep 2023 11:14:45 +0200 Subject: [PATCH 025/124] more refactoring --- src/cardano_account_pandas_dumper/__main__.py | 8 +---- .../cardano_account_pandas_dumper.py | 36 +++++++------------ 2 files changed, 14 insertions(+), 30 deletions(-) diff --git a/src/cardano_account_pandas_dumper/__main__.py b/src/cardano_account_pandas_dumper/__main__.py index 72e7705..c448f15 100644 --- a/src/cardano_account_pandas_dumper/__main__.py +++ b/src/cardano_account_pandas_dumper/__main__.py @@ -157,13 +157,7 @@ def main(): else: parser.exit(status=1, message="Staking address(es) required.") reporter = AccountPandasDumper( - data=data_from_api, - known_dict=known_dict_from_file, - detail_level=args.detail_level, - unmute=args.unmute, - truncate_length=None if args.no_truncate else args.truncate_length, - raw_asset=args.raw_asset or False, - rewards=data_from_api.rewards, + data=data_from_api, known_dict=known_dict_from_file, args=args ) dataframe = reporter.make_transaction_frame() if args.pandas_output: diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 733629d..399c7a3 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -1,4 +1,5 @@ """ Cardano Account To Pandas Dumper.""" +import argparse import datetime import functools import itertools @@ -210,30 +211,19 @@ class AccountPandasDumper: SCRIPTS_KEY = "scripts" LABELS_KEY = "labels" - def __init__( - self, - data: AccountData, - known_dict: Any, - detail_level: int, - unmute: bool, - truncate_length: Optional[int], - raw_asset: bool, - rewards: bool, - ): - self.known_dict = known_dict + def __init__(self, data: AccountData, known_dict: Any, args: argparse.Namespace): self.data = data - self.truncate_length = truncate_length - self.detail_level = detail_level - self.unmute = unmute - self.raw_asset = raw_asset - self.rewards = rewards + self.known_dict = known_dict + self.args = args def _format_asset(self, asset: str) -> Optional[str]: return self.data.assets[asset].metadata.name def _truncate(self, value: str) -> str: return ( - (value[: self.truncate_length] + "...") if self.truncate_length else value + value + if self.args.no_truncate + else (value[: self.args.truncate_length] + "...") ) def _format_policy(self, policy: str) -> Optional[str]: @@ -259,14 +249,14 @@ def _munge_metadata(self, obj: blockfrost.utils.Namespace) -> Any: hex_name = self._is_hex_number(att) value = getattr(obj, att) hex_value = isinstance(value, str) and self._is_hex_number(value) - if (hex_name and hex_value) and not self.unmute: + if (hex_name and hex_value) and not self.args.unmute: continue out_att = self._truncate(att) if hex_name else att value = self._munge_metadata(value) if value: result[out_att] = value return result - elif isinstance(obj, str) and self._is_hex_number(obj) and not self.unmute: + elif isinstance(obj, str) and self._is_hex_number(obj) and not self.args.unmute: return {} else: return obj @@ -282,7 +272,7 @@ def _format_message(self, tx_obj: blockfrost.utils.Namespace) -> str: if ( self._is_hex_number(label) and (not val or self._is_hex_number(val)) - and not self.unmute + and not self.args.unmute ): continue result.append(label) @@ -396,7 +386,7 @@ def _drop_foreign_assets(self, balance: pd.DataFrame) -> None: def _drop_muted_policies(self, balance: pd.DataFrame) -> None: if ( - (not self.unmute) + (not self.args.unmute) and self.MUTED_POLICIES_KEY in self.known_dict and self.known_dict[self.MUTED_POLICIES_KEY] ): @@ -431,7 +421,7 @@ def make_transaction_frame(self) -> pd.DataFrame: transactions = pd.concat( objs=[ self.data.transactions, - self.data.reward_transactions if self.rewards else pd.Series(), + pd.Series() if self.args.no_rewards else self.data.reward_transactions, ], ).rename("transactions") timestamp = transactions.rename("timestamp").map(self._extract_timestamp) @@ -457,6 +447,6 @@ def make_transaction_frame(self) -> pd.DataFrame: ] ) frame = frame.join(balance) - frame.drop_duplicates(inplace=True) + # frame.drop_duplicates(inplace=True) frame.sort_values(by=frame.columns[0], inplace=True) return frame From 7e31a6a6f7537d4d0d587733df4f6b2a25417058 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Tue, 12 Sep 2023 11:37:31 +0200 Subject: [PATCH 026/124] fix duplicates --- .../cardano_account_pandas_dumper.py | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 399c7a3..c892538 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -66,32 +66,33 @@ def _transaction_data( self, api: BlockFrostApi, ) -> pd.Series: - result_list = [] + transaction_set = set() for addr in self.own_addresses: for outer_tx in api.address_transactions( addr, to_block=self.to_block, gather_pages=True, ): - transaction = api.transaction(outer_tx.tx_hash) - transaction.utxos = api.transaction_utxos(outer_tx.tx_hash) - transaction.utxos.nonref_inputs = [ - i for i in transaction.utxos.inputs if not i.reference - ] - transaction.metadata = api.transaction_metadata(outer_tx.tx_hash) - transaction.redeemers = ( - api.transaction_redeemers(outer_tx.tx_hash) - if transaction.redeemer_count - else [] - ) - transaction.withdrawals = ( - api.transaction_withdrawals(outer_tx.tx_hash) - if transaction.withdrawal_count - else [] - ) - transaction.reward_amount = None + transaction_set.add(outer_tx.tx_hash) + result_list = [] + for tx_hash in transaction_set: + transaction = api.transaction(tx_hash) + transaction.utxos = api.transaction_utxos(tx_hash) + transaction.utxos.nonref_inputs = [ + i for i in transaction.utxos.inputs if not i.reference + ] + transaction.metadata = api.transaction_metadata(tx_hash) + transaction.redeemers = ( + api.transaction_redeemers(tx_hash) if transaction.redeemer_count else [] + ) + transaction.withdrawals = ( + api.transaction_withdrawals(tx_hash) + if transaction.withdrawal_count + else [] + ) + transaction.reward_amount = None - result_list.append(transaction) + result_list.append(transaction) return pd.Series( name="Transactions", data=result_list, index=[t.hash for t in result_list] ).sort_index() @@ -447,6 +448,5 @@ def make_transaction_frame(self) -> pd.DataFrame: ] ) frame = frame.join(balance) - # frame.drop_duplicates(inplace=True) frame.sort_values(by=frame.columns[0], inplace=True) return frame From 6da1aa9f39a2adea5dff65c755d696a424d72ed0 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Tue, 12 Sep 2023 12:16:22 +0200 Subject: [PATCH 027/124] cleanup --- .../cardano_account_pandas_dumper.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index c892538..596164f 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -200,9 +200,6 @@ def _reward_transactions(self, api: BlockFrostApi) -> pd.Series: ).sort_index() -TRANSACTION_OFFSET = np.timedelta64(1000, "ns") - - class AccountPandasDumper: """Hold logic to convert an instance of AccountData to a Pandas dataframe.""" @@ -211,6 +208,7 @@ class AccountPandasDumper: MUTED_POLICIES_KEY = "muted_policies" SCRIPTS_KEY = "scripts" LABELS_KEY = "labels" + TRANSACTION_OFFSET = np.timedelta64(1000, "ns") def __init__(self, data: AccountData, known_dict: Any, args: argparse.Namespace): self.data = data @@ -364,11 +362,11 @@ def _transaction_balance(self, transaction: blockfrost.utils.Namespace) -> Any: return result - @staticmethod - def _extract_timestamp(transaction: blockfrost.utils.Namespace) -> Any: + @classmethod + def _extract_timestamp(cls, transaction: blockfrost.utils.Namespace) -> Any: return np.datetime64( datetime.datetime.fromtimestamp(transaction.block_time) - ) + (int(transaction.index) * TRANSACTION_OFFSET) + ) + (int(transaction.index) * cls.TRANSACTION_OFFSET) def _drop_foreign_assets(self, balance: pd.DataFrame) -> None: # Drop assets that only touch foreign addresses @@ -436,14 +434,12 @@ def make_transaction_frame(self) -> pd.DataFrame: self._drop_muted_policies(balance) # self._relabel_assets(balance) balance.columns = pd.MultiIndex.from_tuples(balance.columns) - balance.sort_index(axis=1, level=0, sort_remaining=True, inplace=True) - balance_column_index_length = len(balance.columns[0]) frame = pd.concat([timestamp, tx_hash, message], axis=1) frame.reset_index(drop=True, inplace=True) frame.columns = pd.MultiIndex.from_tuples( [ - ("metadata", c) + (balance_column_index_length - 2) * ("",) + ("metadata", c) + (len(balance.columns[0]) - 2) * ("",) for c in frame.columns ] ) From daec5e63e7a0f4e63323f29cb005e0c9af44e4ef Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Tue, 12 Sep 2023 17:47:38 +0200 Subject: [PATCH 028/124] raw_assets -> raw_values --- src/cardano_account_pandas_dumper/__main__.py | 4 +- .../cardano_account_pandas_dumper.py | 96 +++++++++---------- src/cardano_account_pandas_dumper/known.jsonc | 75 ++++++++------- 3 files changed, 87 insertions(+), 88 deletions(-) diff --git a/src/cardano_account_pandas_dumper/__main__.py b/src/cardano_account_pandas_dumper/__main__.py index c448f15..2be9c96 100644 --- a/src/cardano_account_pandas_dumper/__main__.py +++ b/src/cardano_account_pandas_dumper/__main__.py @@ -82,8 +82,8 @@ def _create_arg_parser(): action="store_true", ) result.add_argument( - "--raw_asset", - help="Add header row with concatenation of policy_id and hex-encoded asset_name.", + "--raw_values", + help="Keep assets, policies and addresses as hex instead of looking up names.", action="store_true", ) result.add_argument( diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 596164f..9530b48 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -66,16 +66,18 @@ def _transaction_data( self, api: BlockFrostApi, ) -> pd.Series: - transaction_set = set() - for addr in self.own_addresses: - for outer_tx in api.address_transactions( - addr, - to_block=self.to_block, - gather_pages=True, - ): - transaction_set.add(outer_tx.tx_hash) result_list = [] - for tx_hash in transaction_set: + for tx_hash in frozenset( + [ + outer_tx.tx_hash + for addr in self.own_addresses + for outer_tx in api.address_transactions( + addr, + to_block=self.to_block, + gather_pages=True, + ) + ] + ): transaction = api.transaction(tx_hash) transaction.utxos = api.transaction_utxos(tx_hash) transaction.utxos.nonref_inputs = [ @@ -203,20 +205,17 @@ def _reward_transactions(self, api: BlockFrostApi) -> pd.Series: class AccountPandasDumper: """Hold logic to convert an instance of AccountData to a Pandas dataframe.""" - ADDRESSES_KEY = "addresses" - POLICIES_KEY = "policies" - MUTED_POLICIES_KEY = "muted_policies" - SCRIPTS_KEY = "scripts" - LABELS_KEY = "labels" TRANSACTION_OFFSET = np.timedelta64(1000, "ns") def __init__(self, data: AccountData, known_dict: Any, args: argparse.Namespace): self.data = data self.known_dict = known_dict self.args = args - - def _format_asset(self, asset: str) -> Optional[str]: - return self.data.assets[asset].metadata.name + self.addresses = pd.Series(known_dict.get("addresses", {})) + self.policies = pd.Series(known_dict.get("policies", {})) + self.muted_policies = pd.Series(known_dict.get("muted_policies", [])) + self.scripts = pd.Series(known_dict.get("scripts", {})) + self.labels = pd.Series(known_dict.get("labels", {})) def _truncate(self, value: str) -> str: return ( @@ -225,9 +224,6 @@ def _truncate(self, value: str) -> str: else (value[: self.args.truncate_length] + "...") ) - def _format_policy(self, policy: str) -> Optional[str]: - return self.known_dict[self.POLICIES_KEY].get(policy, self._truncate(policy)) - @staticmethod def _is_hex_number(num: Any) -> bool: return isinstance(num, str) and not bool( @@ -263,10 +259,9 @@ def _munge_metadata(self, obj: blockfrost.utils.Namespace) -> Any: def _format_message(self, tx_obj: blockfrost.utils.Namespace) -> str: result: List[str] = [] for metadata_key in tx_obj.metadata: - if metadata_key.label in self.known_dict[self.LABELS_KEY]: - label = self.known_dict[self.LABELS_KEY][metadata_key.label] - else: - label = metadata_key.label + label = self.labels.get( + metadata_key.label, self._truncate(metadata_key.label) + ) val = self._munge_metadata(metadata_key.json_metadata) if ( self._is_hex_number(label) @@ -300,18 +295,19 @@ def _format_message(self, tx_obj: blockfrost.utils.Namespace) -> str: return " ".join(result).removeprefix("Message : ") def _format_script(self, script: str) -> str: - return self.known_dict[self.SCRIPTS_KEY].get( - script, - self._truncate(script), - ) + return self.scripts.get(script, self._truncate(script)) def _format_address(self, address: str) -> str: - if address in self.data.own_addresses: - return " own" - return self.known_dict[self.ADDRESSES_KEY].get( - address, - "other", - ) + return self.addresses.get(address, self._truncate(address)) + + def _format_asset_name(self, name: str) -> str: + try: + return bytes.fromhex(name).decode() + except UnicodeDecodeError: + return name + + def _format_policy(self, policy: str) -> Optional[str]: + return self.policies.get(policy, self._truncate(policy)) def _decimals_for_asset(self, asset: str) -> np.longlong: return np.longlong(self.data.assets[asset].metadata.decimals) @@ -384,23 +380,18 @@ def _drop_foreign_assets(self, balance: pd.DataFrame) -> None: balance.drop(assets_to_drop, axis=1, inplace=True) def _drop_muted_policies(self, balance: pd.DataFrame) -> None: - if ( - (not self.args.unmute) - and self.MUTED_POLICIES_KEY in self.known_dict - and self.known_dict[self.MUTED_POLICIES_KEY] - ): - policies_to_mute = frozenset(self.known_dict[self.MUTED_POLICIES_KEY]) - policies_to_drop = frozenset( - [x[0] for x in balance.columns if x[0] in policies_to_mute] - ) - balance.sort_index(inplace=True, axis=1) - balance.drop(policies_to_drop, axis=1, inplace=True) + policies_to_drop = frozenset( + [x[0] for x in balance.columns if x[0] in self.muted_policies] + ) + balance.sort_index(inplace=True, axis=1) + balance.drop(policies_to_drop, axis=1, inplace=True) def _relabel_assets(self, balance: pd.DataFrame) -> None: - new_columns = [ - (self.data.assets[x[0]].metadata.name,) + x[1:] for x in balance.columns + new_columns: List[Tuple] = [ + (self._format_policy(x[0]), self._format_asset_name(x[1])) + tuple(x[2:]) + for x in balance.columns ] - balance.columns = new_columns + balance.columns = pd.MultiIndex.from_tuples(new_columns) def make_transaction_frame(self) -> pd.DataFrame: """Build a transaction spreadsheet.""" @@ -431,8 +422,13 @@ def make_transaction_frame(self) -> pd.DataFrame: dtype=pd.Int64Dtype, ) self._drop_foreign_assets(balance) - self._drop_muted_policies(balance) - # self._relabel_assets(balance) + if not self.args.unmute: + self._drop_muted_policies(balance) + if self.args.detail_level == 1: + balance.drop(labels=False, axis=1, level=3, inplace=True) + if not self.args.raw_values: + self._relabel_assets(balance) + balance.columns = pd.MultiIndex.from_tuples(balance.columns) balance.sort_index(axis=1, level=0, sort_remaining=True, inplace=True) frame = pd.concat([timestamp, tx_hash, message], axis=1) diff --git a/src/cardano_account_pandas_dumper/known.jsonc b/src/cardano_account_pandas_dumper/known.jsonc index 71d6fc6..1c5b28e 100644 --- a/src/cardano_account_pandas_dumper/known.jsonc +++ b/src/cardano_account_pandas_dumper/known.jsonc @@ -5,7 +5,6 @@ "addr1v95sf69jcfhnmknvffwmfvlvnccatqwfjcyh0nlfc6gh5scta2yzg": "CoinBase", "addr1v9u7va2sktlnz8dp3qadpnxxlv4m03672jy6elmntlxxs7q9nnwl9": "CoinBase", "addr1vydqq7fww4sga05jvtfckw3z9z8n8n2zxj0fyelvdl86acqhd9l70": "CoinBase", - "addr1wy49natatgz5swjenvvqatwn48uj5p0lvw86uhvjjuth8xs2yr9fw": "DripDropz", "addr1w80ptp0qgmcklhmeweesqgeurtlma8fsxsr9dt8au30fzss0czhl9": "Indigo Protocol (Stability pool)", "addr1w849xkxel6pvc7kcmc88dd82hkz32fjq3ega42wchd93xlgqxxpv5": "Indigo Protocol (CDP creator)", "addr1w89s3lfv7gkugker5llecq6x3k2vjvfnvp4692laeqe6w6s93vj3j": "Explosif (Direct sale)", @@ -24,6 +23,7 @@ "addr1wxr2a8htmzuhj39y2gq7ftkpxv98y2g67tg8zezthgq4jkg0a4ul4": "Wingriders (Request)", "addr1wxvd7wcq59gqljmhm2s9yp2slvygljfr8xtc3wykx7uaukgt8lqh6": "Minswap (Harvest v1)", "addr1wxwl25gyxf4ryexcq02yakr389vp68y39cnt4rnnhsu9utcjfhaef": "Minswap (MINt staking liquidity)", + "addr1wy49natatgz5swjenvvqatwn48uj5p0lvw86uhvjjuth8xs2yr9fw": "DripDropz", "addr1wycwqtlc2a3f3w4axqwwukyj34anyqmy404w24kydc70gtg5w4elq": "MELD (staking 2)", "addr1wypr0np3xatwhddulsnj3aaac65qg768zgs2xpd2xuaj0zscmvh0n": "Wingriders (Farm/Voting)", "addr1z8nvjzjeydcn4atcd93aac8allvrpjn7pjr2qsweukpnay0ycn5y6v5mjqqcs9xqa24kv5qk73tqsf3ufrtvxh9kts0qveyg6l": "Wingriders (Pool payment credential)", @@ -39,15 +39,49 @@ "addr1zxj47sy4qxlktqzmkrw8dahe46gtv8seakrshsqz26qnvzypw288a4x0xf8pxgcntelxmyclq83s0ykeehchz2wtspksr3q9nx": "jpg.store (Offers)", "addr1zyq0kyrml023kwjk8zr86d5gaxrt5w8lxnah8r6m6s4jp4g3r6dxnzml343sx8jweqn4vn3fz2kj8kgu9czghx0jrsyqqktyhv": "MuesliSwap (Partial Match Order)" }, + // Known metadata labels + "labels": { + "61284": "Catalyst", + "61285": "Catalyst Sig", + "674": "Message", + "914425": "Indigo SP Reward" + }, + // Policies to mute + "muted_policies": [ + "0029cb7c88c7567b63d1a512c0ed626aa169688ec980730c0473b913", + "026a18d04a0c642759bb3d83b12e3344894e5c1c7b2aeb1a2113a570", + "0be55d262b29f564998ff81efe21bdc0022621c12f15af08d0f2ddb1", + "13aa2accf2e1561723aa26871e071fdf32c867cff7e7d50ad470d62f", + "24b458412c2a7f9acb9c53c7ec4325b36806912ed56d2f91bfcf4d26", + "2f2e0404310c106e2a260e8eb5a7e43f00cff42c667489d30e179816", + "2fccae8bc1c8553a2185b2e77ccdea22f2e1d6e87beb80ef4eaf8cce", + "408f13b240f57c3473b5727a68a88c50c1b6bb15c0c10912008dc59e", + "5817c34e5702473304f3cf676299176d3824e55b8c0bfa94830429fd", + "636c06ed152e70e9a1240da585a2841c15e4c3174044ca9a62cadab2", + "72c8ee38fc1adfc5db49f7384bd64d41e9b9ce46ec11ba9a292e5d02", + "8a7b0a43a169218fb66996c3945f53d7712eae9e31db193adb55663f", + "909133088303c49f3a30f1cc8ed553a73857a29779f6c6561cd8093f", + "97da12de04a6b527cc3b3469c5e5485cf258dfd1021f12e728f2e714", + "af3d70acf4bd5b3abb319a7d75c89fb3e56eafcdd46b2e9b57a2557f", + "d195ca7db29f0f13a00cac7fca70426ff60bad4e1e87d3757fae8484", + "de9b756719341e79785aa13c164e7fe68c189ed04d61c9876b2fe53f", + "e3455f2715338b454fb853442f72dc03b98396854f97510027fe22ff", + "e4214b7cce62ac6fbba385d164df48e157eae5863521b4b67ca71d86", + "e8a447d4e19016ca2aa74d20b4c4de87adb1f21dfb5493bf2d7281a6", + "f66d78b4a3cb3d37afa0ec36461e51ecbde00f26c8f0a68f94b69880", + "21ca2902ad0406bc2ce361ad91157b3542544d86d4e3d3e3c43f1763", + "2960fe7a50e78948eb7219079c39b8169e34a4464a3f1aea9328d87c" + ], // Known policies "policies": { - "0029cb7c88c7567b63d1a512c0ed626aa169688ec980730c0473b913": "Sundaeswap", + "0029cb7c88c7567b63d1a512c0ed626aa169688ec980730c0473b913": "Sundaeswap LP", "026a18d04a0c642759bb3d83b12e3344894e5c1c7b2aeb1a2113a570": "Wingriders Factory/Liquidity token v1", "0be55d262b29f564998ff81efe21bdc0022621c12f15af08d0f2ddb1": "Minswap LP NFT 2", "13aa2accf2e1561723aa26871e071fdf32c867cff7e7d50ad470d62f": "Minswap LP NFT 2", "21ca2902ad0406bc2ce361ad91157b3542544d86d4e3d3e3c43f1763": "VyFI NFT 1", "24b458412c2a7f9acb9c53c7ec4325b36806912ed56d2f91bfcf4d26": "INDY staking NFT", "2960fe7a50e78948eb7219079c39b8169e34a4464a3f1aea9328d87c": "VyFI NFT 2", + "29d222ce763455e3d7a09a665ce554f00ac89d2e99a1a83d267170c6": "Minswap policy", "2f2e0404310c106e2a260e8eb5a7e43f00cff42c667489d30e179816": "Minswap NFT ?", "2fccae8bc1c8553a2185b2e77ccdea22f2e1d6e87beb80ef4eaf8cce": "INDY Gov NFT", "408f13b240f57c3473b5727a68a88c50c1b6bb15c0c10912008dc59e": "INDY iBTC mint NFT", @@ -57,6 +91,7 @@ "636c06ed152e70e9a1240da585a2841c15e4c3174044ca9a62cadab2": "NFT test", "708f5e6d597fc038d09a738d7be32edd6ea779d6feb32a53668d9050": "INDY CDP NFT", "72c8ee38fc1adfc5db49f7384bd64d41e9b9ce46ec11ba9a292e5d02": "NFT test", + "869fc72c11977e4be3e8e1cc63cca008a925886332795c9601f965ca": "USDC policy", "8a7b0a43a169218fb66996c3945f53d7712eae9e31db193adb55663f": "NFT test", "909133088303c49f3a30f1cc8ed553a73857a29779f6c6561cd8093f": "MuesliSwap NFT Policy v2", "97da12de04a6b527cc3b3469c5e5485cf258dfd1021f12e728f2e714": "INDY IASSET NFT", @@ -67,34 +102,9 @@ "e4214b7cce62ac6fbba385d164df48e157eae5863521b4b67ca71d86": "Minswap NFT", "e8a447d4e19016ca2aa74d20b4c4de87adb1f21dfb5493bf2d7281a6": "Sundae scooper", "f66d78b4a3cb3d37afa0ec36461e51ecbde00f26c8f0a68f94b69880": "Indigo", - "fd0d72fafee1d230a74c31ac503a192abd5b71888ae3f94128c1e634": "INDY staking NFT" + "fd0d72fafee1d230a74c31ac503a192abd5b71888ae3f94128c1e634": "INDY staking NFT", + "9a9693a9a37912a5097918f97918d15240c92ab729a0b7c4aa144d77": "Sundae policy", }, - // Policies to mute - "muted_policies": [ - "0029cb7c88c7567b63d1a512c0ed626aa169688ec980730c0473b913", - "026a18d04a0c642759bb3d83b12e3344894e5c1c7b2aeb1a2113a570", - "0be55d262b29f564998ff81efe21bdc0022621c12f15af08d0f2ddb1", - "13aa2accf2e1561723aa26871e071fdf32c867cff7e7d50ad470d62f", - "24b458412c2a7f9acb9c53c7ec4325b36806912ed56d2f91bfcf4d26", - "2f2e0404310c106e2a260e8eb5a7e43f00cff42c667489d30e179816", - "2fccae8bc1c8553a2185b2e77ccdea22f2e1d6e87beb80ef4eaf8cce", - "408f13b240f57c3473b5727a68a88c50c1b6bb15c0c10912008dc59e", - "5817c34e5702473304f3cf676299176d3824e55b8c0bfa94830429fd", - "636c06ed152e70e9a1240da585a2841c15e4c3174044ca9a62cadab2", - "72c8ee38fc1adfc5db49f7384bd64d41e9b9ce46ec11ba9a292e5d02", - "8a7b0a43a169218fb66996c3945f53d7712eae9e31db193adb55663f", - "909133088303c49f3a30f1cc8ed553a73857a29779f6c6561cd8093f", - "97da12de04a6b527cc3b3469c5e5485cf258dfd1021f12e728f2e714", - "af3d70acf4bd5b3abb319a7d75c89fb3e56eafcdd46b2e9b57a2557f", - "d195ca7db29f0f13a00cac7fca70426ff60bad4e1e87d3757fae8484", - "de9b756719341e79785aa13c164e7fe68c189ed04d61c9876b2fe53f", - "e3455f2715338b454fb853442f72dc03b98396854f97510027fe22ff", - "e4214b7cce62ac6fbba385d164df48e157eae5863521b4b67ca71d86", - "e8a447d4e19016ca2aa74d20b4c4de87adb1f21dfb5493bf2d7281a6", - "f66d78b4a3cb3d37afa0ec36461e51ecbde00f26c8f0a68f94b69880", - "21ca2902ad0406bc2ce361ad91157b3542544d86d4e3d3e3c43f1763", - "2960fe7a50e78948eb7219079c39b8169e34a4464a3f1aea9328d87c" - ], // Known script hashes "scripts": { "00fb107bfbd51b3a5638867d3688e986ba38ff34fb738f5bd42b20d5": "MuesliSwap (Partial Match Order)", @@ -124,12 +134,5 @@ "e4d2fb0b8d275852103fd75801e2c7dcf6ed3e276c74cabadbe5b8b6": "Indigo Protocol (CDP)", "e6c90a5923713af5786963dee0fdffd830ca7e0c86a041d9e5833e91": "Wingriders (Pool payment credential)", "ea5358d9fe82cc7ad8de0e76b4eabd851526408e51daa9d8bb4b137d": "Indigo Protocol (CDP creator)" - }, - // Known metadata labels - "labels": { - "674": "Message", - "61284": "Catalyst", - "61285": "Catalyst Sig", - "914425": "Indigo SP Reward" } } \ No newline at end of file From 98c7497a67722de697686f29c4d423e5abaad7b2 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Wed, 13 Sep 2023 17:41:09 +0200 Subject: [PATCH 029/124] add some policies --- src/cardano_account_pandas_dumper/known.jsonc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cardano_account_pandas_dumper/known.jsonc b/src/cardano_account_pandas_dumper/known.jsonc index 1c5b28e..ed48c84 100644 --- a/src/cardano_account_pandas_dumper/known.jsonc +++ b/src/cardano_account_pandas_dumper/known.jsonc @@ -95,6 +95,7 @@ "8a7b0a43a169218fb66996c3945f53d7712eae9e31db193adb55663f": "NFT test", "909133088303c49f3a30f1cc8ed553a73857a29779f6c6561cd8093f": "MuesliSwap NFT Policy v2", "97da12de04a6b527cc3b3469c5e5485cf258dfd1021f12e728f2e714": "INDY IASSET NFT", + "9a9693a9a37912a5097918f97918d15240c92ab729a0b7c4aa144d77": "Sundae policy", "af3d70acf4bd5b3abb319a7d75c89fb3e56eafcdd46b2e9b57a2557f": "MuesliSwap NFT", "d195ca7db29f0f13a00cac7fca70426ff60bad4e1e87d3757fae8484": "Minswap LP NFT 1", "de9b756719341e79785aa13c164e7fe68c189ed04d61c9876b2fe53f": "MuesliSwap Factory Policy v2", @@ -102,8 +103,7 @@ "e4214b7cce62ac6fbba385d164df48e157eae5863521b4b67ca71d86": "Minswap NFT", "e8a447d4e19016ca2aa74d20b4c4de87adb1f21dfb5493bf2d7281a6": "Sundae scooper", "f66d78b4a3cb3d37afa0ec36461e51ecbde00f26c8f0a68f94b69880": "Indigo", - "fd0d72fafee1d230a74c31ac503a192abd5b71888ae3f94128c1e634": "INDY staking NFT", - "9a9693a9a37912a5097918f97918d15240c92ab729a0b7c4aa144d77": "Sundae policy", + "fd0d72fafee1d230a74c31ac503a192abd5b71888ae3f94128c1e634": "INDY staking NFT" }, // Known script hashes "scripts": { From 7645ff21f56ff9eba19137f1dfef5ea2bf78f478 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Fri, 15 Sep 2023 11:49:57 +0200 Subject: [PATCH 030/124] finish labeling and grouping --- .../cardano_account_pandas_dumper.py | 86 ++++++++++++------- 1 file changed, 57 insertions(+), 29 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 9530b48..2a3ade7 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -211,8 +211,31 @@ def __init__(self, data: AccountData, known_dict: Any, args: argparse.Namespace) self.data = data self.known_dict = known_dict self.args = args - self.addresses = pd.Series(known_dict.get("addresses", {})) - self.policies = pd.Series(known_dict.get("policies", {})) + self.address_names = ( + pd.concat( + [ + pd.Series( + known_dict.get("addresses", {}), + ), + pd.Series({a: " wallet" for a in self.data.own_addresses}), + ] + ) + if not args.raw_values + else pd.Series() + ) + self.policy_names = ( + pd.Series(known_dict.get("policies", {})) + if not args.raw_values + else pd.Series() + ) + self.asset_names = pd.Series( + { + asset.asset_id: self._decode_asset_name(asset.raw_name) + if not args.raw_values + else self._truncate(asset.raw_name) + for asset in self.data.assets + } + ) self.muted_policies = pd.Series(known_dict.get("muted_policies", [])) self.scripts = pd.Series(known_dict.get("scripts", {})) self.labels = pd.Series(known_dict.get("labels", {})) @@ -220,10 +243,17 @@ def __init__(self, data: AccountData, known_dict: Any, args: argparse.Namespace) def _truncate(self, value: str) -> str: return ( value - if self.args.no_truncate - else (value[: self.args.truncate_length] + "...") + if self.args.no_truncate or len(value) <= self.args.truncate_length + else ("..." + value[-self.args.truncate_length :]) ) + def _decode_asset_name(self, asset_raw_name: str) -> str: + try: + return bytes.fromhex(asset_raw_name).decode() + except UnicodeDecodeError: + pass + return self._truncate(asset_raw_name) + @staticmethod def _is_hex_number(num: Any) -> bool: return isinstance(num, str) and not bool( @@ -297,34 +327,29 @@ def _format_message(self, tx_obj: blockfrost.utils.Namespace) -> str: def _format_script(self, script: str) -> str: return self.scripts.get(script, self._truncate(script)) - def _format_address(self, address: str) -> str: - return self.addresses.get(address, self._truncate(address)) - - def _format_asset_name(self, name: str) -> str: - try: - return bytes.fromhex(name).decode() - except UnicodeDecodeError: - return name - def _format_policy(self, policy: str) -> Optional[str]: - return self.policies.get(policy, self._truncate(policy)) + return self.policy_names.get(policy, self._truncate(policy)) def _decimals_for_asset(self, asset: str) -> np.longlong: return np.longlong(self.data.assets[asset].metadata.decimals) def _asset_tuple(self, asset_id: str) -> Tuple: asset = self.data.assets[(asset_id,)].iloc[0] - return (asset.policy_id, asset.raw_name) + return ( + self._format_policy(asset.policy_id), + self.asset_names.get(asset_id), + ) def _transaction_balance(self, transaction: blockfrost.utils.Namespace) -> Any: + # Index: (policy,asset,address,address_name,own) result: MutableMapping[Tuple, np.longlong] = defaultdict(lambda: np.longlong(0)) result[ - self._asset_tuple(self.data.LOVELACE_ASSET) + ("fees", True) + self._asset_tuple(self.data.LOVELACE_ASSET) + ("", " fees", True) ] = np.longlong(transaction.fees) result[ - self._asset_tuple(self.data.LOVELACE_ASSET) + ("deposit", True) + self._asset_tuple(self.data.LOVELACE_ASSET) + ("", " deposit", True) ] = np.longlong(transaction.deposit) - result[self._asset_tuple(self.data.LOVELACE_ASSET) + ("rewards", True)] = ( + result[self._asset_tuple(self.data.LOVELACE_ASSET) + ("", " rewards", True)] = ( np.negative( functools.reduce( np.add, @@ -342,6 +367,12 @@ def _transaction_balance(self, transaction: blockfrost.utils.Namespace) -> Any: self._asset_tuple(amount.unit) + ( utxo.address, + self.address_names.get( + utxo.address, + self._truncate(utxo.address) + if self.args.raw_values + else "other", + ), utxo.address in self.data.own_addresses, ) ] -= np.longlong(amount.quantity) @@ -352,6 +383,12 @@ def _transaction_balance(self, transaction: blockfrost.utils.Namespace) -> Any: self._asset_tuple(amount.unit) + ( utxo.address, + self.address_names.get( + utxo.address, + self._truncate(utxo.address) + if self.args.raw_values + else "other", + ), utxo.address in self.data.own_addresses, ) ] += np.longlong(amount.quantity) @@ -386,13 +423,6 @@ def _drop_muted_policies(self, balance: pd.DataFrame) -> None: balance.sort_index(inplace=True, axis=1) balance.drop(policies_to_drop, axis=1, inplace=True) - def _relabel_assets(self, balance: pd.DataFrame) -> None: - new_columns: List[Tuple] = [ - (self._format_policy(x[0]), self._format_asset_name(x[1])) + tuple(x[2:]) - for x in balance.columns - ] - balance.columns = pd.MultiIndex.from_tuples(new_columns) - def make_transaction_frame(self) -> pd.DataFrame: """Build a transaction spreadsheet.""" @@ -425,12 +455,10 @@ def make_transaction_frame(self) -> pd.DataFrame: if not self.args.unmute: self._drop_muted_policies(balance) if self.args.detail_level == 1: - balance.drop(labels=False, axis=1, level=3, inplace=True) - if not self.args.raw_values: - self._relabel_assets(balance) - + balance.drop(labels=False, axis=1, level=4, inplace=True) balance.columns = pd.MultiIndex.from_tuples(balance.columns) balance.sort_index(axis=1, level=0, sort_remaining=True, inplace=True) + balance = balance.groupby(axis=1, level=(0, 1, 3)).sum(numeric_only=True) frame = pd.concat([timestamp, tx_hash, message], axis=1) frame.reset_index(drop=True, inplace=True) frame.columns = pd.MultiIndex.from_tuples( From 4888064281c8f7f45c888f7b69d4d0618898fbfc Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Fri, 15 Sep 2023 17:09:45 +0200 Subject: [PATCH 031/124] fix warnings --- .../cardano_account_pandas_dumper.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 2a3ade7..fd8a9ee 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -449,7 +449,6 @@ def make_transaction_frame(self) -> pd.DataFrame: message = transactions.rename("message").map(self._format_message) balance = pd.DataFrame( data=[self._transaction_balance(x) for x in transactions], - dtype=pd.Int64Dtype, ) self._drop_foreign_assets(balance) if not self.args.unmute: @@ -458,7 +457,7 @@ def make_transaction_frame(self) -> pd.DataFrame: balance.drop(labels=False, axis=1, level=4, inplace=True) balance.columns = pd.MultiIndex.from_tuples(balance.columns) balance.sort_index(axis=1, level=0, sort_remaining=True, inplace=True) - balance = balance.groupby(axis=1, level=(0, 1, 3)).sum(numeric_only=True) + balance = balance.T.groupby(level=(0, 1, 3)).sum().T frame = pd.concat([timestamp, tx_hash, message], axis=1) frame.reset_index(drop=True, inplace=True) frame.columns = pd.MultiIndex.from_tuples( From 8dc1138c162997fd2a04451b78b31c42151d39e1 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Sat, 16 Sep 2023 13:17:20 +0200 Subject: [PATCH 032/124] add decimals --- .../cardano_account_pandas_dumper.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index fd8a9ee..16bc178 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -331,17 +331,18 @@ def _format_policy(self, policy: str) -> Optional[str]: return self.policy_names.get(policy, self._truncate(policy)) def _decimals_for_asset(self, asset: str) -> np.longlong: - return np.longlong(self.data.assets[asset].metadata.decimals) + return np.longlong(self.data.assets[(asset,)].iloc(0)[0].metadata.decimals) def _asset_tuple(self, asset_id: str) -> Tuple: asset = self.data.assets[(asset_id,)].iloc[0] return ( self._format_policy(asset.policy_id), self.asset_names.get(asset_id), + self._decimals_for_asset(asset_id), ) def _transaction_balance(self, transaction: blockfrost.utils.Namespace) -> Any: - # Index: (policy,asset,address,address_name,own) + # Index: (policy,asset,decimals, address,address_name,own) result: MutableMapping[Tuple, np.longlong] = defaultdict(lambda: np.longlong(0)) result[ self._asset_tuple(self.data.LOVELACE_ASSET) + ("", " fees", True) @@ -449,15 +450,18 @@ def make_transaction_frame(self) -> pd.DataFrame: message = transactions.rename("message").map(self._format_message) balance = pd.DataFrame( data=[self._transaction_balance(x) for x in transactions], + dtype="Int64", ) self._drop_foreign_assets(balance) if not self.args.unmute: self._drop_muted_policies(balance) if self.args.detail_level == 1: - balance.drop(labels=False, axis=1, level=4, inplace=True) + balance.drop(labels=False, axis=1, level=5, inplace=True) balance.columns = pd.MultiIndex.from_tuples(balance.columns) balance.sort_index(axis=1, level=0, sort_remaining=True, inplace=True) - balance = balance.T.groupby(level=(0, 1, 3)).sum().T + balance = balance.T.groupby(level=(0, 1, 2, 4)).sum(numeric_only=True).T + + # balance = pd.concat([balance[c] * np.exp()]) frame = pd.concat([timestamp, tx_hash, message], axis=1) frame.reset_index(drop=True, inplace=True) frame.columns = pd.MultiIndex.from_tuples( From b974c3ad2e6cf1d6827cb8294c63e4ce040c94c4 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Sat, 16 Sep 2023 20:56:31 +0200 Subject: [PATCH 033/124] scale decimals --- .../cardano_account_pandas_dumper.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 16bc178..997bb2f 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -461,7 +461,10 @@ def make_transaction_frame(self) -> pd.DataFrame: balance.sort_index(axis=1, level=0, sort_remaining=True, inplace=True) balance = balance.T.groupby(level=(0, 1, 2, 4)).sum(numeric_only=True).T - # balance = pd.concat([balance[c] * np.exp()]) + balance = balance * [ + np.float_power(10, np.negative(c[2])) for c in balance.columns + ] + balance.columns = balance.columns.droplevel(2) frame = pd.concat([timestamp, tx_hash, message], axis=1) frame.reset_index(drop=True, inplace=True) frame.columns = pd.MultiIndex.from_tuples( From c501bafae49d428c0b35ea7fc984efb45b5dd8a1 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Sat, 16 Sep 2023 20:56:40 +0200 Subject: [PATCH 034/124] add coinbase addresses --- src/cardano_account_pandas_dumper/known.jsonc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/cardano_account_pandas_dumper/known.jsonc b/src/cardano_account_pandas_dumper/known.jsonc index ed48c84..e7ddebd 100644 --- a/src/cardano_account_pandas_dumper/known.jsonc +++ b/src/cardano_account_pandas_dumper/known.jsonc @@ -2,9 +2,17 @@ // Known addresses "addresses": { "addr1v82lk3a7uln834rk0pjtszfnvvfhtdyc37ja7lvhxqmxalcxqnwsn": "CoinBase", + "addr1v83gh2sjr92d7x73mczk94sh5gztutxp7wqhwy9j6jz32hqnm9n9p": "CoinBase", + "addr1v8v3auqmw0eszza3ww29ea2pwftuqrqqyu26zvzjq9dt2ncydzvs5": "CoinBase", + "addr1v929zljj5jc24zptkhqcr3873krkdk8de4xzut79r2r0x5ss7x2fc": "CoinBase", "addr1v95sf69jcfhnmknvffwmfvlvnccatqwfjcyh0nlfc6gh5scta2yzg": "CoinBase", + "addr1v9edgwtmte2chm4u4rva8dx3puwdwxq6nnj7pd0hvjkdvuq59z6aw": "CoinBase", + "addr1v9h2m9lyg4awrlm2wkqlm6tjqf8cup5snxgnw9eskvdzclcyxl2jf": "CoinBase", "addr1v9u7va2sktlnz8dp3qadpnxxlv4m03672jy6elmntlxxs7q9nnwl9": "CoinBase", + "addr1vxkpskjw3umzqd7vudhq23gm3mx6fda8jh0kp5qf7vql9gs4fw7ve": "CoinBase", "addr1vydqq7fww4sga05jvtfckw3z9z8n8n2zxj0fyelvdl86acqhd9l70": "CoinBase", + "addr1vypr00ss7hkqejmvh53xkyf0p9q0a4z2uprxmx6njc463vgst3pe4": "CoinBase", + "addr1vyrgfvphe5xgefvj0efsucpcc574pvuxwzkxpve3qhkp2xq3j7tjn": "CoinBase", "addr1w80ptp0qgmcklhmeweesqgeurtlma8fsxsr9dt8au30fzss0czhl9": "Indigo Protocol (Stability pool)", "addr1w849xkxel6pvc7kcmc88dd82hkz32fjq3ega42wchd93xlgqxxpv5": "Indigo Protocol (CDP creator)", "addr1w89s3lfv7gkugker5llecq6x3k2vjvfnvp4692laeqe6w6s93vj3j": "Explosif (Direct sale)", From b45e75021e415de4337faf6545e94418ec602555 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Sat, 16 Sep 2023 21:02:21 +0200 Subject: [PATCH 035/124] split rewards/withdrawal --- .../cardano_account_pandas_dumper.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 997bb2f..11b5e60 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -350,17 +350,21 @@ def _transaction_balance(self, transaction: blockfrost.utils.Namespace) -> Any: result[ self._asset_tuple(self.data.LOVELACE_ASSET) + ("", " deposit", True) ] = np.longlong(transaction.deposit) - result[self._asset_tuple(self.data.LOVELACE_ASSET) + ("", " rewards", True)] = ( - np.negative( + if transaction.reward_amount: + result[ + self._asset_tuple(self.data.LOVELACE_ASSET) + ("", " rewards", True) + ] = np.longlong(transaction.reward_amount) + if transaction.withdrawals: + result[ + self._asset_tuple(self.data.LOVELACE_ASSET) + + ("", " rewards withdrawal", True) + ] = np.negative( functools.reduce( np.add, [np.longlong(w.amount) for w in transaction.withdrawals], np.longlong(0), ) ) - if not transaction.reward_amount - else np.longlong(transaction.reward_amount) - ) for utxo in transaction.utxos.nonref_inputs: if not utxo.collateral or not transaction.valid_contract: for amount in utxo.amount: From 279700a23183ec776ab86a0c052ff813e3bb45dd Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Sat, 16 Sep 2023 21:23:36 +0200 Subject: [PATCH 036/124] fix other/own, extract balance to method --- .../cardano_account_pandas_dumper.py | 64 ++++++++++++------- 1 file changed, 41 insertions(+), 23 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 11b5e60..a4188bb 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -206,6 +206,8 @@ class AccountPandasDumper: """Hold logic to convert an instance of AccountData to a Pandas dataframe.""" TRANSACTION_OFFSET = np.timedelta64(1000, "ns") + OWN_LABEL = "own" + OTHER_LABEL = "other" def __init__(self, data: AccountData, known_dict: Any, args: argparse.Namespace): self.data = data @@ -345,19 +347,21 @@ def _transaction_balance(self, transaction: blockfrost.utils.Namespace) -> Any: # Index: (policy,asset,decimals, address,address_name,own) result: MutableMapping[Tuple, np.longlong] = defaultdict(lambda: np.longlong(0)) result[ - self._asset_tuple(self.data.LOVELACE_ASSET) + ("", " fees", True) + self._asset_tuple(self.data.LOVELACE_ASSET) + ("", " fees", self.OWN_LABEL) ] = np.longlong(transaction.fees) result[ - self._asset_tuple(self.data.LOVELACE_ASSET) + ("", " deposit", True) + self._asset_tuple(self.data.LOVELACE_ASSET) + + ("", " deposit", self.OWN_LABEL) ] = np.longlong(transaction.deposit) if transaction.reward_amount: result[ - self._asset_tuple(self.data.LOVELACE_ASSET) + ("", " rewards", True) + self._asset_tuple(self.data.LOVELACE_ASSET) + + ("", " rewards", self.OWN_LABEL) ] = np.longlong(transaction.reward_amount) if transaction.withdrawals: result[ self._asset_tuple(self.data.LOVELACE_ASSET) - + ("", " rewards withdrawal", True) + + ("", " rewards withdrawal", self.OWN_LABEL) ] = np.negative( functools.reduce( np.add, @@ -376,9 +380,11 @@ def _transaction_balance(self, transaction: blockfrost.utils.Namespace) -> Any: utxo.address, self._truncate(utxo.address) if self.args.raw_values - else "other", + else self.OTHER_LABEL, ), - utxo.address in self.data.own_addresses, + self.OWN_LABEL + if utxo.address in self.data.own_addresses + else self.OTHER_LABEL, ) ] -= np.longlong(amount.quantity) @@ -392,9 +398,11 @@ def _transaction_balance(self, transaction: blockfrost.utils.Namespace) -> Any: utxo.address, self._truncate(utxo.address) if self.args.raw_values - else "other", + else self.OTHER_LABEL, ), - utxo.address in self.data.own_addresses, + self.OWN_LABEL + if utxo.address in self.data.own_addresses + else self.OTHER_LABEL, ) ] += np.longlong(amount.quantity) @@ -412,11 +420,11 @@ def _drop_foreign_assets(self, balance: pd.DataFrame) -> None: assets_to_drop = frozenset( # Assets that touch other addresses x[:2] - for x in balance.xs(False, level=-1, axis=1).columns + for x in balance.xs(self.OTHER_LABEL, level=-1, axis=1).columns ) - frozenset( # Assets that touch own addresses x[:2] - for x in balance.xs(True, level=-1, axis=1).columns + for x in balance.xs(self.OWN_LABEL, level=-1, axis=1).columns ) balance.sort_index(inplace=True, axis=1) balance.drop(assets_to_drop, axis=1, inplace=True) @@ -452,6 +460,20 @@ def make_transaction_frame(self) -> pd.DataFrame: timestamp = transactions.rename("timestamp").map(self._extract_timestamp) tx_hash = transactions.rename("hash").map(lambda x: x.hash) message = transactions.rename("message").map(self._format_message) + balance = self.make_balance_frame(transactions) + frame = pd.concat([timestamp, tx_hash, message], axis=1) + frame.reset_index(drop=True, inplace=True) + frame.columns = pd.MultiIndex.from_tuples( + [ + ("metadata", c) + (len(balance.columns[0]) - 2) * ("",) + for c in frame.columns + ] + ) + frame = frame.join(balance) + frame.sort_values(by=frame.columns[0], inplace=True) + return frame + + def make_balance_frame(self, transactions): balance = pd.DataFrame( data=[self._transaction_balance(x) for x in transactions], dtype="Int64", @@ -460,23 +482,19 @@ def make_transaction_frame(self) -> pd.DataFrame: if not self.args.unmute: self._drop_muted_policies(balance) if self.args.detail_level == 1: - balance.drop(labels=False, axis=1, level=5, inplace=True) + balance.drop(labels=self.OTHER_LABEL, axis=1, level=5, inplace=True) balance.columns = pd.MultiIndex.from_tuples(balance.columns) balance.sort_index(axis=1, level=0, sort_remaining=True, inplace=True) - balance = balance.T.groupby(level=(0, 1, 2, 4)).sum(numeric_only=True).T + balance = ( + balance.T.groupby( + level=(0, 1, 2, 4) if not self.args.raw_values else (0, 1, 2, 4, 5) + ) + .sum(numeric_only=True) + .T + ) balance = balance * [ np.float_power(10, np.negative(c[2])) for c in balance.columns ] balance.columns = balance.columns.droplevel(2) - frame = pd.concat([timestamp, tx_hash, message], axis=1) - frame.reset_index(drop=True, inplace=True) - frame.columns = pd.MultiIndex.from_tuples( - [ - ("metadata", c) + (len(balance.columns[0]) - 2) * ("",) - for c in frame.columns - ] - ) - frame = frame.join(balance) - frame.sort_values(by=frame.columns[0], inplace=True) - return frame + return balance From 810c496b806ba42fdd5162e0c8ce16884be078bc Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Sat, 16 Sep 2023 21:42:07 +0200 Subject: [PATCH 037/124] move transaction creation out to main, add flags for cols --- src/cardano_account_pandas_dumper/__main__.py | 9 ++++++- .../cardano_account_pandas_dumper.py | 24 ++++++++++--------- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/src/cardano_account_pandas_dumper/__main__.py b/src/cardano_account_pandas_dumper/__main__.py index 2be9c96..183e9f4 100644 --- a/src/cardano_account_pandas_dumper/__main__.py +++ b/src/cardano_account_pandas_dumper/__main__.py @@ -6,6 +6,7 @@ from json import JSONDecodeError import jstyleson +import pandas as pd from blockfrost import ApiError, BlockFrostApi from .cardano_account_pandas_dumper import AccountData, AccountPandasDumper @@ -159,7 +160,13 @@ def main(): reporter = AccountPandasDumper( data=data_from_api, known_dict=known_dict_from_file, args=args ) - dataframe = reporter.make_transaction_frame() + transactions = pd.concat( + objs=[ + data_from_api.transactions, + pd.Series() if args.no_rewards else data_from_api.reward_transactions, + ], + ).rename("transactions") + dataframe = reporter.make_transaction_frame(transactions) if args.pandas_output: try: dataframe.to_pickle(args.pandas_output) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index a4188bb..9abc33e 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -436,7 +436,12 @@ def _drop_muted_policies(self, balance: pd.DataFrame) -> None: balance.sort_index(inplace=True, axis=1) balance.drop(policies_to_drop, axis=1, inplace=True) - def make_transaction_frame(self) -> pd.DataFrame: + def make_transaction_frame( + self, + transactions: pd.Series, + with_tx_hash: bool = True, + with_tx_message: bool = True, + ) -> pd.DataFrame: """Build a transaction spreadsheet.""" # Add total line at the bottom @@ -451,17 +456,13 @@ def make_transaction_frame(self) -> pd.DataFrame: # ) # ) # outputs.loc["Total"] = total - transactions = pd.concat( - objs=[ - self.data.transactions, - pd.Series() if self.args.no_rewards else self.data.reward_transactions, - ], - ).rename("transactions") - timestamp = transactions.rename("timestamp").map(self._extract_timestamp) - tx_hash = transactions.rename("hash").map(lambda x: x.hash) - message = transactions.rename("message").map(self._format_message) + columns = [transactions.rename("timestamp").map(self._extract_timestamp)] + if with_tx_hash: + columns.append(transactions.rename("hash").map(lambda x: x.hash)) + if with_tx_message: + columns.append(transactions.rename("message").map(self._format_message)) balance = self.make_balance_frame(transactions) - frame = pd.concat([timestamp, tx_hash, message], axis=1) + frame = pd.concat(columns, axis=1) frame.reset_index(drop=True, inplace=True) frame.columns = pd.MultiIndex.from_tuples( [ @@ -474,6 +475,7 @@ def make_transaction_frame(self) -> pd.DataFrame: return frame def make_balance_frame(self, transactions): + """Make DataFrame with transaction balances.""" balance = pd.DataFrame( data=[self._transaction_balance(x) for x in transactions], dtype="Int64", From d0794cdb8c4b0fa9b1e75f5286a6cf6fa5e0bcc5 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Sat, 16 Sep 2023 21:48:54 +0200 Subject: [PATCH 038/124] update README --- README.md | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index f17af0e..42dee9f 100644 --- a/README.md +++ b/README.md @@ -100,16 +100,11 @@ This flag lets you specify another truncation length. : Do not truncate numerical identifiers. If you need numerical hex values to not be truncated at all (see `--truncate_length`above), specify this flag. -`--raw_asset` -: Add header row with concatenation of policy_id and hex-encoded asset_name. -This is useful if you need to look up a specific asset. +`--raw_values` +: Do not translate policies, assets and addresses to their names, keep them as hex. -## Calculations and precision - -All calculations are done using Python decimals to preserve accuracy. -However when importing into a spreadsheet, the values are usually converted to floats and some rounding errors can occur. -This is a spreadsheet issue, there isn't much that can be done by this tool to avoid it. -If you want to preserve accuracy the best way is probably to write a serialized [Pandas](https://pandas.pydata.org/) dataframe and write some code to process it. +`--no_rewards` +: Do not add pseudo-transactions with rewards for each epoch. ## Possible improvements @@ -152,7 +147,6 @@ Here is a comparison table for both projects (please submit corrections if you t | Knows about assets other than ADA |✔️|❌| | Knows about DeFI contract addresses |✔️[^2]|❌| | Extracts useful information from tx metadata |✔️|❌| -| Decimal arithmetic for absolute precision |✔️|❌| | .xlsx output |❌[^3]|✔️| | [Pandas](https://pandas.pydata.org/) compatible |✔️|❌| | Ready to use after one-liner install command |✔️|❌| From c56ef329d855a29b6217efe2cd8cc3cc0cbca803 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Mon, 18 Sep 2023 09:13:38 +0200 Subject: [PATCH 039/124] add total line, handle none decimals --- .../cardano_account_pandas_dumper.py | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 9abc33e..40d3ff5 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -333,7 +333,7 @@ def _format_policy(self, policy: str) -> Optional[str]: return self.policy_names.get(policy, self._truncate(policy)) def _decimals_for_asset(self, asset: str) -> np.longlong: - return np.longlong(self.data.assets[(asset,)].iloc(0)[0].metadata.decimals) + return np.longlong(self.data.assets[(asset,)].iloc(0)[0].metadata.decimals or 0) def _asset_tuple(self, asset_id: str) -> Tuple: asset = self.data.assets[(asset_id,)].iloc[0] @@ -441,26 +441,18 @@ def make_transaction_frame( transactions: pd.Series, with_tx_hash: bool = True, with_tx_message: bool = True, + with_total: bool = True, ) -> pd.DataFrame: """Build a transaction spreadsheet.""" - # Add total line at the bottom - # total = [] - # for column in outputs.columns: - # # Only NaN is float in the column - # total.append( - # functools.reduce( - # self.np.longlong_context.add, - # [a for a in outputs[column] if type(a) is type(np.longlong(0))], - # np.longlong(0), - # ) - # ) - # outputs.loc["Total"] = total + total = [""] columns = [transactions.rename("timestamp").map(self._extract_timestamp)] if with_tx_hash: columns.append(transactions.rename("hash").map(lambda x: x.hash)) + total.append("") if with_tx_message: columns.append(transactions.rename("message").map(self._format_message)) + total.append("Total") balance = self.make_balance_frame(transactions) frame = pd.concat(columns, axis=1) frame.reset_index(drop=True, inplace=True) @@ -472,6 +464,12 @@ def make_transaction_frame( ) frame = frame.join(balance) frame.sort_values(by=frame.columns[0], inplace=True) + # Add total line at the bottom + if with_total: + for column in balance.columns: + # Only NaN is float in the column + total.append(balance[column].sum()) + frame.loc["Total"] = total return frame def make_balance_frame(self, transactions): From afd196c54b2ed481150da6b5d6b9b178180861d6 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Mon, 18 Sep 2023 09:22:52 +0200 Subject: [PATCH 040/124] add policies --- src/cardano_account_pandas_dumper/known.jsonc | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/cardano_account_pandas_dumper/known.jsonc b/src/cardano_account_pandas_dumper/known.jsonc index e7ddebd..7cfba78 100644 --- a/src/cardano_account_pandas_dumper/known.jsonc +++ b/src/cardano_account_pandas_dumper/known.jsonc @@ -84,6 +84,7 @@ "policies": { "0029cb7c88c7567b63d1a512c0ed626aa169688ec980730c0473b913": "Sundaeswap LP", "026a18d04a0c642759bb3d83b12e3344894e5c1c7b2aeb1a2113a570": "Wingriders Factory/Liquidity token v1", + "078eafce5cd7edafdf63900edef2c1ea759e77f30ca81d6bbdeec924": "YUMMI Policy", "0be55d262b29f564998ff81efe21bdc0022621c12f15af08d0f2ddb1": "Minswap LP NFT 2", "13aa2accf2e1561723aa26871e071fdf32c867cff7e7d50ad470d62f": "Minswap LP NFT 2", "21ca2902ad0406bc2ce361ad91157b3542544d86d4e3d3e3c43f1763": "VyFI NFT 1", @@ -111,7 +112,17 @@ "e4214b7cce62ac6fbba385d164df48e157eae5863521b4b67ca71d86": "Minswap NFT", "e8a447d4e19016ca2aa74d20b4c4de87adb1f21dfb5493bf2d7281a6": "Sundae scooper", "f66d78b4a3cb3d37afa0ec36461e51ecbde00f26c8f0a68f94b69880": "Indigo", - "fd0d72fafee1d230a74c31ac503a192abd5b71888ae3f94128c1e634": "INDY staking NFT" + "fd0d72fafee1d230a74c31ac503a192abd5b71888ae3f94128c1e634": "INDY staking NFT", + "1d7f33bd23d85e1a25d87d86fac4f199c3197a2f7afeb662a0f34e1e": "WMT Policy", + "25f0fc240e91bd95dcdaebd2ba7713fc5168ac77234a3d79449fc20c": "SOCIETY Policy", + "533bb94a8850ee3ccbe483106489399112b74c905342cb1792a797a0": "INDY Policy", + "6ac8ef33b510ec004fe11585f7c5a9f0c07f0c23428ab4f29c1d7d10": "MELD Policy", + "8a1cfae21368b8bebbbed9800fec304e95cce39a2a57dc35e2e3ebaa": "MILK Policy", + "8f9c32977d2bacb87836b64f7811e99734c6368373958da20172afba": "MYIELD Policy", + "8fef2d34078659493ce161a6c7fba4b56afefa8535296a5743f69587": "Lenfi DAO Policy", + "a0028f350aaabe0545fdcb56b039bfb08e4bb4d8c4d7c3c7d481c235": "HOSKY Policy", + "c0ee29a85b13209423b10447d3c2e6a50641a15c57770e27cb9d5073": "WRT Policy", + "f43a62fdc3965df486de8a0d32fe800963589c41b38946602a0dc535": "AGIX Policy" }, // Known script hashes "scripts": { From 652447e3ba3a0e1b8725d9d52cd6ea47c5992ada Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Mon, 18 Sep 2023 11:07:50 +0200 Subject: [PATCH 041/124] fix mute --- .../cardano_account_pandas_dumper.py | 82 +++++++++---------- 1 file changed, 39 insertions(+), 43 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 40d3ff5..0dc4565 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -335,8 +335,10 @@ def _format_policy(self, policy: str) -> Optional[str]: def _decimals_for_asset(self, asset: str) -> np.longlong: return np.longlong(self.data.assets[(asset,)].iloc(0)[0].metadata.decimals or 0) - def _asset_tuple(self, asset_id: str) -> Tuple: + def _asset_tuple(self, asset_id: str) -> Optional[Tuple]: asset = self.data.assets[(asset_id,)].iloc[0] + if not self.args.unmute and any(self.muted_policies == asset.policy_id): + return None return ( self._format_policy(asset.policy_id), self.asset_names.get(asset_id), @@ -346,22 +348,21 @@ def _asset_tuple(self, asset_id: str) -> Tuple: def _transaction_balance(self, transaction: blockfrost.utils.Namespace) -> Any: # Index: (policy,asset,decimals, address,address_name,own) result: MutableMapping[Tuple, np.longlong] = defaultdict(lambda: np.longlong(0)) - result[ - self._asset_tuple(self.data.LOVELACE_ASSET) + ("", " fees", self.OWN_LABEL) - ] = np.longlong(transaction.fees) - result[ - self._asset_tuple(self.data.LOVELACE_ASSET) - + ("", " deposit", self.OWN_LABEL) - ] = np.longlong(transaction.deposit) + ada_asset = self._asset_tuple(self.data.LOVELACE_ASSET) + assert ada_asset is not None, "Asset for ADA unexpectedly None" + result[ada_asset + ("", " fees", self.OWN_LABEL)] = np.longlong( + transaction.fees + ) + result[ada_asset + ("", " deposit", self.OWN_LABEL)] = np.longlong( + transaction.deposit + ) if transaction.reward_amount: - result[ - self._asset_tuple(self.data.LOVELACE_ASSET) - + ("", " rewards", self.OWN_LABEL) - ] = np.longlong(transaction.reward_amount) + result[ada_asset + ("", " rewards", self.OWN_LABEL)] = np.longlong( + transaction.reward_amount + ) if transaction.withdrawals: result[ - self._asset_tuple(self.data.LOVELACE_ASSET) - + ("", " rewards withdrawal", self.OWN_LABEL) + ada_asset + ("", " rewards withdrawal", self.OWN_LABEL) ] = np.negative( functools.reduce( np.add, @@ -372,8 +373,30 @@ def _transaction_balance(self, transaction: blockfrost.utils.Namespace) -> Any: for utxo in transaction.utxos.nonref_inputs: if not utxo.collateral or not transaction.valid_contract: for amount in utxo.amount: + asset_tuple = self._asset_tuple(amount.unit) + if asset_tuple is not None: + result[ + asset_tuple + + ( + utxo.address, + self.address_names.get( + utxo.address, + self._truncate(utxo.address) + if self.args.raw_values + else self.OTHER_LABEL, + ), + self.OWN_LABEL + if utxo.address in self.data.own_addresses + else self.OTHER_LABEL, + ) + ] -= np.longlong(amount.quantity) + + for utxo in transaction.utxos.outputs: + for amount in utxo.amount: + asset_tuple = self._asset_tuple(amount.unit) + if asset_tuple is not None: result[ - self._asset_tuple(amount.unit) + asset_tuple + ( utxo.address, self.address_names.get( @@ -386,25 +409,7 @@ def _transaction_balance(self, transaction: blockfrost.utils.Namespace) -> Any: if utxo.address in self.data.own_addresses else self.OTHER_LABEL, ) - ] -= np.longlong(amount.quantity) - - for utxo in transaction.utxos.outputs: - for amount in utxo.amount: - result[ - self._asset_tuple(amount.unit) - + ( - utxo.address, - self.address_names.get( - utxo.address, - self._truncate(utxo.address) - if self.args.raw_values - else self.OTHER_LABEL, - ), - self.OWN_LABEL - if utxo.address in self.data.own_addresses - else self.OTHER_LABEL, - ) - ] += np.longlong(amount.quantity) + ] += np.longlong(amount.quantity) return result @@ -429,13 +434,6 @@ def _drop_foreign_assets(self, balance: pd.DataFrame) -> None: balance.sort_index(inplace=True, axis=1) balance.drop(assets_to_drop, axis=1, inplace=True) - def _drop_muted_policies(self, balance: pd.DataFrame) -> None: - policies_to_drop = frozenset( - [x[0] for x in balance.columns if x[0] in self.muted_policies] - ) - balance.sort_index(inplace=True, axis=1) - balance.drop(policies_to_drop, axis=1, inplace=True) - def make_transaction_frame( self, transactions: pd.Series, @@ -479,8 +477,6 @@ def make_balance_frame(self, transactions): dtype="Int64", ) self._drop_foreign_assets(balance) - if not self.args.unmute: - self._drop_muted_policies(balance) if self.args.detail_level == 1: balance.drop(labels=self.OTHER_LABEL, axis=1, level=5, inplace=True) balance.columns = pd.MultiIndex.from_tuples(balance.columns) From 8cc6a55fb82adf8c56982978983d59409dacc5dc Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Mon, 18 Sep 2023 11:37:31 +0200 Subject: [PATCH 042/124] cleanup, 0->NA --- .../cardano_account_pandas_dumper.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 0dc4565..f6383bd 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -324,7 +324,12 @@ def _format_message(self, tx_obj: blockfrost.utils.Namespace) -> str: ] ): result = ["(internal)"] - return " ".join(result).removeprefix("Message : ") + return ( + " ".join(result) + .removeprefix("Message : ") + .removeprefix("{'msg': ['") + .removesuffix("']}") + ) def _format_script(self, script: str) -> str: return self.scripts.get(script, self._truncate(script)) @@ -465,7 +470,6 @@ def make_transaction_frame( # Add total line at the bottom if with_total: for column in balance.columns: - # Only NaN is float in the column total.append(balance[column].sum()) frame.loc["Total"] = total return frame @@ -492,5 +496,6 @@ def make_balance_frame(self, transactions): balance = balance * [ np.float_power(10, np.negative(c[2])) for c in balance.columns ] + balance.replace(np.float64(0), pd.NA, inplace=True) balance.columns = balance.columns.droplevel(2) return balance From d5cc1b5e2cfea4ba092170ed12514da06db45237 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Mon, 18 Sep 2023 11:55:29 +0200 Subject: [PATCH 043/124] add suffixes --- src/cardano_account_pandas_dumper/known.jsonc | 134 +++++++++--------- 1 file changed, 67 insertions(+), 67 deletions(-) diff --git a/src/cardano_account_pandas_dumper/known.jsonc b/src/cardano_account_pandas_dumper/known.jsonc index 7cfba78..2d585cb 100644 --- a/src/cardano_account_pandas_dumper/known.jsonc +++ b/src/cardano_account_pandas_dumper/known.jsonc @@ -82,76 +82,76 @@ ], // Known policies "policies": { - "0029cb7c88c7567b63d1a512c0ed626aa169688ec980730c0473b913": "Sundaeswap LP", - "026a18d04a0c642759bb3d83b12e3344894e5c1c7b2aeb1a2113a570": "Wingriders Factory/Liquidity token v1", - "078eafce5cd7edafdf63900edef2c1ea759e77f30ca81d6bbdeec924": "YUMMI Policy", - "0be55d262b29f564998ff81efe21bdc0022621c12f15af08d0f2ddb1": "Minswap LP NFT 2", - "13aa2accf2e1561723aa26871e071fdf32c867cff7e7d50ad470d62f": "Minswap LP NFT 2", - "21ca2902ad0406bc2ce361ad91157b3542544d86d4e3d3e3c43f1763": "VyFI NFT 1", - "24b458412c2a7f9acb9c53c7ec4325b36806912ed56d2f91bfcf4d26": "INDY staking NFT", - "2960fe7a50e78948eb7219079c39b8169e34a4464a3f1aea9328d87c": "VyFI NFT 2", - "29d222ce763455e3d7a09a665ce554f00ac89d2e99a1a83d267170c6": "Minswap policy", - "2f2e0404310c106e2a260e8eb5a7e43f00cff42c667489d30e179816": "Minswap NFT ?", - "2fccae8bc1c8553a2185b2e77ccdea22f2e1d6e87beb80ef4eaf8cce": "INDY Gov NFT", - "408f13b240f57c3473b5727a68a88c50c1b6bb15c0c10912008dc59e": "INDY iBTC mint NFT", - "443c51db609bba8b2aa4c8af248bf797cbfcfa1e413c443296a50813": "INDY SP Account", - "4c820aadf6ae8c755430455f5803d283bde0b20114bd93a8f381c72c": "MELD", - "5817c34e5702473304f3cf676299176d3824e55b8c0bfa94830429fd": "MuesliSwap NFT", - "636c06ed152e70e9a1240da585a2841c15e4c3174044ca9a62cadab2": "NFT test", - "708f5e6d597fc038d09a738d7be32edd6ea779d6feb32a53668d9050": "INDY CDP NFT", - "72c8ee38fc1adfc5db49f7384bd64d41e9b9ce46ec11ba9a292e5d02": "NFT test", - "869fc72c11977e4be3e8e1cc63cca008a925886332795c9601f965ca": "USDC policy", - "8a7b0a43a169218fb66996c3945f53d7712eae9e31db193adb55663f": "NFT test", - "909133088303c49f3a30f1cc8ed553a73857a29779f6c6561cd8093f": "MuesliSwap NFT Policy v2", - "97da12de04a6b527cc3b3469c5e5485cf258dfd1021f12e728f2e714": "INDY IASSET NFT", - "9a9693a9a37912a5097918f97918d15240c92ab729a0b7c4aa144d77": "Sundae policy", - "af3d70acf4bd5b3abb319a7d75c89fb3e56eafcdd46b2e9b57a2557f": "MuesliSwap NFT", - "d195ca7db29f0f13a00cac7fca70426ff60bad4e1e87d3757fae8484": "Minswap LP NFT 1", - "de9b756719341e79785aa13c164e7fe68c189ed04d61c9876b2fe53f": "MuesliSwap Factory Policy v2", - "e3455f2715338b454fb853442f72dc03b98396854f97510027fe22ff": "INDY iUSD NFT", - "e4214b7cce62ac6fbba385d164df48e157eae5863521b4b67ca71d86": "Minswap NFT", - "e8a447d4e19016ca2aa74d20b4c4de87adb1f21dfb5493bf2d7281a6": "Sundae scooper", - "f66d78b4a3cb3d37afa0ec36461e51ecbde00f26c8f0a68f94b69880": "Indigo", - "fd0d72fafee1d230a74c31ac503a192abd5b71888ae3f94128c1e634": "INDY staking NFT", - "1d7f33bd23d85e1a25d87d86fac4f199c3197a2f7afeb662a0f34e1e": "WMT Policy", - "25f0fc240e91bd95dcdaebd2ba7713fc5168ac77234a3d79449fc20c": "SOCIETY Policy", - "533bb94a8850ee3ccbe483106489399112b74c905342cb1792a797a0": "INDY Policy", - "6ac8ef33b510ec004fe11585f7c5a9f0c07f0c23428ab4f29c1d7d10": "MELD Policy", - "8a1cfae21368b8bebbbed9800fec304e95cce39a2a57dc35e2e3ebaa": "MILK Policy", - "8f9c32977d2bacb87836b64f7811e99734c6368373958da20172afba": "MYIELD Policy", - "8fef2d34078659493ce161a6c7fba4b56afefa8535296a5743f69587": "Lenfi DAO Policy", - "a0028f350aaabe0545fdcb56b039bfb08e4bb4d8c4d7c3c7d481c235": "HOSKY Policy", - "c0ee29a85b13209423b10447d3c2e6a50641a15c57770e27cb9d5073": "WRT Policy", + "0029cb7c88c7567b63d1a512c0ed626aa169688ec980730c0473b913": "Sundaeswap LP-b913", + "026a18d04a0c642759bb3d83b12e3344894e5c1c7b2aeb1a2113a570": "Wingriders Factory/Liquidity token v1-a570", + "078eafce5cd7edafdf63900edef2c1ea759e77f30ca81d6bbdeec924": "YUMMI Policy-c924", + "0be55d262b29f564998ff81efe21bdc0022621c12f15af08d0f2ddb1": "Minswap LP NFT 2-ddb1", + "13aa2accf2e1561723aa26871e071fdf32c867cff7e7d50ad470d62f": "Minswap LP NFT 2-d62f", + "21ca2902ad0406bc2ce361ad91157b3542544d86d4e3d3e3c43f1763": "VyFI NFT 1-1763", + "24b458412c2a7f9acb9c53c7ec4325b36806912ed56d2f91bfcf4d26": "INDY staking NFT-4d26", + "2960fe7a50e78948eb7219079c39b8169e34a4464a3f1aea9328d87c": "VyFI NFT 2-d87c", + "29d222ce763455e3d7a09a665ce554f00ac89d2e99a1a83d267170c6": "Minswap policy-70c6", + "2f2e0404310c106e2a260e8eb5a7e43f00cff42c667489d30e179816": "Minswap NFT ?-9816", + "2fccae8bc1c8553a2185b2e77ccdea22f2e1d6e87beb80ef4eaf8cce": "INDY Gov NFT-8cce", + "408f13b240f57c3473b5727a68a88c50c1b6bb15c0c10912008dc59e": "INDY iBTC mint NFT-c59e", + "443c51db609bba8b2aa4c8af248bf797cbfcfa1e413c443296a50813": "INDY SP Account-0813", + "4c820aadf6ae8c755430455f5803d283bde0b20114bd93a8f381c72c": "MELD-c72c", + "5817c34e5702473304f3cf676299176d3824e55b8c0bfa94830429fd": "MuesliSwap NFT-29fd", + "636c06ed152e70e9a1240da585a2841c15e4c3174044ca9a62cadab2": "NFT test-dab2", + "708f5e6d597fc038d09a738d7be32edd6ea779d6feb32a53668d9050": "INDY CDP NFT-9050", + "72c8ee38fc1adfc5db49f7384bd64d41e9b9ce46ec11ba9a292e5d02": "NFT test-5d02", + "869fc72c11977e4be3e8e1cc63cca008a925886332795c9601f965ca": "USDC policy-65ca", + "8a7b0a43a169218fb66996c3945f53d7712eae9e31db193adb55663f": "NFT test-663f", + "909133088303c49f3a30f1cc8ed553a73857a29779f6c6561cd8093f": "MuesliSwap NFT Policy v2-093f", + "97da12de04a6b527cc3b3469c5e5485cf258dfd1021f12e728f2e714": "INDY IASSET NFT-e714", + "9a9693a9a37912a5097918f97918d15240c92ab729a0b7c4aa144d77": "Sundae policy-4d77", + "af3d70acf4bd5b3abb319a7d75c89fb3e56eafcdd46b2e9b57a2557f": "MuesliSwap NFT-557f", + "d195ca7db29f0f13a00cac7fca70426ff60bad4e1e87d3757fae8484": "Minswap LP NFT 1-8484", + "de9b756719341e79785aa13c164e7fe68c189ed04d61c9876b2fe53f": "MuesliSwap Factory Policy v2-e53f", + "e3455f2715338b454fb853442f72dc03b98396854f97510027fe22ff": "INDY iUSD NFT-22ff", + "e4214b7cce62ac6fbba385d164df48e157eae5863521b4b67ca71d86": "Minswap NFT-1d86", + "e8a447d4e19016ca2aa74d20b4c4de87adb1f21dfb5493bf2d7281a6": "Sundae scooper-81a6", + "f66d78b4a3cb3d37afa0ec36461e51ecbde00f26c8f0a68f94b69880": "Indigo-9880", + "fd0d72fafee1d230a74c31ac503a192abd5b71888ae3f94128c1e634": "INDY staking NFT-e634", + "1d7f33bd23d85e1a25d87d86fac4f199c3197a2f7afeb662a0f34e1e": "WMT Policy-4e1e", + "25f0fc240e91bd95dcdaebd2ba7713fc5168ac77234a3d79449fc20c": "SOCIETY Policy-c20c", + "533bb94a8850ee3ccbe483106489399112b74c905342cb1792a797a0": "INDY Policy-97a0", + "6ac8ef33b510ec004fe11585f7c5a9f0c07f0c23428ab4f29c1d7d10": "MELD Policy-7d10", + "8a1cfae21368b8bebbbed9800fec304e95cce39a2a57dc35e2e3ebaa": "MILK Policy-ebaa", + "8f9c32977d2bacb87836b64f7811e99734c6368373958da20172afba": "MYIELD Policy-afba", + "8fef2d34078659493ce161a6c7fba4b56afefa8535296a5743f69587": "Lenfi DAO Policy-9587", + "a0028f350aaabe0545fdcb56b039bfb08e4bb4d8c4d7c3c7d481c235": "HOSKY Policy-c235", + "c0ee29a85b13209423b10447d3c2e6a50641a15c57770e27cb9d5073": "WRT Policy-5073", "f43a62fdc3965df486de8a0d32fe800963589c41b38946602a0dc535": "AGIX Policy" }, // Known script hashes "scripts": { - "00fb107bfbd51b3a5638867d3688e986ba38ff34fb738f5bd42b20d5": "MuesliSwap (Partial Match Order)", - "0237cc313756ebb5bcfc2728f7bdc6a8047b471220a305aa373b278a": "Wingriders (Farm/Voting)", - "30e02ff8576298babd301cee58928d7b320364abeae556c46e3cf42d": "MELD (staking 2)", - "4020e7fc2de75a0729c3cc3af715b34d98381e0cdbcfa99c950bc3ac": "SundaeSwap (Liquidity Pool)", - "54e8d424816f5bbd423353009ff7c31ada2cbdefe651175014117f46": "Indigo Protocol (Indy staking)", - "7045237d1eb0199c84dffe58fe6df7dc5d255eb4d418e4146d5721f8": "MuesliSwap (Pool Contract) ", - "73ede893f547edbd25da6953fda33caacd01f44047922bf7c5ceb951": "MuesliSwap (Batch Order)", - "74db9c7a23605cacae4ee635a15af6e63c5a39915c481254eeb9758b": "SundaeSwap (Farming)", - "86ae9eebd8b97944a45201e4aec1330a72291af2d071644bba015959": "Wingriders (Request)", - "9068a7a3f008803edac87af1619860f2cdcde40c26987325ace138ad": "jpg.store (n/a) Script", - "98df3b00a1500fcb77daa0520550fb088fc923399788b89637b9de59": "Minswap (Harvest v1)", - "9b85d5e8611945505f078aeededcbed1d6ca11053f61e3f9d999fe44": "Minswap (Yield Farming v1)", - "9df55104326a3264d803d44ed87139581d1c912e26ba8e73bc385e2f": "Minswap (MINt staking liquidity)", - "9fd47e0d578fb539b9ec7e016dc5ee25844110a96712d55989b0751b": "Wingriders (Farm/Voting)", - "a473cb8eb0b61c03b8696fceab1c1a89fa3ec834572850e7c2abe783": "Indigo Protocol (Stability pool account)", - "a55f409501bf65805bb0dc76f6f9ae90b61e19ed870bc00256813608": "jpg.store (Offers)", - "a65ca58a4e9c755fa830173d2a5caed458ac0c73f97db7faae2e7e3b": "Minswap (Batch Order)", - "ba158766c1bae60e2117ee8987621441fac66a5e0fb9c7aca58cf20a": "SundaeSwap (Order Book / Escrow)", - "ba3501cd170c96349c342c5ef4242c596b58afaecdcffc0bb04af0ec": "MELD (staking 1)", - "c1af46643698bc73f9857a0cc21ec949ce6919974b435d057dc448b8": "Indigo Protocol (Poll Manager)", - "c2afd87ff836f64a20c33dc252e850cdd55f31627d64012d0960856f": "MuesliSwap (Farming)", - "cb08fd2cf22dc45b23a7ff9c03468d94c93133606ba2abfdc833a76a": "Explosif (Direct sale)", - "de1585e046f16fdf79767300233c1affbe9d30340656acfde45e9142": "Indigo Protocol (Stability pool)", - "e1317b152faac13426e6a83e06ff88a4d62cce3c1634ab0a5ec13309": "Minswap (Liquidity Pool)", - "e4d2fb0b8d275852103fd75801e2c7dcf6ed3e276c74cabadbe5b8b6": "Indigo Protocol (CDP)", - "e6c90a5923713af5786963dee0fdffd830ca7e0c86a041d9e5833e91": "Wingriders (Pool payment credential)", - "ea5358d9fe82cc7ad8de0e76b4eabd851526408e51daa9d8bb4b137d": "Indigo Protocol (CDP creator)" + "00fb107bfbd51b3a5638867d3688e986ba38ff34fb738f5bd42b20d5": "MuesliSwap (Partial Match Order)-20d5", + "0237cc313756ebb5bcfc2728f7bdc6a8047b471220a305aa373b278a": "Wingriders (Farm/Voting)-278a", + "30e02ff8576298babd301cee58928d7b320364abeae556c46e3cf42d": "MELD (staking 2)-f42d", + "4020e7fc2de75a0729c3cc3af715b34d98381e0cdbcfa99c950bc3ac": "SundaeSwap (Liquidity Pool)-c3ac", + "54e8d424816f5bbd423353009ff7c31ada2cbdefe651175014117f46": "Indigo Protocol (Indy staking)-7f46", + "7045237d1eb0199c84dffe58fe6df7dc5d255eb4d418e4146d5721f8": "MuesliSwap (Pool Contract)-21f8", + "73ede893f547edbd25da6953fda33caacd01f44047922bf7c5ceb951": "MuesliSwap (Batch Order)-b951", + "74db9c7a23605cacae4ee635a15af6e63c5a39915c481254eeb9758b": "SundaeSwap (Farming)-758b", + "86ae9eebd8b97944a45201e4aec1330a72291af2d071644bba015959": "Wingriders (Request)-5959", + "9068a7a3f008803edac87af1619860f2cdcde40c26987325ace138ad": "jpg.store (n/a) Script-38ad", + "98df3b00a1500fcb77daa0520550fb088fc923399788b89637b9de59": "Minswap (Harvest v1)-de59", + "9b85d5e8611945505f078aeededcbed1d6ca11053f61e3f9d999fe44": "Minswap (Yield Farming v1)-fe44", + "9df55104326a3264d803d44ed87139581d1c912e26ba8e73bc385e2f": "Minswap (MINt staking liquidity)-5e2f", + "9fd47e0d578fb539b9ec7e016dc5ee25844110a96712d55989b0751b": "Wingriders (Farm/Voting)-751b", + "a473cb8eb0b61c03b8696fceab1c1a89fa3ec834572850e7c2abe783": "Indigo Protocol (Stability pool account)-e783", + "a55f409501bf65805bb0dc76f6f9ae90b61e19ed870bc00256813608": "jpg.store (Offers)-3608", + "a65ca58a4e9c755fa830173d2a5caed458ac0c73f97db7faae2e7e3b": "Minswap (Batch Order)-7e3b", + "ba158766c1bae60e2117ee8987621441fac66a5e0fb9c7aca58cf20a": "SundaeSwap (Order Book / Escrow)-f20a", + "ba3501cd170c96349c342c5ef4242c596b58afaecdcffc0bb04af0ec": "MELD (staking 1)-f0ec", + "c1af46643698bc73f9857a0cc21ec949ce6919974b435d057dc448b8": "Indigo Protocol (Poll Manager)-48b8", + "c2afd87ff836f64a20c33dc252e850cdd55f31627d64012d0960856f": "MuesliSwap (Farming)-856f", + "cb08fd2cf22dc45b23a7ff9c03468d94c93133606ba2abfdc833a76a": "Explosif (Direct sale)-a76a", + "de1585e046f16fdf79767300233c1affbe9d30340656acfde45e9142": "Indigo Protocol (Stability pool)-9142", + "e1317b152faac13426e6a83e06ff88a4d62cce3c1634ab0a5ec13309": "Minswap (Liquidity Pool)-3309", + "e4d2fb0b8d275852103fd75801e2c7dcf6ed3e276c74cabadbe5b8b6": "Indigo Protocol (CDP)-b8b6", + "e6c90a5923713af5786963dee0fdffd830ca7e0c86a041d9e5833e91": "Wingriders (Pool payment credential)-3e91", + "ea5358d9fe82cc7ad8de0e76b4eabd851526408e51daa9d8bb4b137d": "Indigo Protocol (CDP creator)-137d" } } \ No newline at end of file From 4e726b7cf446b6e677a7f08d1d73f1d60484af1c Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Mon, 18 Sep 2023 11:56:02 +0200 Subject: [PATCH 044/124] cache _asset_tuple --- .../cardano_account_pandas_dumper.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index f6383bd..77abc77 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -340,6 +340,7 @@ def _format_policy(self, policy: str) -> Optional[str]: def _decimals_for_asset(self, asset: str) -> np.longlong: return np.longlong(self.data.assets[(asset,)].iloc(0)[0].metadata.decimals or 0) + @functools.lru_cache(maxsize=10000) def _asset_tuple(self, asset_id: str) -> Optional[Tuple]: asset = self.data.assets[(asset_id,)].iloc[0] if not self.args.unmute and any(self.muted_policies == asset.policy_id): From ed457caee1ed5b9d6d9ea69ff334a7d52f5c4e09 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Mon, 18 Sep 2023 12:06:45 +0200 Subject: [PATCH 045/124] remove most mutes --- src/cardano_account_pandas_dumper/known.jsonc | 25 ++----------------- 1 file changed, 2 insertions(+), 23 deletions(-) diff --git a/src/cardano_account_pandas_dumper/known.jsonc b/src/cardano_account_pandas_dumper/known.jsonc index 2d585cb..eafaf60 100644 --- a/src/cardano_account_pandas_dumper/known.jsonc +++ b/src/cardano_account_pandas_dumper/known.jsonc @@ -56,29 +56,8 @@ }, // Policies to mute "muted_policies": [ - "0029cb7c88c7567b63d1a512c0ed626aa169688ec980730c0473b913", - "026a18d04a0c642759bb3d83b12e3344894e5c1c7b2aeb1a2113a570", - "0be55d262b29f564998ff81efe21bdc0022621c12f15af08d0f2ddb1", - "13aa2accf2e1561723aa26871e071fdf32c867cff7e7d50ad470d62f", - "24b458412c2a7f9acb9c53c7ec4325b36806912ed56d2f91bfcf4d26", - "2f2e0404310c106e2a260e8eb5a7e43f00cff42c667489d30e179816", - "2fccae8bc1c8553a2185b2e77ccdea22f2e1d6e87beb80ef4eaf8cce", - "408f13b240f57c3473b5727a68a88c50c1b6bb15c0c10912008dc59e", - "5817c34e5702473304f3cf676299176d3824e55b8c0bfa94830429fd", "636c06ed152e70e9a1240da585a2841c15e4c3174044ca9a62cadab2", - "72c8ee38fc1adfc5db49f7384bd64d41e9b9ce46ec11ba9a292e5d02", - "8a7b0a43a169218fb66996c3945f53d7712eae9e31db193adb55663f", - "909133088303c49f3a30f1cc8ed553a73857a29779f6c6561cd8093f", - "97da12de04a6b527cc3b3469c5e5485cf258dfd1021f12e728f2e714", - "af3d70acf4bd5b3abb319a7d75c89fb3e56eafcdd46b2e9b57a2557f", - "d195ca7db29f0f13a00cac7fca70426ff60bad4e1e87d3757fae8484", - "de9b756719341e79785aa13c164e7fe68c189ed04d61c9876b2fe53f", - "e3455f2715338b454fb853442f72dc03b98396854f97510027fe22ff", - "e4214b7cce62ac6fbba385d164df48e157eae5863521b4b67ca71d86", - "e8a447d4e19016ca2aa74d20b4c4de87adb1f21dfb5493bf2d7281a6", - "f66d78b4a3cb3d37afa0ec36461e51ecbde00f26c8f0a68f94b69880", - "21ca2902ad0406bc2ce361ad91157b3542544d86d4e3d3e3c43f1763", - "2960fe7a50e78948eb7219079c39b8169e34a4464a3f1aea9328d87c" + "8a7b0a43a169218fb66996c3945f53d7712eae9e31db193adb55663f" ], // Known policies "policies": { @@ -122,7 +101,7 @@ "8fef2d34078659493ce161a6c7fba4b56afefa8535296a5743f69587": "Lenfi DAO Policy-9587", "a0028f350aaabe0545fdcb56b039bfb08e4bb4d8c4d7c3c7d481c235": "HOSKY Policy-c235", "c0ee29a85b13209423b10447d3c2e6a50641a15c57770e27cb9d5073": "WRT Policy-5073", - "f43a62fdc3965df486de8a0d32fe800963589c41b38946602a0dc535": "AGIX Policy" + "f43a62fdc3965df486de8a0d32fe800963589c41b38946602a0dc535": "AGIX Policy-c535" }, // Known script hashes "scripts": { From e74c3642c7003dfb0cfc028e187e482eb46a263b Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Mon, 18 Sep 2023 17:28:22 +0200 Subject: [PATCH 046/124] fix total type, move replace NA to main --- src/cardano_account_pandas_dumper/__main__.py | 3 ++- .../cardano_account_pandas_dumper.py | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/cardano_account_pandas_dumper/__main__.py b/src/cardano_account_pandas_dumper/__main__.py index 183e9f4..c1b610d 100644 --- a/src/cardano_account_pandas_dumper/__main__.py +++ b/src/cardano_account_pandas_dumper/__main__.py @@ -6,6 +6,7 @@ from json import JSONDecodeError import jstyleson +import numpy as np import pandas as pd from blockfrost import ApiError, BlockFrostApi @@ -174,7 +175,7 @@ def main(): warnings.warn(f"Failed to write pandas file: {exception}") if args.csv_output: try: - dataframe.to_csv(args.csv_output, index=False) + dataframe.replace(np.float64(0), pd.NA).to_csv(args.csv_output, index=False) except OSError as exception: warnings.warn(f"Failed to write CSV file: {exception}") print("Done.") diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 77abc77..452fb91 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -449,8 +449,8 @@ def make_transaction_frame( ) -> pd.DataFrame: """Build a transaction spreadsheet.""" - total = [""] columns = [transactions.rename("timestamp").map(self._extract_timestamp)] + total: List[Any] = [columns[0].max() + self.TRANSACTION_OFFSET] if with_tx_hash: columns.append(transactions.rename("hash").map(lambda x: x.hash)) total.append("") @@ -472,7 +472,9 @@ def make_transaction_frame( if with_total: for column in balance.columns: total.append(balance[column].sum()) - frame.loc["Total"] = total + frame = pd.concat( + [frame, pd.DataFrame(data=[total], columns=frame.columns)] + ) return frame def make_balance_frame(self, transactions): @@ -497,6 +499,5 @@ def make_balance_frame(self, transactions): balance = balance * [ np.float_power(10, np.negative(c[2])) for c in balance.columns ] - balance.replace(np.float64(0), pd.NA, inplace=True) balance.columns = balance.columns.droplevel(2) return balance From 7f43c70881342bb2eefddf471a8e99934fbb73ed Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Wed, 20 Sep 2023 12:04:37 +0200 Subject: [PATCH 047/124] update README --- README.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/README.md b/README.md index 42dee9f..018b36a 100644 --- a/README.md +++ b/README.md @@ -106,6 +106,29 @@ If you need numerical hex values to not be truncated at all (see `--truncate_len `--no_rewards` : Do not add pseudo-transactions with rewards for each epoch. +## Output format + +### CSV + +column 0: +transaction timestamp + +column 1; +transaction hash + +column 2: +transaction message + +columns 3-...: +transaction input (positive) or output (negative) for each asset and address. + +row 0: asset policy +row 1: asset name +row 2: address + +If the `--raw_values` flag is passed, row 3 is inserted, with a value of `own`for own addresses (belonging to the specified staking addresses) +and `other` for other addresses (if `--raw_values`is not passed, this information is on row 2). + ## Possible improvements * The first obvious possible improvement would be to replace the static `--known_file` that lists the known addresses, policies and scripts with a dynamic API. From 717f7e5d4fe80b0a947c8a4b0700674d6c703c9d Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Wed, 20 Sep 2023 16:59:09 +0200 Subject: [PATCH 048/124] dedupe index creation and sorting --- .../cardano_account_pandas_dumper.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 452fb91..4f5ba95 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -427,7 +427,6 @@ def _extract_timestamp(cls, transaction: blockfrost.utils.Namespace) -> Any: def _drop_foreign_assets(self, balance: pd.DataFrame) -> None: # Drop assets that only touch foreign addresses - balance.columns = pd.MultiIndex.from_tuples(balance.columns) assets_to_drop = frozenset( # Assets that touch other addresses x[:2] @@ -437,7 +436,6 @@ def _drop_foreign_assets(self, balance: pd.DataFrame) -> None: x[:2] for x in balance.xs(self.OWN_LABEL, level=-1, axis=1).columns ) - balance.sort_index(inplace=True, axis=1) balance.drop(assets_to_drop, axis=1, inplace=True) def make_transaction_frame( @@ -483,11 +481,11 @@ def make_balance_frame(self, transactions): data=[self._transaction_balance(x) for x in transactions], dtype="Int64", ) + balance.columns = pd.MultiIndex.from_tuples(balance.columns) + balance.sort_index(axis=1, level=0, sort_remaining=True, inplace=True) self._drop_foreign_assets(balance) if self.args.detail_level == 1: balance.drop(labels=self.OTHER_LABEL, axis=1, level=5, inplace=True) - balance.columns = pd.MultiIndex.from_tuples(balance.columns) - balance.sort_index(axis=1, level=0, sort_remaining=True, inplace=True) balance = ( balance.T.groupby( level=(0, 1, 2, 4) if not self.args.raw_values else (0, 1, 2, 4, 5) From d5ecff05568884e8216cd1bf2e7b1302c43fc892 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Wed, 20 Sep 2023 17:57:31 +0200 Subject: [PATCH 049/124] add pinned policies --- .../cardano_account_pandas_dumper.py | 25 ++++++++-- src/cardano_account_pandas_dumper/known.jsonc | 47 ++++++++++++++----- 2 files changed, 54 insertions(+), 18 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 4f5ba95..f9fb2fb 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -239,6 +239,8 @@ def __init__(self, data: AccountData, known_dict: Any, args: argparse.Namespace) } ) self.muted_policies = pd.Series(known_dict.get("muted_policies", [])) + self.pinned_policies = pd.Series(known_dict.get("pinned_policies", [])) + self.scripts = pd.Series(known_dict.get("scripts", {})) self.labels = pd.Series(known_dict.get("labels", {})) @@ -341,9 +343,13 @@ def _decimals_for_asset(self, asset: str) -> np.longlong: return np.longlong(self.data.assets[(asset,)].iloc(0)[0].metadata.decimals or 0) @functools.lru_cache(maxsize=10000) - def _asset_tuple(self, asset_id: str) -> Optional[Tuple]: + def _asset_tuple(self, asset_id: str, unmute: bool) -> Optional[Tuple]: asset = self.data.assets[(asset_id,)].iloc[0] - if not self.args.unmute and any(self.muted_policies == asset.policy_id): + if ( + not unmute + and any(self.muted_policies == asset.policy_id) + and not any(self.pinned_policies == asset.policy_id) + ): return None return ( self._format_policy(asset.policy_id), @@ -354,7 +360,7 @@ def _asset_tuple(self, asset_id: str) -> Optional[Tuple]: def _transaction_balance(self, transaction: blockfrost.utils.Namespace) -> Any: # Index: (policy,asset,decimals, address,address_name,own) result: MutableMapping[Tuple, np.longlong] = defaultdict(lambda: np.longlong(0)) - ada_asset = self._asset_tuple(self.data.LOVELACE_ASSET) + ada_asset = self._asset_tuple(self.data.LOVELACE_ASSET, True) assert ada_asset is not None, "Asset for ADA unexpectedly None" result[ada_asset + ("", " fees", self.OWN_LABEL)] = np.longlong( transaction.fees @@ -379,7 +385,7 @@ def _transaction_balance(self, transaction: blockfrost.utils.Namespace) -> Any: for utxo in transaction.utxos.nonref_inputs: if not utxo.collateral or not transaction.valid_contract: for amount in utxo.amount: - asset_tuple = self._asset_tuple(amount.unit) + asset_tuple = self._asset_tuple(amount.unit, self.args.unmute) if asset_tuple is not None: result[ asset_tuple @@ -399,7 +405,7 @@ def _transaction_balance(self, transaction: blockfrost.utils.Namespace) -> Any: for utxo in transaction.utxos.outputs: for amount in utxo.amount: - asset_tuple = self._asset_tuple(amount.unit) + asset_tuple = self._asset_tuple(amount.unit, self.args.unmute) if asset_tuple is not None: result[ asset_tuple @@ -435,6 +441,15 @@ def _drop_foreign_assets(self, balance: pd.DataFrame) -> None: # Assets that touch own addresses x[:2] for x in balance.xs(self.OWN_LABEL, level=-1, axis=1).columns + ).union( + # Assets with pinned policies + frozenset( + [ + self._asset_tuple(asset.asset_id, True)[:2] # type: ignore[index] + for asset in self.data.assets + if any(self.pinned_policies == asset.policy_id) + ] + ) ) balance.drop(assets_to_drop, axis=1, inplace=True) diff --git a/src/cardano_account_pandas_dumper/known.jsonc b/src/cardano_account_pandas_dumper/known.jsonc index eafaf60..dd161d9 100644 --- a/src/cardano_account_pandas_dumper/known.jsonc +++ b/src/cardano_account_pandas_dumper/known.jsonc @@ -54,10 +54,28 @@ "674": "Message", "914425": "Indigo SP Reward" }, - // Policies to mute + // Policies to mute from assets "muted_policies": [ "636c06ed152e70e9a1240da585a2841c15e4c3174044ca9a62cadab2", - "8a7b0a43a169218fb66996c3945f53d7712eae9e31db193adb55663f" + "8a7b0a43a169218fb66996c3945f53d7712eae9e31db193adb55663f", + "72c8ee38fc1adfc5db49f7384bd64d41e9b9ce46ec11ba9a292e5d02" + ], + // Policies to always show in assets (takes precedence over mute) + "pinned_policies": [ + "24b458412c2a7f9acb9c53c7ec4325b36806912ed56d2f91bfcf4d26", + "2fccae8bc1c8553a2185b2e77ccdea22f2e1d6e87beb80ef4eaf8cce", + "408f13b240f57c3473b5727a68a88c50c1b6bb15c0c10912008dc59e", + "443c51db609bba8b2aa4c8af248bf797cbfcfa1e413c443296a50813", + "533bb94a8850ee3ccbe483106489399112b74c905342cb1792a797a0", + "708f5e6d597fc038d09a738d7be32edd6ea779d6feb32a53668d9050", + "97da12de04a6b527cc3b3469c5e5485cf258dfd1021f12e728f2e714", + "e3455f2715338b454fb853442f72dc03b98396854f97510027fe22ff", + "fd0d72fafee1d230a74c31ac503a192abd5b71888ae3f94128c1e634", + "54e8d424816f5bbd423353009ff7c31ada2cbdefe651175014117f46", + "4c820aadf6ae8c755430455f5803d283bde0b20114bd93a8f381c72c", + "735b37149eb0c2a5fb590bd60e39fe90ae3a96b6065b05d7aca99ebb", + "f9b162ea9529e639a083595294006a833473883a75d6df1e4c22dd4f", + "3f28fb7d6c40468262dffb1c3adb568b342499826b664d940085d022" ], // Known policies "policies": { @@ -66,42 +84,45 @@ "078eafce5cd7edafdf63900edef2c1ea759e77f30ca81d6bbdeec924": "YUMMI Policy-c924", "0be55d262b29f564998ff81efe21bdc0022621c12f15af08d0f2ddb1": "Minswap LP NFT 2-ddb1", "13aa2accf2e1561723aa26871e071fdf32c867cff7e7d50ad470d62f": "Minswap LP NFT 2-d62f", + "1d7f33bd23d85e1a25d87d86fac4f199c3197a2f7afeb662a0f34e1e": "WMT Policy-4e1e", "21ca2902ad0406bc2ce361ad91157b3542544d86d4e3d3e3c43f1763": "VyFI NFT 1-1763", "24b458412c2a7f9acb9c53c7ec4325b36806912ed56d2f91bfcf4d26": "INDY staking NFT-4d26", + "25f0fc240e91bd95dcdaebd2ba7713fc5168ac77234a3d79449fc20c": "SOCIETY Policy-c20c", "2960fe7a50e78948eb7219079c39b8169e34a4464a3f1aea9328d87c": "VyFI NFT 2-d87c", "29d222ce763455e3d7a09a665ce554f00ac89d2e99a1a83d267170c6": "Minswap policy-70c6", "2f2e0404310c106e2a260e8eb5a7e43f00cff42c667489d30e179816": "Minswap NFT ?-9816", "2fccae8bc1c8553a2185b2e77ccdea22f2e1d6e87beb80ef4eaf8cce": "INDY Gov NFT-8cce", + "3f28fb7d6c40468262dffb1c3adb568b342499826b664d940085d022": "Indigo Protocol (Stability pool account)", "408f13b240f57c3473b5727a68a88c50c1b6bb15c0c10912008dc59e": "INDY iBTC mint NFT-c59e", "443c51db609bba8b2aa4c8af248bf797cbfcfa1e413c443296a50813": "INDY SP Account-0813", "4c820aadf6ae8c755430455f5803d283bde0b20114bd93a8f381c72c": "MELD-c72c", + "533bb94a8850ee3ccbe483106489399112b74c905342cb1792a797a0": "INDY Policy-97a0", "5817c34e5702473304f3cf676299176d3824e55b8c0bfa94830429fd": "MuesliSwap NFT-29fd", "636c06ed152e70e9a1240da585a2841c15e4c3174044ca9a62cadab2": "NFT test-dab2", + "6ac8ef33b510ec004fe11585f7c5a9f0c07f0c23428ab4f29c1d7d10": "MELD Policy-7d10", "708f5e6d597fc038d09a738d7be32edd6ea779d6feb32a53668d9050": "INDY CDP NFT-9050", "72c8ee38fc1adfc5db49f7384bd64d41e9b9ce46ec11ba9a292e5d02": "NFT test-5d02", + "735b37149eb0c2a5fb590bd60e39fe90ae3a96b6065b05d7aca99ebb": "Indigo Protocol (CDP creator)", "869fc72c11977e4be3e8e1cc63cca008a925886332795c9601f965ca": "USDC policy-65ca", + "8a1cfae21368b8bebbbed9800fec304e95cce39a2a57dc35e2e3ebaa": "MILK Policy-ebaa", "8a7b0a43a169218fb66996c3945f53d7712eae9e31db193adb55663f": "NFT test-663f", + "8f9c32977d2bacb87836b64f7811e99734c6368373958da20172afba": "MYIELD Policy-afba", + "8fef2d34078659493ce161a6c7fba4b56afefa8535296a5743f69587": "Lenfi DAO Policy-9587", "909133088303c49f3a30f1cc8ed553a73857a29779f6c6561cd8093f": "MuesliSwap NFT Policy v2-093f", "97da12de04a6b527cc3b3469c5e5485cf258dfd1021f12e728f2e714": "INDY IASSET NFT-e714", "9a9693a9a37912a5097918f97918d15240c92ab729a0b7c4aa144d77": "Sundae policy-4d77", + "a0028f350aaabe0545fdcb56b039bfb08e4bb4d8c4d7c3c7d481c235": "HOSKY Policy-c235", "af3d70acf4bd5b3abb319a7d75c89fb3e56eafcdd46b2e9b57a2557f": "MuesliSwap NFT-557f", + "c0ee29a85b13209423b10447d3c2e6a50641a15c57770e27cb9d5073": "WRT Policy-5073", "d195ca7db29f0f13a00cac7fca70426ff60bad4e1e87d3757fae8484": "Minswap LP NFT 1-8484", "de9b756719341e79785aa13c164e7fe68c189ed04d61c9876b2fe53f": "MuesliSwap Factory Policy v2-e53f", "e3455f2715338b454fb853442f72dc03b98396854f97510027fe22ff": "INDY iUSD NFT-22ff", "e4214b7cce62ac6fbba385d164df48e157eae5863521b4b67ca71d86": "Minswap NFT-1d86", "e8a447d4e19016ca2aa74d20b4c4de87adb1f21dfb5493bf2d7281a6": "Sundae scooper-81a6", + "f43a62fdc3965df486de8a0d32fe800963589c41b38946602a0dc535": "AGIX Policy-c535", "f66d78b4a3cb3d37afa0ec36461e51ecbde00f26c8f0a68f94b69880": "Indigo-9880", - "fd0d72fafee1d230a74c31ac503a192abd5b71888ae3f94128c1e634": "INDY staking NFT-e634", - "1d7f33bd23d85e1a25d87d86fac4f199c3197a2f7afeb662a0f34e1e": "WMT Policy-4e1e", - "25f0fc240e91bd95dcdaebd2ba7713fc5168ac77234a3d79449fc20c": "SOCIETY Policy-c20c", - "533bb94a8850ee3ccbe483106489399112b74c905342cb1792a797a0": "INDY Policy-97a0", - "6ac8ef33b510ec004fe11585f7c5a9f0c07f0c23428ab4f29c1d7d10": "MELD Policy-7d10", - "8a1cfae21368b8bebbbed9800fec304e95cce39a2a57dc35e2e3ebaa": "MILK Policy-ebaa", - "8f9c32977d2bacb87836b64f7811e99734c6368373958da20172afba": "MYIELD Policy-afba", - "8fef2d34078659493ce161a6c7fba4b56afefa8535296a5743f69587": "Lenfi DAO Policy-9587", - "a0028f350aaabe0545fdcb56b039bfb08e4bb4d8c4d7c3c7d481c235": "HOSKY Policy-c235", - "c0ee29a85b13209423b10447d3c2e6a50641a15c57770e27cb9d5073": "WRT Policy-5073", - "f43a62fdc3965df486de8a0d32fe800963589c41b38946602a0dc535": "AGIX Policy-c535" + "f9b162ea9529e639a083595294006a833473883a75d6df1e4c22dd4f": "Indigo Protocol (Poll Manager)", + "fd0d72fafee1d230a74c31ac503a192abd5b71888ae3f94128c1e634": "INDY staking NFT-e634" }, // Known script hashes "scripts": { From f0086ace1712e63ac35eef7a27fd03373fb479e3 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Tue, 26 Sep 2023 18:39:17 +0200 Subject: [PATCH 050/124] Use (asset,addr) as key --- .../cardano_account_pandas_dumper.py | 112 +++++++++--------- 1 file changed, 53 insertions(+), 59 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index f9fb2fb..5e562ac 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -342,39 +342,22 @@ def _format_policy(self, policy: str) -> Optional[str]: def _decimals_for_asset(self, asset: str) -> np.longlong: return np.longlong(self.data.assets[(asset,)].iloc(0)[0].metadata.decimals or 0) - @functools.lru_cache(maxsize=10000) - def _asset_tuple(self, asset_id: str, unmute: bool) -> Optional[Tuple]: - asset = self.data.assets[(asset_id,)].iloc[0] - if ( - not unmute - and any(self.muted_policies == asset.policy_id) - and not any(self.pinned_policies == asset.policy_id) - ): - return None - return ( - self._format_policy(asset.policy_id), - self.asset_names.get(asset_id), - self._decimals_for_asset(asset_id), - ) - def _transaction_balance(self, transaction: blockfrost.utils.Namespace) -> Any: - # Index: (policy,asset,decimals, address,address_name,own) + # Index: (asset_id, address_name, own) result: MutableMapping[Tuple, np.longlong] = defaultdict(lambda: np.longlong(0)) - ada_asset = self._asset_tuple(self.data.LOVELACE_ASSET, True) - assert ada_asset is not None, "Asset for ADA unexpectedly None" - result[ada_asset + ("", " fees", self.OWN_LABEL)] = np.longlong( + result[(self.data.LOVELACE_ASSET, " fees", self.OWN_LABEL)] = np.longlong( transaction.fees ) - result[ada_asset + ("", " deposit", self.OWN_LABEL)] = np.longlong( + result[(self.data.LOVELACE_ASSET, " deposit", self.OWN_LABEL)] = np.longlong( transaction.deposit ) if transaction.reward_amount: - result[ada_asset + ("", " rewards", self.OWN_LABEL)] = np.longlong( - transaction.reward_amount - ) + result[ + (self.data.LOVELACE_ASSET, " rewards", self.OWN_LABEL) + ] = np.longlong(transaction.reward_amount) if transaction.withdrawals: result[ - ada_asset + ("", " rewards withdrawal", self.OWN_LABEL) + (self.data.LOVELACE_ASSET, " rewards withdrawal", self.OWN_LABEL) ] = np.negative( functools.reduce( np.add, @@ -385,32 +368,9 @@ def _transaction_balance(self, transaction: blockfrost.utils.Namespace) -> Any: for utxo in transaction.utxos.nonref_inputs: if not utxo.collateral or not transaction.valid_contract: for amount in utxo.amount: - asset_tuple = self._asset_tuple(amount.unit, self.args.unmute) - if asset_tuple is not None: - result[ - asset_tuple - + ( - utxo.address, - self.address_names.get( - utxo.address, - self._truncate(utxo.address) - if self.args.raw_values - else self.OTHER_LABEL, - ), - self.OWN_LABEL - if utxo.address in self.data.own_addresses - else self.OTHER_LABEL, - ) - ] -= np.longlong(amount.quantity) - - for utxo in transaction.utxos.outputs: - for amount in utxo.amount: - asset_tuple = self._asset_tuple(amount.unit, self.args.unmute) - if asset_tuple is not None: result[ - asset_tuple - + ( - utxo.address, + ( + amount.unit, self.address_names.get( utxo.address, self._truncate(utxo.address) @@ -421,7 +381,24 @@ def _transaction_balance(self, transaction: blockfrost.utils.Namespace) -> Any: if utxo.address in self.data.own_addresses else self.OTHER_LABEL, ) - ] += np.longlong(amount.quantity) + ] -= np.longlong(amount.quantity) + + for utxo in transaction.utxos.outputs: + for amount in utxo.amount: + result[ + ( + amount.unit, + self.address_names.get( + utxo.address, + self._truncate(utxo.address) + if self.args.raw_values + else self.OTHER_LABEL, + ), + self.OWN_LABEL + if utxo.address in self.data.own_addresses + else self.OTHER_LABEL, + ) + ] += np.longlong(amount.quantity) return result @@ -431,21 +408,30 @@ def _extract_timestamp(cls, transaction: blockfrost.utils.Namespace) -> Any: datetime.datetime.fromtimestamp(transaction.block_time) ) + (int(transaction.index) * cls.TRANSACTION_OFFSET) - def _drop_foreign_assets(self, balance: pd.DataFrame) -> None: + def _drop_muted_assets(self, balance: pd.DataFrame) -> None: # Drop assets that only touch foreign addresses assets_to_drop = frozenset( # Assets that touch other addresses - x[:2] + x[:1] for x in balance.xs(self.OTHER_LABEL, level=-1, axis=1).columns + ).union( + # Assets with muted policies + frozenset( + [ + (asset.asset_id,) + for asset in self.data.assets + if any(self.muted_policies == asset.policy_id) + ] + ) ) - frozenset( # Assets that touch own addresses - x[:2] + x[:1] for x in balance.xs(self.OWN_LABEL, level=-1, axis=1).columns ).union( # Assets with pinned policies frozenset( [ - self._asset_tuple(asset.asset_id, True)[:2] # type: ignore[index] + (asset.asset_id,) for asset in self.data.assets if any(self.pinned_policies == asset.policy_id) ] @@ -498,19 +484,27 @@ def make_balance_frame(self, transactions): ) balance.columns = pd.MultiIndex.from_tuples(balance.columns) balance.sort_index(axis=1, level=0, sort_remaining=True, inplace=True) - self._drop_foreign_assets(balance) + self._drop_muted_assets(balance) if self.args.detail_level == 1: - balance.drop(labels=self.OTHER_LABEL, axis=1, level=5, inplace=True) + balance.drop(labels=self.OTHER_LABEL, axis=1, level=2, inplace=True) balance = ( balance.T.groupby( - level=(0, 1, 2, 4) if not self.args.raw_values else (0, 1, 2, 4, 5) + level=(0, 1) + if not (self.args.raw_values and self.args.detail_level > 1) + else (0, 1, 2) ) .sum(numeric_only=True) .T ) balance = balance * [ - np.float_power(10, np.negative(c[2])) for c in balance.columns + np.float_power(10, np.negative(self._decimals_for_asset(c[0]))) + for c in balance.columns ] - balance.columns = balance.columns.droplevel(2) + if not self.args.raw_values: + balance.columns = pd.MultiIndex.from_tuples( + [(self.asset_names[c[0]], c[1]) for c in balance.columns] + ) + balance.sort_index(axis=1, level=0, sort_remaining=True, inplace=True) + return balance From 616114bc0d0272e71e232861a6d5d62ecda56ffb Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Tue, 26 Sep 2023 18:52:45 +0200 Subject: [PATCH 051/124] fix muting --- .../cardano_account_pandas_dumper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 5e562ac..d71441a 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -418,7 +418,7 @@ def _drop_muted_assets(self, balance: pd.DataFrame) -> None: # Assets with muted policies frozenset( [ - (asset.asset_id,) + asset.asset_id for asset in self.data.assets if any(self.muted_policies == asset.policy_id) ] @@ -431,7 +431,7 @@ def _drop_muted_assets(self, balance: pd.DataFrame) -> None: # Assets with pinned policies frozenset( [ - (asset.asset_id,) + asset.asset_id for asset in self.data.assets if any(self.pinned_policies == asset.policy_id) ] From 475ae3cb54f82f82a2a51880da1406eb11a447e4 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Wed, 27 Sep 2023 18:28:46 +0200 Subject: [PATCH 052/124] simplify checkpoint --- .../cardano_account_pandas_dumper.py | 106 +++++++----------- 1 file changed, 42 insertions(+), 64 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index d71441a..22775a6 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -99,21 +99,6 @@ def _transaction_data( name="Transactions", data=result_list, index=[t.hash for t in result_list] ).sort_index() - @classmethod - def _fix_api_asset( - cls, asset_id: str, asset: blockfrost.utils.Namespace - ) -> blockfrost.utils.Namespace: - asset.asset_id = asset_id - if not hasattr(asset, "metadata") or asset.metadata is None: - asset.metadata = blockfrost.utils.Namespace() - if not (hasattr(asset.metadata, "name") and asset.metadata.name): - asset.raw_name = asset_id.removeprefix(asset.policy_id) - else: - asset.raw_name = str(bytes(asset.metadata.name, "utf-8").hex()) - if not hasattr(asset.metadata, "decimals"): - asset.metadata.decimals = 0 - return asset - def _assets_from_transactions(self, api: BlockFrostApi) -> pd.Series: all_asset_ids: Set[str] = set() for tx_obj in self.transactions: # pylint: disable=not-an-iterable @@ -122,29 +107,15 @@ def _assets_from_transactions(self, api: BlockFrostApi) -> pd.Series: [a.unit for i in tx_obj.utxos.inputs for a in i.amount] + [a.unit for i in tx_obj.utxos.outputs for a in i.amount] ) - lovelace_asset_obj = blockfrost.utils.Namespace() - lovelace_asset_obj.metadata = blockfrost.utils.Namespace() - lovelace_asset_obj.metadata.name = "ADA" - lovelace_asset_obj.metadata.decimals = self.LOVELACE_DECIMALS - lovelace_asset_obj.policy_id = "" - lovelace_asset_obj.asset_name = "ADA" asset_list = [ - self._fix_api_asset( - asset, - api.asset(asset) - if asset != self.LOVELACE_ASSET - else lovelace_asset_obj, - ) + (asset, api.asset(asset)) for asset in all_asset_ids + if asset != self.LOVELACE_ASSET ] return pd.Series( - data=asset_list, - index=pd.MultiIndex.from_tuples( - [ - (asset.asset_id, asset.policy_id, asset.raw_name) - for asset in asset_list - ] - ), + name="Assets", + data=[a[1] for a in asset_list], + index=[a[0] for a in asset_list], ).sort_index() @staticmethod @@ -187,18 +158,17 @@ def _reward_transactions(self, api: BlockFrostApi) -> pd.Series: if a_r.epoch < self.end_epoch ] - pool_result_list = { + pool_dict = { pool: api.pool_metadata(pool) for pool in frozenset([r.pool_id for r in reward_list]) } reward_result_list = [ - self._reward_transaction(api=api, reward=a_r, pools=pool_result_list) + self._reward_transaction(api=api, reward=a_r, pools=pool_dict) for a_r in reward_list ] return pd.Series( name="Rewards", data=reward_result_list, - index=[t.hash for t in reward_result_list], ).sort_index() @@ -211,32 +181,37 @@ class AccountPandasDumper: def __init__(self, data: AccountData, known_dict: Any, args: argparse.Namespace): self.data = data - self.known_dict = known_dict self.args = args - self.address_names = ( - pd.concat( - [ - pd.Series( - known_dict.get("addresses", {}), - ), - pd.Series({a: " wallet" for a in self.data.own_addresses}), - ] + self.address_names = pd.Series( + ( + {a: " wallet" for a in self.data.own_addresses} + | known_dict.get("addresses", {}) ) if not args.raw_values - else pd.Series() + else {} ) - self.policy_names = ( - pd.Series(known_dict.get("policies", {})) - if not args.raw_values - else pd.Series() + self.policy_names = pd.Series( + known_dict.get("policies", {}) if not args.raw_values else {} ) self.asset_names = pd.Series( - { - asset.asset_id: self._decode_asset_name(asset.raw_name) + ( + { + asset.asset: self._decode_asset_name(asset) + for asset in self.data.assets + } if not args.raw_values - else self._truncate(asset.raw_name) + else {} + ) + | {self.data.LOVELACE_ASSET: " ADA"} + ) + self.asset_decimals = pd.Series( + { + asset.asset: np.longlong(asset.metadata.decimals or 0) + if hasattr(asset, "metadata") and hasattr(asset.metadata, "decimals") + else 0 for asset in self.data.assets } + | {self.data.LOVELACE_ASSET: self.data.LOVELACE_DECIMALS} ) self.muted_policies = pd.Series(known_dict.get("muted_policies", [])) self.pinned_policies = pd.Series(known_dict.get("pinned_policies", [])) @@ -251,12 +226,18 @@ def _truncate(self, value: str) -> str: else ("..." + value[-self.args.truncate_length :]) ) - def _decode_asset_name(self, asset_raw_name: str) -> str: + def _decode_asset_name(self, asset: blockfrost.utils.Namespace) -> str: + if ( + hasattr(asset, "metadata") + and hasattr(asset.metadata, "name") + and asset.metadata.name + ): + return asset.metadata.name + asset_hex_name = asset.asset.removeprefix(asset.policy_id) try: - return bytes.fromhex(asset_raw_name).decode() + return bytes.fromhex(asset_hex_name).decode() except UnicodeDecodeError: - pass - return self._truncate(asset_raw_name) + return f"{self._format_policy(asset.policy_id)}@{self._truncate(asset_hex_name)}" @staticmethod def _is_hex_number(num: Any) -> bool: @@ -339,9 +320,6 @@ def _format_script(self, script: str) -> str: def _format_policy(self, policy: str) -> Optional[str]: return self.policy_names.get(policy, self._truncate(policy)) - def _decimals_for_asset(self, asset: str) -> np.longlong: - return np.longlong(self.data.assets[(asset,)].iloc(0)[0].metadata.decimals or 0) - def _transaction_balance(self, transaction: blockfrost.utils.Namespace) -> Any: # Index: (asset_id, address_name, own) result: MutableMapping[Tuple, np.longlong] = defaultdict(lambda: np.longlong(0)) @@ -418,7 +396,7 @@ def _drop_muted_assets(self, balance: pd.DataFrame) -> None: # Assets with muted policies frozenset( [ - asset.asset_id + asset.asset for asset in self.data.assets if any(self.muted_policies == asset.policy_id) ] @@ -431,7 +409,7 @@ def _drop_muted_assets(self, balance: pd.DataFrame) -> None: # Assets with pinned policies frozenset( [ - asset.asset_id + asset.asset for asset in self.data.assets if any(self.pinned_policies == asset.policy_id) ] @@ -498,7 +476,7 @@ def make_balance_frame(self, transactions): ) balance = balance * [ - np.float_power(10, np.negative(self._decimals_for_asset(c[0]))) + np.float_power(10, np.negative(self.asset_decimals[c[0]])) for c in balance.columns ] if not self.args.raw_values: From 7120b723de1eb48b226ab1d020a25e2cfda0bdd1 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Thu, 28 Sep 2023 17:56:11 +0200 Subject: [PATCH 053/124] more checkpoint simplification --- src/cardano_account_pandas_dumper/__main__.py | 8 +- .../cardano_account_pandas_dumper.py | 168 +++++++++--------- 2 files changed, 86 insertions(+), 90 deletions(-) diff --git a/src/cardano_account_pandas_dumper/__main__.py b/src/cardano_account_pandas_dumper/__main__.py index c1b610d..84bda6d 100644 --- a/src/cardano_account_pandas_dumper/__main__.py +++ b/src/cardano_account_pandas_dumper/__main__.py @@ -139,7 +139,7 @@ def main(): api=api_instance, staking_addresses=staking_addresses_set, to_block=args.to_block, - rewards=not args.no_rewards, + include_rewards=not args.no_rewards, ) except (ApiError, JSONDecodeError, OSError) as exception: parser.exit( @@ -164,7 +164,11 @@ def main(): transactions = pd.concat( objs=[ data_from_api.transactions, - pd.Series() if args.no_rewards else data_from_api.reward_transactions, + pd.Series( + [] + if args.no_rewards + else [reporter.reward_transaction(r) for r in data_from_api.rewards] + ), ], ).rename("transactions") dataframe = reporter.make_transaction_frame(transactions) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 22775a6..50a146f 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -9,10 +9,8 @@ Dict, FrozenSet, List, - Mapping, MutableMapping, Optional, - Set, Tuple, ) import pandas as pd @@ -32,25 +30,15 @@ def __init__( api: BlockFrostApi, staking_addresses: FrozenSet[str], to_block: Optional[int], - rewards: bool, + include_rewards: bool, ) -> None: self.staking_addresses = staking_addresses - if to_block is None: - to_block = int(api.block_latest().height - 1) - self.to_block = to_block + self.to_block = to_block or int(api.block_latest().height - 1) block_last = api.block(self.to_block) block_after_last = api.block(self.to_block + 1) self.end_time = block_after_last.time self.end_epoch = block_last.epoch + 1 - self.own_addresses: FrozenSet[str] = self._own_addresses(api) - self.rewards = rewards - if self.rewards: - self.reward_transactions = self._reward_transactions(api) - self.transactions = self._transaction_data(api) - self.assets = self._assets_from_transactions(api) - - def _own_addresses(self, api: BlockFrostApi) -> FrozenSet[str]: - return frozenset( + self.own_addresses = frozenset( [ a.address for a in itertools.chain( @@ -61,6 +49,52 @@ def _own_addresses(self, api: BlockFrostApi) -> FrozenSet[str]: ) ] ) + self.transactions = self._transaction_data(api) + self.assets = pd.Series( + name="Assets", + data={ + a: api.asset(a) + for a in frozenset( + [ + a.unit + for tx_obj in self.transactions # pylint: disable=not-an-iterable + for i in (tx_obj.utxos.inputs + tx_obj.utxos.outputs) + for a in i.amount + ] + ).difference([self.LOVELACE_ASSET]) + }, + ).sort_index() + self.rewards = pd.Series( + name="Rewards", + data=[ + (s_a, a_r) + for s_a in self.staking_addresses + for a_r in api.account_rewards(s_a, gather_pages=True) + if a_r.epoch < self.end_epoch + ] + if include_rewards + else [], + ) + self.pools = pd.Series( + name="Pools", + data={ + pool: api.pool_metadata(pool) + for pool in frozenset([r[1].pool_id for r in self.rewards]) + } + if include_rewards + else {}, + ).sort_index() + self.epochs = pd.Series( + name="Epochs", + data={ + e: api.epoch(e) + for e in frozenset([r[1].epoch for r in self.rewards]).union( + [r[1].epoch + 1 for r in self.rewards] + ) + } + if include_rewards + else {}, + ).sort_index() def _transaction_data( self, @@ -96,79 +130,7 @@ def _transaction_data( result_list.append(transaction) return pd.Series( - name="Transactions", data=result_list, index=[t.hash for t in result_list] - ).sort_index() - - def _assets_from_transactions(self, api: BlockFrostApi) -> pd.Series: - all_asset_ids: Set[str] = set() - for tx_obj in self.transactions: # pylint: disable=not-an-iterable - if hasattr(tx_obj, "utxos"): - all_asset_ids.update( - [a.unit for i in tx_obj.utxos.inputs for a in i.amount] - + [a.unit for i in tx_obj.utxos.outputs for a in i.amount] - ) - asset_list = [ - (asset, api.asset(asset)) - for asset in all_asset_ids - if asset != self.LOVELACE_ASSET - ] - return pd.Series( - name="Assets", - data=[a[1] for a in asset_list], - index=[a[0] for a in asset_list], - ).sort_index() - - @staticmethod - def _reward_transaction( - api: BlockFrostApi, - reward: blockfrost.utils.Namespace, - pools: Mapping[str, blockfrost.utils.Namespace], - ) -> blockfrost.utils.Namespace: - result = blockfrost.utils.Namespace() - result.tx_hash = None - pool_name = ( - pools[reward.pool_id].name if reward.pool_id in pools else reward.pool_id - ) - result.metadata = [ - blockfrost.utils.Namespace( - label="674", - json_metadata=f"Reward: {reward.type} - {pool_name} - {reward.epoch}", - ) - ] - result.reward_amount = reward.amount - epoch = api.epoch(reward.epoch + 1) # Time is right before start of next epoch. - result.block_time = epoch.start_time - result.index = -1 - result.fees = "0" - result.deposit = "0" - result.redeemers = [] - result.hash = None - result.withdrawals = [] - result.utxos = blockfrost.utils.Namespace() - result.utxos.inputs = [] - result.utxos.outputs = [] - result.utxos.nonref_inputs = [] - return result - - def _reward_transactions(self, api: BlockFrostApi) -> pd.Series: - reward_list = [ - a_r - for s_a in self.staking_addresses - for a_r in api.account_rewards(s_a, gather_pages=True) - if a_r.epoch < self.end_epoch - ] - - pool_dict = { - pool: api.pool_metadata(pool) - for pool in frozenset([r.pool_id for r in reward_list]) - } - reward_result_list = [ - self._reward_transaction(api=api, reward=a_r, pools=pool_dict) - for a_r in reward_list - ] - return pd.Series( - name="Rewards", - data=reward_result_list, + name="Transactions", data={t.hash: t for t in result_list} ).sort_index() @@ -417,6 +379,36 @@ def _drop_muted_assets(self, balance: pd.DataFrame) -> None: ) balance.drop(assets_to_drop, axis=1, inplace=True) + def reward_transaction( + self, reward: Tuple[str, blockfrost.utils.Namespace] + ) -> blockfrost.utils.Namespace: + """Build reward pseudo-transaction for tuple (staking_addr, reward).""" + result = blockfrost.utils.Namespace() + result.tx_hash = None + result.metadata = [ + blockfrost.utils.Namespace( + label="674", + json_metadata=f"Reward: {reward[1].type} - {self.data.pools[reward[1].pool_id].name}" + + f" - {reward[0]} - {reward[1].epoch}", + ) + ] + result.reward_amount = reward[1].amount + epoch = self.data.epochs[ + reward[1].epoch + 1 + ] # Time is right before start of next epoch. + result.block_time = epoch.start_time + result.index = -1 + result.fees = "0" + result.deposit = "0" + result.redeemers = [] + result.hash = None + result.withdrawals = [] + result.utxos = blockfrost.utils.Namespace() + result.utxos.inputs = [] + result.utxos.outputs = [] + result.utxos.nonref_inputs = [] + return result + def make_transaction_frame( self, transactions: pd.Series, From 3e2305f87af9c0bc1264b70256958a6d61129c7c Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Thu, 28 Sep 2023 17:57:10 +0200 Subject: [PATCH 054/124] bump version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e010679..02205cd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "cardano_account_pandas_dumper" -version = "2023.1.0" +version = "2023.2.0" description = "Create a spreadsheet with the owned amount of any Cardano asset at the end of a specific block, and a record of the transactions that affected it." readme = "README.md" requires-python = ">=3.11" From 586d22ab898eb4fe897747ba0fd77abe247b92c3 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Fri, 29 Sep 2023 09:30:25 +0200 Subject: [PATCH 055/124] add back xlsx, remove pandas --- .gitignore | 1 + pyproject.toml | 2 +- src/cardano_account_pandas_dumper/__main__.py | 24 ++++++++++++------- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index 5334e1a..15ad5c3 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ .~* *.pickle *.csv +*.xlsx *.bak build *.egg-info diff --git a/pyproject.toml b/pyproject.toml index 02205cd..0b77a02 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ readme = "README.md" requires-python = ">=3.11" license = { file = "LICENSE" } keywords = ["Cardano", "Pandas", "report", "wallet"] -dependencies = ["jstyleson", "pandas", "blockfrost-python"] +dependencies = ["jstyleson", "pandas", "blockfrost-python", "openpyxl"] [project.scripts] cardano_account_pandas_dumper = "cardano_account_pandas_dumper.__main__:main" diff --git a/src/cardano_account_pandas_dumper/__main__.py b/src/cardano_account_pandas_dumper/__main__.py index 84bda6d..1c32548 100644 --- a/src/cardano_account_pandas_dumper/__main__.py +++ b/src/cardano_account_pandas_dumper/__main__.py @@ -52,8 +52,8 @@ def _create_arg_parser(): type=argparse.FileType("rb"), ) result.add_argument( - "--pandas_output", - help="Path to pickled Pandas dataframe output file.", + "--xlsx_output", + help="Path to .xlsx output file.", type=argparse.FileType("wb"), ) result.add_argument( @@ -100,11 +100,11 @@ def main(): """Main function.""" parser = _create_arg_parser() args = parser.parse_args() - if not any([args.checkpoint_output, args.csv_output, args.pandas_output]): + if not any([args.checkpoint_output, args.csv_output, args.xlsx_output]): parser.exit( status=1, message="No output specified, neeed at least one of --checkpoint_output," - + " --csv_output, --pandas_output.\n", + + " --csv_output, --xlsx_output.\n", ) known_dict_from_file = jstyleson.load(args.known_file) if args.known_file else {} staking_addresses_set = frozenset(args.staking_address) @@ -172,16 +172,22 @@ def main(): ], ).rename("transactions") dataframe = reporter.make_transaction_frame(transactions) - if args.pandas_output: - try: - dataframe.to_pickle(args.pandas_output) - except (pickle.PicklingError, OSError) as exception: - warnings.warn(f"Failed to write pandas file: {exception}") if args.csv_output: try: dataframe.replace(np.float64(0), pd.NA).to_csv(args.csv_output, index=False) except OSError as exception: warnings.warn(f"Failed to write CSV file: {exception}") + if args.xlsx_output: + try: + dataframe.replace(np.float64(0), pd.NA).to_excel( + args.xlsx_output, + index=True, + sheet_name=f"Transactions until block {args.to_block}", + merge_cells=True, + freeze_panes=(3 if args.raw_values else 2, 3), + ) + except OSError as exception: + warnings.warn(f"Failed to write .xlsx file: {exception}") print("Done.") From cdee2f7170520862ea001d17f09bec3b37fec259 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Fri, 29 Sep 2023 09:35:46 +0200 Subject: [PATCH 056/124] update README --- README.md | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 018b36a..4a84a83 100644 --- a/README.md +++ b/README.md @@ -69,16 +69,13 @@ For instance, block 8211670 matches EOY 2022 pretty closely. : Path to checkpoint file to read, if any. The checkpoint must have been created with the `--checkpoint_output` flag. -`--pandas_output PANDAS_OUTPUT` -: Path to pickled Pandas dataframe output file. +`--xlsx_output XLSX_OUTPUT` +: Path to Excel spreadsheet output file. If you want to further process the data with [Pandas](https://pandas.pydata.org/), you can serialize the generated `DataFrame` into a file. `--csv_output CSV_OUTPUT` : Path to CSV output file. -This the flag most people will need, it specifies the CSV file to write the output to. -Each row is a transaction, each column is a combination of asset + address. -Addresses belonging to one of the specified staking addresses are labeled as `own`. -With `--detail_level=2`, known addresses are listed with their name, other addresses are labeled as `other`. +Specifies the CSV file to write the output to. `--detail_level DETAIL_LEVEL` : Level of detail of report (1=only own addresses, 2=other addresses as well). @@ -108,7 +105,7 @@ If you need numerical hex values to not be truncated at all (see `--truncate_len ## Output format -### CSV +### CSV and XLSX column 0: transaction timestamp @@ -128,6 +125,8 @@ row 2: address If the `--raw_values` flag is passed, row 3 is inserted, with a value of `own`for own addresses (belonging to the specified staking addresses) and `other` for other addresses (if `--raw_values`is not passed, this information is on row 2). +Addresses belonging to one of the specified staking addresses are labeled as `own`. +With `--detail_level=2`, known addresses are listed with their name, other addresses are labeled as `other`. ## Possible improvements @@ -170,8 +169,7 @@ Here is a comparison table for both projects (please submit corrections if you t | Knows about assets other than ADA |✔️|❌| | Knows about DeFI contract addresses |✔️[^2]|❌| | Extracts useful information from tx metadata |✔️|❌| -| .xlsx output |❌[^3]|✔️| -| [Pandas](https://pandas.pydata.org/) compatible |✔️|❌| +| .xlsx output |✔️|✔️| | Ready to use after one-liner install command |✔️|❌| | Code is [Mypy](https://mypy-lang.org/) clean |✔️|❌| | Lines of Python code in repo (2023-09-01)| 529 | 1011| @@ -182,5 +180,3 @@ Here is a comparison table for both projects (please submit corrections if you t [^1]: Could not get this to work [^2]: With `--detail_level=2` - -[^3]: Deliberate, since this format is lossy From 636c7ed16301d54a39d2e22e049401b580f0798d Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Fri, 29 Sep 2023 10:06:35 +0200 Subject: [PATCH 057/124] tweak checkpoint --- .../cardano_account_pandas_dumper.py | 67 ++++++++++--------- 1 file changed, 35 insertions(+), 32 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 50a146f..69d7f41 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -38,6 +38,39 @@ def __init__( block_after_last = api.block(self.to_block + 1) self.end_time = block_after_last.time self.end_epoch = block_last.epoch + 1 + self.rewards = pd.Series( + name="Rewards", + data=[ + (s_a, a_r) + for s_a in self.staking_addresses + for a_r in api.account_rewards(s_a, gather_pages=True) + if a_r.epoch < self.end_epoch + ] + if include_rewards + else [], + ) + self.epochs = pd.Series( + name="Epochs", + data={ + e: api.epoch(e) + for e in frozenset( + itertools.chain( + *[[r[1].epoch, r[1].epoch + 1] for r in self.rewards] + ) + ) + } + if include_rewards + else {}, + ).sort_index() + self.pools = pd.Series( + name="Pools", + data={ + pool: api.pool_metadata(pool) + for pool in frozenset([r[1].pool_id for r in self.rewards]) + } + if include_rewards + else {}, + ).sort_index() self.own_addresses = frozenset( [ a.address @@ -64,37 +97,6 @@ def __init__( ).difference([self.LOVELACE_ASSET]) }, ).sort_index() - self.rewards = pd.Series( - name="Rewards", - data=[ - (s_a, a_r) - for s_a in self.staking_addresses - for a_r in api.account_rewards(s_a, gather_pages=True) - if a_r.epoch < self.end_epoch - ] - if include_rewards - else [], - ) - self.pools = pd.Series( - name="Pools", - data={ - pool: api.pool_metadata(pool) - for pool in frozenset([r[1].pool_id for r in self.rewards]) - } - if include_rewards - else {}, - ).sort_index() - self.epochs = pd.Series( - name="Epochs", - data={ - e: api.epoch(e) - for e in frozenset([r[1].epoch for r in self.rewards]).union( - [r[1].epoch + 1 for r in self.rewards] - ) - } - if include_rewards - else {}, - ).sort_index() def _transaction_data( self, @@ -388,7 +390,8 @@ def reward_transaction( result.metadata = [ blockfrost.utils.Namespace( label="674", - json_metadata=f"Reward: {reward[1].type} - {self.data.pools[reward[1].pool_id].name}" + json_metadata="Reward: " + + f"{reward[1].type} - {self.data.pools[reward[1].pool_id].name}" + f" - {reward[0]} - {reward[1].epoch}", ) ] From 0db9d17a21a3997a1f1608eedf368b797591cbb4 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Fri, 29 Sep 2023 12:36:35 +0200 Subject: [PATCH 058/124] refactor args --- src/cardano_account_pandas_dumper/__main__.py | 20 +- .../cardano_account_pandas_dumper.py | 247 ++++++++---------- 2 files changed, 125 insertions(+), 142 deletions(-) diff --git a/src/cardano_account_pandas_dumper/__main__.py b/src/cardano_account_pandas_dumper/__main__.py index 1c32548..7eb7900 100644 --- a/src/cardano_account_pandas_dumper/__main__.py +++ b/src/cardano_account_pandas_dumper/__main__.py @@ -74,15 +74,10 @@ def _create_arg_parser(): ) result.add_argument( "--truncate_length", - help="Length to truncate numerical identifiers to.", + help="Length to truncate numerical identifiers to, 0= do not truncate.", type=int, default=6, ) - result.add_argument( - "--no_truncate", - help="Do not truncate numerical identifiers.", - action="store_true", - ) result.add_argument( "--raw_values", help="Keep assets, policies and addresses as hex instead of looking up names.", @@ -159,7 +154,11 @@ def main(): else: parser.exit(status=1, message="Staking address(es) required.") reporter = AccountPandasDumper( - data=data_from_api, known_dict=known_dict_from_file, args=args + data=data_from_api, + known_dict=known_dict_from_file, + truncate_length=args.truncate_length, + raw_values=args.raw_values, + unmute=args.unmute, ) transactions = pd.concat( objs=[ @@ -171,7 +170,10 @@ def main(): ), ], ).rename("transactions") - dataframe = reporter.make_transaction_frame(transactions) + dataframe = reporter.make_transaction_frame( + transactions, + detail_level=args.detail_level, + ) if args.csv_output: try: dataframe.replace(np.float64(0), pd.NA).to_csv(args.csv_output, index=False) @@ -182,7 +184,7 @@ def main(): dataframe.replace(np.float64(0), pd.NA).to_excel( args.xlsx_output, index=True, - sheet_name=f"Transactions until block {args.to_block}", + sheet_name=f"Transactions to block {args.to_block}", merge_cells=True, freeze_panes=(3 if args.raw_values else 2, 3), ) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 69d7f41..68099c2 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -1,29 +1,20 @@ """ Cardano Account To Pandas Dumper.""" -import argparse import datetime import functools import itertools from collections import defaultdict -from typing import ( - Any, - Dict, - FrozenSet, - List, - MutableMapping, - Optional, - Tuple, -) -import pandas as pd +from typing import Any, Dict, FrozenSet, List, MutableMapping, Optional, Tuple + +import blockfrost.utils import numpy as np +import pandas as pd from blockfrost import BlockFrostApi -import blockfrost.utils class AccountData: """Hold data retrieved from the API to allow checkpointing it.""" LOVELACE_ASSET = "lovelace" - LOVELACE_DECIMALS = 6 def __init__( self, @@ -142,31 +133,29 @@ class AccountPandasDumper: TRANSACTION_OFFSET = np.timedelta64(1000, "ns") OWN_LABEL = "own" OTHER_LABEL = "other" + ADA_ASSET = " ADA" + ADA_DECIMALS = 6 - def __init__(self, data: AccountData, known_dict: Any, args: argparse.Namespace): + def __init__( + self, + data: AccountData, + known_dict: Any, + truncate_length: int, + raw_values: bool, + unmute: bool, + ): self.data = data - self.args = args + self.truncate_length = truncate_length + self.raw_values = raw_values + self.unmute = unmute self.address_names = pd.Series( - ( - {a: " wallet" for a in self.data.own_addresses} - | known_dict.get("addresses", {}) - ) - if not args.raw_values - else {} - ) - self.policy_names = pd.Series( - known_dict.get("policies", {}) if not args.raw_values else {} + {a: " wallet" for a in self.data.own_addresses} + | known_dict.get("addresses", {}) ) + self.policy_names = pd.Series(known_dict.get("policies", {})) self.asset_names = pd.Series( - ( - { - asset.asset: self._decode_asset_name(asset) - for asset in self.data.assets - } - if not args.raw_values - else {} - ) - | {self.data.LOVELACE_ASSET: " ADA"} + {asset.asset: self._decode_asset_name(asset) for asset in self.data.assets} + | {self.ADA_ASSET: self.ADA_ASSET} ) self.asset_decimals = pd.Series( { @@ -175,7 +164,7 @@ def __init__(self, data: AccountData, known_dict: Any, args: argparse.Namespace) else 0 for asset in self.data.assets } - | {self.data.LOVELACE_ASSET: self.data.LOVELACE_DECIMALS} + | {self.ADA_ASSET: self.ADA_DECIMALS} ) self.muted_policies = pd.Series(known_dict.get("muted_policies", [])) self.pinned_policies = pd.Series(known_dict.get("pinned_policies", [])) @@ -186,8 +175,8 @@ def __init__(self, data: AccountData, known_dict: Any, args: argparse.Namespace) def _truncate(self, value: str) -> str: return ( value - if self.args.no_truncate or len(value) <= self.args.truncate_length - else ("..." + value[-self.args.truncate_length :]) + if not self.truncate_length or len(value) <= self.truncate_length + else ("..." + value[-self.truncate_length :]) ) def _decode_asset_name(self, asset: blockfrost.utils.Namespace) -> str: @@ -223,14 +212,14 @@ def _munge_metadata(self, obj: blockfrost.utils.Namespace) -> Any: hex_name = self._is_hex_number(att) value = getattr(obj, att) hex_value = isinstance(value, str) and self._is_hex_number(value) - if (hex_name and hex_value) and not self.args.unmute: + if (hex_name and hex_value) and not self.unmute: continue out_att = self._truncate(att) if hex_name else att value = self._munge_metadata(value) if value: result[out_att] = value return result - elif isinstance(obj, str) and self._is_hex_number(obj) and not self.args.unmute: + elif isinstance(obj, str) and self._is_hex_number(obj) and not self.unmute: return {} else: return obj @@ -245,7 +234,7 @@ def _format_message(self, tx_obj: blockfrost.utils.Namespace) -> str: if ( self._is_hex_number(label) and (not val or self._is_hex_number(val)) - and not self.args.unmute + and not self.unmute ): continue result.append(label) @@ -284,66 +273,6 @@ def _format_script(self, script: str) -> str: def _format_policy(self, policy: str) -> Optional[str]: return self.policy_names.get(policy, self._truncate(policy)) - def _transaction_balance(self, transaction: blockfrost.utils.Namespace) -> Any: - # Index: (asset_id, address_name, own) - result: MutableMapping[Tuple, np.longlong] = defaultdict(lambda: np.longlong(0)) - result[(self.data.LOVELACE_ASSET, " fees", self.OWN_LABEL)] = np.longlong( - transaction.fees - ) - result[(self.data.LOVELACE_ASSET, " deposit", self.OWN_LABEL)] = np.longlong( - transaction.deposit - ) - if transaction.reward_amount: - result[ - (self.data.LOVELACE_ASSET, " rewards", self.OWN_LABEL) - ] = np.longlong(transaction.reward_amount) - if transaction.withdrawals: - result[ - (self.data.LOVELACE_ASSET, " rewards withdrawal", self.OWN_LABEL) - ] = np.negative( - functools.reduce( - np.add, - [np.longlong(w.amount) for w in transaction.withdrawals], - np.longlong(0), - ) - ) - for utxo in transaction.utxos.nonref_inputs: - if not utxo.collateral or not transaction.valid_contract: - for amount in utxo.amount: - result[ - ( - amount.unit, - self.address_names.get( - utxo.address, - self._truncate(utxo.address) - if self.args.raw_values - else self.OTHER_LABEL, - ), - self.OWN_LABEL - if utxo.address in self.data.own_addresses - else self.OTHER_LABEL, - ) - ] -= np.longlong(amount.quantity) - - for utxo in transaction.utxos.outputs: - for amount in utxo.amount: - result[ - ( - amount.unit, - self.address_names.get( - utxo.address, - self._truncate(utxo.address) - if self.args.raw_values - else self.OTHER_LABEL, - ), - self.OWN_LABEL - if utxo.address in self.data.own_addresses - else self.OTHER_LABEL, - ) - ] += np.longlong(amount.quantity) - - return result - @classmethod def _extract_timestamp(cls, transaction: blockfrost.utils.Namespace) -> Any: return np.datetime64( @@ -412,9 +341,94 @@ def reward_transaction( result.utxos.nonref_inputs = [] return result + def _column_key(self, utxo, amount): + return ( + amount.unit if amount.unit != self.data.LOVELACE_ASSET else self.ADA_ASSET, + self._truncate(utxo.address) + if self.raw_values + else self.address_names.get( + utxo.address, + self.OTHER_LABEL, + ), + self.OWN_LABEL + if utxo.address in self.data.own_addresses + else self.OTHER_LABEL, + ) + + def _transaction_balance(self, transaction: blockfrost.utils.Namespace) -> Any: + # Index: (asset_id, address_name, own) + result: MutableMapping[Tuple, np.longlong] = defaultdict(lambda: np.longlong(0)) + result[(self.ADA_ASSET, " fees", self.OWN_LABEL)] = np.longlong( + transaction.fees + ) + result[(self.ADA_ASSET, " deposit", self.OWN_LABEL)] = np.longlong( + transaction.deposit + ) + if transaction.reward_amount: + result[(self.ADA_ASSET, " rewards", self.OWN_LABEL)] = np.longlong( + transaction.reward_amount + ) + if transaction.withdrawals: + result[ + (self.ADA_ASSET, " rewards withdrawal", self.OWN_LABEL) + ] = np.negative( + functools.reduce( + np.add, + [np.longlong(w.amount) for w in transaction.withdrawals], + np.longlong(0), + ) + ) + for utxo in transaction.utxos.nonref_inputs: + if not utxo.collateral or not transaction.valid_contract: + for amount in utxo.amount: + result[self._column_key(utxo, amount)] -= np.longlong( + amount.quantity + ) + + for utxo in transaction.utxos.outputs: + for amount in utxo.amount: + result[self._column_key(utxo, amount)] += np.longlong(amount.quantity) + + return result + + def make_balance_frame(self, transactions: pd.Series, detail_level: int): + """Make DataFrame with transaction balances.""" + balance = pd.DataFrame( + data=[self._transaction_balance(x) for x in transactions], + dtype="Int64", + ) + balance.columns = pd.MultiIndex.from_tuples(balance.columns) + balance.sort_index(axis=1, level=0, sort_remaining=True, inplace=True) + if not self.unmute: + self._drop_muted_assets(balance) + if detail_level == 1: + balance.drop(labels=self.OTHER_LABEL, axis=1, level=2, inplace=True) + balance = ( + balance.T.groupby( + level=(0, 1) + if not (self.raw_values and detail_level > 1) + else (0, 1, 2) + ) + .sum(numeric_only=True) + .T + ) + + balance = balance * [ + np.float_power(10, np.negative(self.asset_decimals[c[0]])) + for c in balance.columns + ] + if not self.raw_values: + balance.columns = pd.MultiIndex.from_tuples( + [(self.asset_names[c[0]], c[1]) for c in balance.columns] + ) + balance.sort_index(axis=1, level=0, sort_remaining=True, inplace=True) + + return balance + def make_transaction_frame( self, transactions: pd.Series, + detail_level: int, with_tx_hash: bool = True, with_tx_message: bool = True, with_total: bool = True, @@ -429,7 +443,7 @@ def make_transaction_frame( if with_tx_message: columns.append(transactions.rename("message").map(self._format_message)) total.append("Total") - balance = self.make_balance_frame(transactions) + balance = self.make_balance_frame(transactions, detail_level) frame = pd.concat(columns, axis=1) frame.reset_index(drop=True, inplace=True) frame.columns = pd.MultiIndex.from_tuples( @@ -448,36 +462,3 @@ def make_transaction_frame( [frame, pd.DataFrame(data=[total], columns=frame.columns)] ) return frame - - def make_balance_frame(self, transactions): - """Make DataFrame with transaction balances.""" - balance = pd.DataFrame( - data=[self._transaction_balance(x) for x in transactions], - dtype="Int64", - ) - balance.columns = pd.MultiIndex.from_tuples(balance.columns) - balance.sort_index(axis=1, level=0, sort_remaining=True, inplace=True) - self._drop_muted_assets(balance) - if self.args.detail_level == 1: - balance.drop(labels=self.OTHER_LABEL, axis=1, level=2, inplace=True) - balance = ( - balance.T.groupby( - level=(0, 1) - if not (self.args.raw_values and self.args.detail_level > 1) - else (0, 1, 2) - ) - .sum(numeric_only=True) - .T - ) - - balance = balance * [ - np.float_power(10, np.negative(self.asset_decimals[c[0]])) - for c in balance.columns - ] - if not self.args.raw_values: - balance.columns = pd.MultiIndex.from_tuples( - [(self.asset_names[c[0]], c[1]) for c in balance.columns] - ) - balance.sort_index(axis=1, level=0, sort_remaining=True, inplace=True) - - return balance From f2287bf8b9c3af73ad10d78d21e70ae48e326187 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Fri, 29 Sep 2023 14:55:02 +0200 Subject: [PATCH 059/124] More metadata parsing improvements --- .../cardano_account_pandas_dumper.py | 73 ++++++++++++------- src/cardano_account_pandas_dumper/known.jsonc | 1 - 2 files changed, 45 insertions(+), 29 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 68099c2..f920a7f 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -3,7 +3,7 @@ import functools import itertools from collections import defaultdict -from typing import Any, Dict, FrozenSet, List, MutableMapping, Optional, Tuple +from typing import Any, Dict, FrozenSet, List, MutableMapping, Optional, Set, Tuple import blockfrost.utils import numpy as np @@ -135,6 +135,8 @@ class AccountPandasDumper: OTHER_LABEL = "other" ADA_ASSET = " ADA" ADA_DECIMALS = 6 + METADATA_MESSAGE_LABEL = "674" + METADATA_NFT_MINT_LABEL = "721" def __init__( self, @@ -168,7 +170,6 @@ def __init__( ) self.muted_policies = pd.Series(known_dict.get("muted_policies", [])) self.pinned_policies = pd.Series(known_dict.get("pinned_policies", [])) - self.scripts = pd.Series(known_dict.get("scripts", {})) self.labels = pd.Series(known_dict.get("labels", {})) @@ -211,48 +212,65 @@ def _munge_metadata(self, obj: blockfrost.utils.Namespace) -> Any: continue hex_name = self._is_hex_number(att) value = getattr(obj, att) - hex_value = isinstance(value, str) and self._is_hex_number(value) - if (hex_name and hex_value) and not self.unmute: + if (hex_name and self._is_hex_number(value)) and not self.unmute: continue - out_att = self._truncate(att) if hex_name else att value = self._munge_metadata(value) if value: + out_att = self._truncate(att) if hex_name else att + if out_att == "msg": + return ( + " ".join(value) if isinstance(value, list) else str(value) + ) result[out_att] = value return result - elif isinstance(obj, str) and self._is_hex_number(obj) and not self.unmute: + elif self._is_hex_number(obj) and not self.unmute: return {} else: return obj + def _parse_nft_mint(self, meta: blockfrost.utils.Namespace) -> str: + meta_dict = meta.to_dict() + result = "NFT Mint:" + for policy, v in meta_dict.items(): + if policy == "version": + continue + for asset_name in v.to_dict().keys(): + result += f"{self._format_policy(policy)}@{asset_name} " + return result + def _format_message(self, tx_obj: blockfrost.utils.Namespace) -> str: result: List[str] = [] for metadata_key in tx_obj.metadata: - label = self.labels.get( - metadata_key.label, self._truncate(metadata_key.label) - ) - val = self._munge_metadata(metadata_key.json_metadata) + if metadata_key.label == self.METADATA_NFT_MINT_LABEL: + label = None + val = self._parse_nft_mint(metadata_key.json_metadata) + else: + if metadata_key.label == self.METADATA_MESSAGE_LABEL: + label = None + else: + label = self.labels.get( + metadata_key.label, self._truncate(metadata_key.label) + ) + val = self._munge_metadata(metadata_key.json_metadata) if ( self._is_hex_number(label) and (not val or self._is_hex_number(val)) and not self.unmute ): continue - result.append(label) - result.append(":") + if label: + result.extend([label, ":"]) result.append(str(val)) - redeemer_scripts: Dict[str, List] = defaultdict(list) + redeemer_scripts: Dict[str, Set] = defaultdict(set) for redeemer in tx_obj.redeemers: if redeemer.purpose == "spend": - redeemer_scripts["Spend:"].append( + redeemer_scripts["Spend:"].add( self._format_script(redeemer.script_hash) ) elif redeemer.purpose == "mint": - redeemer_scripts["Mint:"].append( - self._format_policy(redeemer.script_hash) - ) + redeemer_scripts["Mint:"].add(self._format_policy(redeemer.script_hash)) for k, redeemer_script in redeemer_scripts.items(): - result.append(k) - result.append(str(redeemer_script)) + result.extend([k, str(redeemer_script)]) if not result and all( [ utxo.address in self.data.own_addresses @@ -260,18 +278,17 @@ def _format_message(self, tx_obj: blockfrost.utils.Namespace) -> str: ] ): result = ["(internal)"] - return ( - " ".join(result) - .removeprefix("Message : ") - .removeprefix("{'msg': ['") - .removesuffix("']}") - ) + return " ".join(result) def _format_script(self, script: str) -> str: - return self.scripts.get(script, self._truncate(script)) + return ({} if self.raw_values else self.scripts).get( + script, self._truncate(script) + ) def _format_policy(self, policy: str) -> Optional[str]: - return self.policy_names.get(policy, self._truncate(policy)) + return ({} if self.raw_values else self.policy_names).get( + policy, self._truncate(policy) + ) @classmethod def _extract_timestamp(cls, transaction: blockfrost.utils.Namespace) -> Any: @@ -318,7 +335,7 @@ def reward_transaction( result.tx_hash = None result.metadata = [ blockfrost.utils.Namespace( - label="674", + label=self.METADATA_MESSAGE_LABEL, json_metadata="Reward: " + f"{reward[1].type} - {self.data.pools[reward[1].pool_id].name}" + f" - {reward[0]} - {reward[1].epoch}", diff --git a/src/cardano_account_pandas_dumper/known.jsonc b/src/cardano_account_pandas_dumper/known.jsonc index dd161d9..1ca3e90 100644 --- a/src/cardano_account_pandas_dumper/known.jsonc +++ b/src/cardano_account_pandas_dumper/known.jsonc @@ -51,7 +51,6 @@ "labels": { "61284": "Catalyst", "61285": "Catalyst Sig", - "674": "Message", "914425": "Indigo SP Reward" }, // Policies to mute from assets From d4017866973d04469c9c361b70e57bee02a5a535 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Fri, 29 Sep 2023 15:11:29 +0200 Subject: [PATCH 060/124] switch own and address index levels --- .../cardano_account_pandas_dumper.py | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index f920a7f..901f2e5 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -131,7 +131,7 @@ class AccountPandasDumper: """Hold logic to convert an instance of AccountData to a Pandas dataframe.""" TRANSACTION_OFFSET = np.timedelta64(1000, "ns") - OWN_LABEL = "own" + OWN_LABEL = " own" OTHER_LABEL = "other" ADA_ASSET = " ADA" ADA_DECIMALS = 6 @@ -301,7 +301,7 @@ def _drop_muted_assets(self, balance: pd.DataFrame) -> None: assets_to_drop = frozenset( # Assets that touch other addresses x[:1] - for x in balance.xs(self.OTHER_LABEL, level=-1, axis=1).columns + for x in balance.xs(self.OTHER_LABEL, level=1, axis=1).columns ).union( # Assets with muted policies frozenset( @@ -314,7 +314,7 @@ def _drop_muted_assets(self, balance: pd.DataFrame) -> None: ) - frozenset( # Assets that touch own addresses x[:1] - for x in balance.xs(self.OWN_LABEL, level=-1, axis=1).columns + for x in balance.xs(self.OWN_LABEL, level=1, axis=1).columns ).union( # Assets with pinned policies frozenset( @@ -359,35 +359,35 @@ def reward_transaction( return result def _column_key(self, utxo, amount): + # Index: (asset_id, own, address_name) return ( amount.unit if amount.unit != self.data.LOVELACE_ASSET else self.ADA_ASSET, + self.OWN_LABEL + if utxo.address in self.data.own_addresses + else self.OTHER_LABEL, self._truncate(utxo.address) if self.raw_values else self.address_names.get( utxo.address, self.OTHER_LABEL, ), - self.OWN_LABEL - if utxo.address in self.data.own_addresses - else self.OTHER_LABEL, ) def _transaction_balance(self, transaction: blockfrost.utils.Namespace) -> Any: - # Index: (asset_id, address_name, own) result: MutableMapping[Tuple, np.longlong] = defaultdict(lambda: np.longlong(0)) - result[(self.ADA_ASSET, " fees", self.OWN_LABEL)] = np.longlong( + result[(self.ADA_ASSET, self.OWN_LABEL, " fees")] = np.longlong( transaction.fees ) - result[(self.ADA_ASSET, " deposit", self.OWN_LABEL)] = np.longlong( + result[(self.ADA_ASSET, self.OWN_LABEL, " deposit")] = np.longlong( transaction.deposit ) if transaction.reward_amount: - result[(self.ADA_ASSET, " rewards", self.OWN_LABEL)] = np.longlong( + result[(self.ADA_ASSET, self.OWN_LABEL, " rewards")] = np.longlong( transaction.reward_amount ) if transaction.withdrawals: result[ - (self.ADA_ASSET, " rewards withdrawal", self.OWN_LABEL) + (self.ADA_ASSET, self.OWN_LABEL, " rewards withdrawal") ] = np.negative( functools.reduce( np.add, @@ -419,10 +419,10 @@ def make_balance_frame(self, transactions: pd.Series, detail_level: int): if not self.unmute: self._drop_muted_assets(balance) if detail_level == 1: - balance.drop(labels=self.OTHER_LABEL, axis=1, level=2, inplace=True) + balance.drop(labels=self.OTHER_LABEL, axis=1, level=1, inplace=True) balance = ( balance.T.groupby( - level=(0, 1) + level=(0, 2) if not (self.raw_values and detail_level > 1) else (0, 1, 2) ) @@ -436,7 +436,7 @@ def make_balance_frame(self, transactions: pd.Series, detail_level: int): ] if not self.raw_values: balance.columns = pd.MultiIndex.from_tuples( - [(self.asset_names[c[0]], c[1]) for c in balance.columns] + [(self.asset_names[c[0]], c[2]) for c in balance.columns] ) balance.sort_index(axis=1, level=0, sort_remaining=True, inplace=True) From e9117a3d624e44a7a9bee2ee6a9e5430cbca092a Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Fri, 29 Sep 2023 15:13:46 +0200 Subject: [PATCH 061/124] fix total if no message --- .../cardano_account_pandas_dumper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 901f2e5..af66774 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -456,7 +456,7 @@ def make_transaction_frame( total: List[Any] = [columns[0].max() + self.TRANSACTION_OFFSET] if with_tx_hash: columns.append(transactions.rename("hash").map(lambda x: x.hash)) - total.append("") + total.append("" if with_tx_message else "Total") if with_tx_message: columns.append(transactions.rename("message").map(self._format_message)) total.append("Total") From a101bf46b6637ddc12cfebf9b44cd4157d5af233 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Fri, 29 Sep 2023 15:24:05 +0200 Subject: [PATCH 062/124] fix lint --- .../cardano_account_pandas_dumper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index af66774..723cd8b 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -231,10 +231,10 @@ def _munge_metadata(self, obj: blockfrost.utils.Namespace) -> Any: def _parse_nft_mint(self, meta: blockfrost.utils.Namespace) -> str: meta_dict = meta.to_dict() result = "NFT Mint:" - for policy, v in meta_dict.items(): + for policy, _v in meta_dict.items(): if policy == "version": continue - for asset_name in v.to_dict().keys(): + for asset_name in _v.to_dict().keys(): result += f"{self._format_policy(policy)}@{asset_name} " return result From b909ff445aea22b1791661b9faf2151d366cfc0b Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Fri, 29 Sep 2023 15:25:17 +0200 Subject: [PATCH 063/124] fix table --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4a84a83..4ea1894 100644 --- a/README.md +++ b/README.md @@ -165,11 +165,11 @@ Here is a comparison table for both projects (please submit corrections if you t | Feature | [cardano_account_pandas_dumper](https://github.com/pixelsoup42/cardano_account_pandas_dumper) | [cardano-accointing-exporter](https://github.com/pabstma/cardano-accointing-exporter) | | ------- | --------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------| | CSV output |✔️|✔️| +| .xlsx output |✔️|✔️| | coingecko integration for fiat price |❌|✔️[^1]| | Knows about assets other than ADA |✔️|❌| | Knows about DeFI contract addresses |✔️[^2]|❌| | Extracts useful information from tx metadata |✔️|❌| -| .xlsx output |✔️|✔️| | Ready to use after one-liner install command |✔️|❌| | Code is [Mypy](https://mypy-lang.org/) clean |✔️|❌| | Lines of Python code in repo (2023-09-01)| 529 | 1011| From dad5610bdddc427bffcd003096e295d49bb8fb9f Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Fri, 29 Sep 2023 15:26:23 +0200 Subject: [PATCH 064/124] improve error handling --- src/cardano_account_pandas_dumper/__main__.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/cardano_account_pandas_dumper/__main__.py b/src/cardano_account_pandas_dumper/__main__.py index 7eb7900..995d9ce 100644 --- a/src/cardano_account_pandas_dumper/__main__.py +++ b/src/cardano_account_pandas_dumper/__main__.py @@ -12,6 +12,8 @@ from .cardano_account_pandas_dumper import AccountData, AccountPandasDumper +PROJECT_KEY_ERROR_CODES = frozenset([402, 403, 418, 429]) + def _create_arg_parser(): result = argparse.ArgumentParser(prog="cardano_account_pandas_dumper") @@ -136,15 +138,24 @@ def main(): to_block=args.to_block, include_rewards=not args.no_rewards, ) - except (ApiError, JSONDecodeError, OSError) as exception: + except ApiError as api_exception: parser.exit( status=2, message=( - f"Failed to read data from blockfrost.io: {exception}," - + " maybe create your own API key at https://blockfrost.io/dashboard and " - + "specify it with the --blockfrost_project_id flag." + f"Failed to read data from blockfrost.io: {api_exception}." + + ( + "\nMaybe create your own API key at https://blockfrost.io/dashboard and " + + "specify it with the --blockfrost_project_id flag." + ) + if api_exception.status_code in PROJECT_KEY_ERROR_CODES + else "" ), ) + except (JSONDecodeError, OSError) as exception: + parser.exit( + status=2, + message=(f"Failed to read data from blockfrost.io: {exception},"), + ) if args.checkpoint_output: try: pickle.dump(obj=data_from_api, file=args.checkpoint_output) From b8b493f4ebb5385bf137a9823c10bae3fca80071 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Fri, 29 Sep 2023 15:45:31 +0200 Subject: [PATCH 065/124] add mising cast --- src/cardano_account_pandas_dumper/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cardano_account_pandas_dumper/__main__.py b/src/cardano_account_pandas_dumper/__main__.py index 995d9ce..67c8d9d 100644 --- a/src/cardano_account_pandas_dumper/__main__.py +++ b/src/cardano_account_pandas_dumper/__main__.py @@ -147,7 +147,7 @@ def main(): "\nMaybe create your own API key at https://blockfrost.io/dashboard and " + "specify it with the --blockfrost_project_id flag." ) - if api_exception.status_code in PROJECT_KEY_ERROR_CODES + if int(api_exception.status_code) in PROJECT_KEY_ERROR_CODES else "" ), ) From 488c0c40f1a83b0a6c94146f30ae06019d296d4a Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Fri, 29 Sep 2023 16:14:29 +0200 Subject: [PATCH 066/124] validate staking addresses --- src/cardano_account_pandas_dumper/__main__.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/cardano_account_pandas_dumper/__main__.py b/src/cardano_account_pandas_dumper/__main__.py index 67c8d9d..590d3aa 100644 --- a/src/cardano_account_pandas_dumper/__main__.py +++ b/src/cardano_account_pandas_dumper/__main__.py @@ -16,7 +16,10 @@ def _create_arg_parser(): - result = argparse.ArgumentParser(prog="cardano_account_pandas_dumper") + result = argparse.ArgumentParser( + prog="cardano_account_pandas_dumper", + description="Retrieve transaction history for Cardano staking addresses.", + ) exclusive_group = result.add_mutually_exclusive_group() result.add_argument( "--blockfrost_project_id", @@ -97,6 +100,14 @@ def main(): """Main function.""" parser = _create_arg_parser() args = parser.parse_args() + invalid_staking_addresses = frozenset( + [a for a in args.staking_address if not a.startswith("stake")] + ) + if invalid_staking_addresses: + parser.exit( + status=1, + message=f"Following addresses do not look like valid staking addresses: {' '.join(invalid_staking_addresses)}", + ) if not any([args.checkpoint_output, args.csv_output, args.xlsx_output]): parser.exit( status=1, From 108a021cc374f294571de578f91158a574b7ee3a Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Fri, 29 Sep 2023 16:38:21 +0200 Subject: [PATCH 067/124] fix index bug --- .../cardano_account_pandas_dumper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 723cd8b..5a28297 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -436,7 +436,7 @@ def make_balance_frame(self, transactions: pd.Series, detail_level: int): ] if not self.raw_values: balance.columns = pd.MultiIndex.from_tuples( - [(self.asset_names[c[0]], c[2]) for c in balance.columns] + [(self.asset_names[c[0]], c[1]) for c in balance.columns] ) balance.sort_index(axis=1, level=0, sort_remaining=True, inplace=True) From fc7711289204102b397fc156ea522b5aadfbdb3a Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Fri, 29 Sep 2023 16:38:34 +0200 Subject: [PATCH 068/124] comment, cleanup arg parsing --- src/cardano_account_pandas_dumper/__main__.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/cardano_account_pandas_dumper/__main__.py b/src/cardano_account_pandas_dumper/__main__.py index 590d3aa..88612be 100644 --- a/src/cardano_account_pandas_dumper/__main__.py +++ b/src/cardano_account_pandas_dumper/__main__.py @@ -12,6 +12,7 @@ from .cardano_account_pandas_dumper import AccountData, AccountPandasDumper +# Error codes due to project key rate limiting or capping PROJECT_KEY_ERROR_CODES = frozenset([402, 403, 418, 429]) @@ -106,7 +107,7 @@ def main(): if invalid_staking_addresses: parser.exit( status=1, - message=f"Following addresses do not look like valid staking addresses: {' '.join(invalid_staking_addresses)}", + message=f"Following addresses do not look like valid staking addresses: {' '.join(invalid_staking_addresses)}.", ) if not any([args.checkpoint_output, args.csv_output, args.xlsx_output]): parser.exit( @@ -140,7 +141,7 @@ def main(): ), status=1, ) - elif staking_addresses_set: + else: try: api_instance = BlockFrostApi(project_id=args.blockfrost_project_id) data_from_api = AccountData( @@ -173,8 +174,6 @@ def main(): args.checkpoint_output.flush() except (pickle.PicklingError, OSError) as exception: warnings.warn(f"Failed to write checkpoint: {exception}") - else: - parser.exit(status=1, message="Staking address(es) required.") reporter = AccountPandasDumper( data=data_from_api, known_dict=known_dict_from_file, From 8400e3ab601dbb07b11678a896b170008a6277dd Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Sat, 30 Sep 2023 13:03:54 +0200 Subject: [PATCH 069/124] add xlsx --- pyproject.toml | 8 ++++- src/cardano_account_pandas_dumper/__main__.py | 24 ++++++++++----- .../cardano_account_pandas_dumper.py | 29 ++++++++++++++----- 3 files changed, 45 insertions(+), 16 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 0b77a02..2b2d164 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,13 @@ readme = "README.md" requires-python = ">=3.11" license = { file = "LICENSE" } keywords = ["Cardano", "Pandas", "report", "wallet"] -dependencies = ["jstyleson", "pandas", "blockfrost-python", "openpyxl"] +dependencies = [ + "jstyleson", + "pandas", + "blockfrost-python", + "openpyxl", + "types-openpyxl", +] [project.scripts] cardano_account_pandas_dumper = "cardano_account_pandas_dumper.__main__:main" diff --git a/src/cardano_account_pandas_dumper/__main__.py b/src/cardano_account_pandas_dumper/__main__.py index 88612be..9a1d4cc 100644 --- a/src/cardano_account_pandas_dumper/__main__.py +++ b/src/cardano_account_pandas_dumper/__main__.py @@ -9,6 +9,7 @@ import numpy as np import pandas as pd from blockfrost import ApiError, BlockFrostApi +from openpyxl.cell.cell import ILLEGAL_CHARACTERS_RE from .cardano_account_pandas_dumper import AccountData, AccountPandasDumper @@ -107,7 +108,8 @@ def main(): if invalid_staking_addresses: parser.exit( status=1, - message=f"Following addresses do not look like valid staking addresses: {' '.join(invalid_staking_addresses)}.", + message="Following addresses do not look like valid staking addresses: " + + " ".join(invalid_staking_addresses), ) if not any([args.checkpoint_output, args.csv_output, args.xlsx_output]): parser.exit( @@ -191,23 +193,29 @@ def main(): ), ], ).rename("transactions") - dataframe = reporter.make_transaction_frame( - transactions, - detail_level=args.detail_level, - ) if args.csv_output: try: - dataframe.replace(np.float64(0), pd.NA).to_csv(args.csv_output, index=False) + reporter.make_transaction_frame( + transactions, + detail_level=args.detail_level, + ).replace(np.float64(0), pd.NA).to_csv(args.csv_output, index=False) except OSError as exception: warnings.warn(f"Failed to write CSV file: {exception}") if args.xlsx_output: try: - dataframe.replace(np.float64(0), pd.NA).to_excel( + reporter.make_transaction_frame( + transactions, + detail_level=args.detail_level, + text_cleaner=lambda x: ILLEGAL_CHARACTERS_RE.sub( + lambda y: "".join(["\\" + hex(ord(y.group(0))).removeprefix("0")]), + x, + ), + ).replace(np.float64(0), pd.NA).to_excel( args.xlsx_output, index=True, sheet_name=f"Transactions to block {args.to_block}", merge_cells=True, - freeze_panes=(3 if args.raw_values else 2, 3), + freeze_panes=(3 if args.raw_values else 2, 4), ) except OSError as exception: warnings.warn(f"Failed to write .xlsx file: {exception}") diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 5a28297..26eddae 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -3,7 +3,17 @@ import functools import itertools from collections import defaultdict -from typing import Any, Dict, FrozenSet, List, MutableMapping, Optional, Set, Tuple +from typing import ( + Any, + Callable, + Dict, + FrozenSet, + List, + MutableMapping, + Optional, + Set, + Tuple, +) import blockfrost.utils import numpy as np @@ -434,11 +444,6 @@ def make_balance_frame(self, transactions: pd.Series, detail_level: int): np.float_power(10, np.negative(self.asset_decimals[c[0]])) for c in balance.columns ] - if not self.raw_values: - balance.columns = pd.MultiIndex.from_tuples( - [(self.asset_names[c[0]], c[1]) for c in balance.columns] - ) - balance.sort_index(axis=1, level=0, sort_remaining=True, inplace=True) return balance @@ -446,6 +451,7 @@ def make_transaction_frame( self, transactions: pd.Series, detail_level: int, + text_cleaner: Callable = lambda x: x, with_tx_hash: bool = True, with_tx_message: bool = True, with_total: bool = True, @@ -458,9 +464,18 @@ def make_transaction_frame( columns.append(transactions.rename("hash").map(lambda x: x.hash)) total.append("" if with_tx_message else "Total") if with_tx_message: - columns.append(transactions.rename("message").map(self._format_message)) + columns.append( + transactions.rename("message").map( + lambda x: text_cleaner(self._format_message(x)) + ) + ) total.append("Total") balance = self.make_balance_frame(transactions, detail_level) + if not self.raw_values: + balance.columns = pd.MultiIndex.from_tuples( + [(text_cleaner(self.asset_names[c[0]]), c[1]) for c in balance.columns] + ) + balance.sort_index(axis=1, level=0, sort_remaining=True, inplace=True) frame = pd.concat(columns, axis=1) frame.reset_index(drop=True, inplace=True) frame.columns = pd.MultiIndex.from_tuples( From 8f73f92772086870724de64ec7bba47baa89e508 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Sat, 30 Sep 2023 13:31:21 +0200 Subject: [PATCH 070/124] fix escape sequence --- src/cardano_account_pandas_dumper/__main__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/cardano_account_pandas_dumper/__main__.py b/src/cardano_account_pandas_dumper/__main__.py index 9a1d4cc..1d97e77 100644 --- a/src/cardano_account_pandas_dumper/__main__.py +++ b/src/cardano_account_pandas_dumper/__main__.py @@ -207,7 +207,9 @@ def main(): transactions, detail_level=args.detail_level, text_cleaner=lambda x: ILLEGAL_CHARACTERS_RE.sub( - lambda y: "".join(["\\" + hex(ord(y.group(0))).removeprefix("0")]), + lambda y: "".join( + ["\\x0" + hex(ord(y.group(0))).removeprefix("0x")] + ), x, ), ).replace(np.float64(0), pd.NA).to_excel( From f58e0aa9d7af56d7d27cb9bf23674a38fb0a32ce Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Sat, 30 Sep 2023 15:53:53 +0200 Subject: [PATCH 071/124] tweak reset_index --- src/cardano_account_pandas_dumper/__main__.py | 2 +- .../cardano_account_pandas_dumper.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/cardano_account_pandas_dumper/__main__.py b/src/cardano_account_pandas_dumper/__main__.py index 1d97e77..9af1d4d 100644 --- a/src/cardano_account_pandas_dumper/__main__.py +++ b/src/cardano_account_pandas_dumper/__main__.py @@ -212,7 +212,7 @@ def main(): ), x, ), - ).replace(np.float64(0), pd.NA).to_excel( + ).replace(np.float64(0), pd.NA).reset_index(drop=True).to_excel( args.xlsx_output, index=True, sheet_name=f"Transactions to block {args.to_block}", diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 26eddae..16b33a2 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -477,7 +477,6 @@ def make_transaction_frame( ) balance.sort_index(axis=1, level=0, sort_remaining=True, inplace=True) frame = pd.concat(columns, axis=1) - frame.reset_index(drop=True, inplace=True) frame.columns = pd.MultiIndex.from_tuples( [ ("metadata", c) + (len(balance.columns[0]) - 2) * ("",) From f07427468b1575bc9ca8df953b8cec4599a7fb46 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Sat, 30 Sep 2023 16:39:13 +0200 Subject: [PATCH 072/124] add rewards column --- src/cardano_account_pandas_dumper/__main__.py | 2 ++ .../cardano_account_pandas_dumper.py | 8 +++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/cardano_account_pandas_dumper/__main__.py b/src/cardano_account_pandas_dumper/__main__.py index 9af1d4d..9c0fea0 100644 --- a/src/cardano_account_pandas_dumper/__main__.py +++ b/src/cardano_account_pandas_dumper/__main__.py @@ -198,6 +198,7 @@ def main(): reporter.make_transaction_frame( transactions, detail_level=args.detail_level, + with_rewards=not args.no_rewards, ).replace(np.float64(0), pd.NA).to_csv(args.csv_output, index=False) except OSError as exception: warnings.warn(f"Failed to write CSV file: {exception}") @@ -206,6 +207,7 @@ def main(): reporter.make_transaction_frame( transactions, detail_level=args.detail_level, + with_rewards=not args.no_rewards, text_cleaner=lambda x: ILLEGAL_CHARACTERS_RE.sub( lambda y: "".join( ["\\x0" + hex(ord(y.group(0))).removeprefix("0x")] diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 16b33a2..a4510f0 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -451,10 +451,11 @@ def make_transaction_frame( self, transactions: pd.Series, detail_level: int, - text_cleaner: Callable = lambda x: x, + with_rewards: bool, with_tx_hash: bool = True, with_tx_message: bool = True, with_total: bool = True, + text_cleaner: Callable = lambda x: x, ) -> pd.DataFrame: """Build a transaction spreadsheet.""" @@ -462,6 +463,11 @@ def make_transaction_frame( total: List[Any] = [columns[0].max() + self.TRANSACTION_OFFSET] if with_tx_hash: columns.append(transactions.rename("hash").map(lambda x: x.hash)) + total.append("" if (with_tx_message or with_rewards) else "Total") + if with_rewards: + columns.append( + transactions.rename("reward").map(lambda x: bool(x.reward_amount)) + ) total.append("" if with_tx_message else "Total") if with_tx_message: columns.append( From c4502424eff91d257193a4e4e888a03fc8664447 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Sun, 1 Oct 2023 15:45:06 +0200 Subject: [PATCH 073/124] add zero_is_nan --- src/cardano_account_pandas_dumper/__main__.py | 7 +++-- .../cardano_account_pandas_dumper.py | 31 ++++++++++--------- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/src/cardano_account_pandas_dumper/__main__.py b/src/cardano_account_pandas_dumper/__main__.py index 9c0fea0..1793229 100644 --- a/src/cardano_account_pandas_dumper/__main__.py +++ b/src/cardano_account_pandas_dumper/__main__.py @@ -6,7 +6,6 @@ from json import JSONDecodeError import jstyleson -import numpy as np import pandas as pd from blockfrost import ApiError, BlockFrostApi from openpyxl.cell.cell import ILLEGAL_CHARACTERS_RE @@ -199,7 +198,8 @@ def main(): transactions, detail_level=args.detail_level, with_rewards=not args.no_rewards, - ).replace(np.float64(0), pd.NA).to_csv(args.csv_output, index=False) + zero_is_nan=True, + ).to_csv(args.csv_output, index=False) except OSError as exception: warnings.warn(f"Failed to write CSV file: {exception}") if args.xlsx_output: @@ -208,13 +208,14 @@ def main(): transactions, detail_level=args.detail_level, with_rewards=not args.no_rewards, + zero_is_nan=True, text_cleaner=lambda x: ILLEGAL_CHARACTERS_RE.sub( lambda y: "".join( ["\\x0" + hex(ord(y.group(0))).removeprefix("0x")] ), x, ), - ).replace(np.float64(0), pd.NA).reset_index(drop=True).to_excel( + ).reset_index(drop=True).to_excel( args.xlsx_output, index=True, sheet_name=f"Transactions to block {args.to_block}", diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index a4510f0..b3013b2 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -444,38 +444,32 @@ def make_balance_frame(self, transactions: pd.Series, detail_level: int): np.float_power(10, np.negative(self.asset_decimals[c[0]])) for c in balance.columns ] - return balance def make_transaction_frame( self, transactions: pd.Series, detail_level: int, + zero_is_nan: bool, with_rewards: bool, - with_tx_hash: bool = True, - with_tx_message: bool = True, with_total: bool = True, text_cleaner: Callable = lambda x: x, ) -> pd.DataFrame: """Build a transaction spreadsheet.""" columns = [transactions.rename("timestamp").map(self._extract_timestamp)] - total: List[Any] = [columns[0].max() + self.TRANSACTION_OFFSET] - if with_tx_hash: - columns.append(transactions.rename("hash").map(lambda x: x.hash)) - total.append("" if (with_tx_message or with_rewards) else "Total") + columns.append(transactions.rename("hash").map(lambda x: x.hash)) if with_rewards: columns.append( - transactions.rename("reward").map(lambda x: bool(x.reward_amount)) - ) - total.append("" if with_tx_message else "Total") - if with_tx_message: - columns.append( - transactions.rename("message").map( - lambda x: text_cleaner(self._format_message(x)) + transactions.rename("reward").map( + lambda x: "True" if x.reward_amount else "False" ) ) - total.append("Total") + columns.append( + transactions.rename("message").map( + lambda x: text_cleaner(self._format_message(x)) + ) + ) balance = self.make_balance_frame(transactions, detail_level) if not self.raw_values: balance.columns = pd.MultiIndex.from_tuples( @@ -493,9 +487,16 @@ def make_transaction_frame( frame.sort_values(by=frame.columns[0], inplace=True) # Add total line at the bottom if with_total: + total = ( + [columns[0].max() + self.TRANSACTION_OFFSET, ""] + + ([""] if with_rewards else []) + + ["Total"] + ) for column in balance.columns: total.append(balance[column].sum()) frame = pd.concat( [frame, pd.DataFrame(data=[total], columns=frame.columns)] ) + if zero_is_nan: + frame.replace(np.float64(0), pd.NA, inplace=True) return frame From 64e2069b7847e9b4cbe1ef689c3d580972807365 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Mon, 2 Oct 2023 10:05:35 +0200 Subject: [PATCH 074/124] use timestamp index --- src/cardano_account_pandas_dumper/__main__.py | 45 ++++++----- .../cardano_account_pandas_dumper.py | 77 +++++++++++-------- 2 files changed, 69 insertions(+), 53 deletions(-) diff --git a/src/cardano_account_pandas_dumper/__main__.py b/src/cardano_account_pandas_dumper/__main__.py index 1793229..01ab1ce 100644 --- a/src/cardano_account_pandas_dumper/__main__.py +++ b/src/cardano_account_pandas_dumper/__main__.py @@ -181,45 +181,50 @@ def main(): truncate_length=args.truncate_length, raw_values=args.raw_values, unmute=args.unmute, - ) - transactions = pd.concat( - objs=[ - data_from_api.transactions, - pd.Series( - [] + with_rewards=not args.no_rewards, + detail_level=args.detail_level, + ) + transactions = ( + pd.Series( + {reporter.extract_timestamp(t): t for t in data_from_api.transactions} + | ( + {} if args.no_rewards - else [reporter.reward_transaction(r) for r in data_from_api.rewards] - ), - ], - ).rename("transactions") + else { + reporter.extract_timestamp(t): t + for t in [ + reporter.reward_transaction(r) for r in data_from_api.rewards + ] + } + ) + ) + .rename("transactions") + .sort_index() + ) if args.csv_output: try: reporter.make_transaction_frame( transactions, - detail_level=args.detail_level, - with_rewards=not args.no_rewards, - zero_is_nan=True, - ).to_csv(args.csv_output, index=False) + ).to_csv( + args.csv_output, + index_label="Timestamp", + ) except OSError as exception: warnings.warn(f"Failed to write CSV file: {exception}") if args.xlsx_output: try: reporter.make_transaction_frame( transactions, - detail_level=args.detail_level, - with_rewards=not args.no_rewards, - zero_is_nan=True, text_cleaner=lambda x: ILLEGAL_CHARACTERS_RE.sub( lambda y: "".join( ["\\x0" + hex(ord(y.group(0))).removeprefix("0x")] ), x, ), - ).reset_index(drop=True).to_excel( + ).to_excel( args.xlsx_output, - index=True, sheet_name=f"Transactions to block {args.to_block}", - merge_cells=True, + index_label="Timestamp", freeze_panes=(3 if args.raw_values else 2, 4), ) except OSError as exception: diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index b3013b2..c29c4a6 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -155,11 +155,15 @@ def __init__( truncate_length: int, raw_values: bool, unmute: bool, + with_rewards: bool, + detail_level: int, ): self.data = data self.truncate_length = truncate_length self.raw_values = raw_values self.unmute = unmute + self.with_rewards = with_rewards + self.detail_level = detail_level self.address_names = pd.Series( {a: " wallet" for a in self.data.own_addresses} | known_dict.get("addresses", {}) @@ -301,7 +305,8 @@ def _format_policy(self, policy: str) -> Optional[str]: ) @classmethod - def _extract_timestamp(cls, transaction: blockfrost.utils.Namespace) -> Any: + def extract_timestamp(cls, transaction: blockfrost.utils.Namespace) -> Any: + """Returns timestamp of transaction.""" return np.datetime64( datetime.datetime.fromtimestamp(transaction.block_time) ) + (int(transaction.index) * cls.TRANSACTION_OFFSET) @@ -391,7 +396,7 @@ def _transaction_balance(self, transaction: blockfrost.utils.Namespace) -> Any: result[(self.ADA_ASSET, self.OWN_LABEL, " deposit")] = np.longlong( transaction.deposit ) - if transaction.reward_amount: + if self.with_rewards and transaction.reward_amount: result[(self.ADA_ASSET, self.OWN_LABEL, " rewards")] = np.longlong( transaction.reward_amount ) @@ -418,22 +423,27 @@ def _transaction_balance(self, transaction: blockfrost.utils.Namespace) -> Any: return result - def make_balance_frame(self, transactions: pd.Series, detail_level: int): + def make_balance_frame( + self, + transactions: pd.Series, + text_cleaner: Callable = lambda x: x, + ): """Make DataFrame with transaction balances.""" balance = pd.DataFrame( data=[self._transaction_balance(x) for x in transactions], + index=transactions.index, dtype="Int64", ) balance.columns = pd.MultiIndex.from_tuples(balance.columns) balance.sort_index(axis=1, level=0, sort_remaining=True, inplace=True) if not self.unmute: self._drop_muted_assets(balance) - if detail_level == 1: + if self.detail_level == 1: balance.drop(labels=self.OTHER_LABEL, axis=1, level=1, inplace=True) balance = ( balance.T.groupby( level=(0, 2) - if not (self.raw_values and detail_level > 1) + if not (self.raw_values and self.detail_level > 1) else (0, 1, 2) ) .sum(numeric_only=True) @@ -444,39 +454,36 @@ def make_balance_frame(self, transactions: pd.Series, detail_level: int): np.float_power(10, np.negative(self.asset_decimals[c[0]])) for c in balance.columns ] + if not self.raw_values: + balance.columns = pd.MultiIndex.from_tuples( + [(text_cleaner(self.asset_names[c[0]]), c[1]) for c in balance.columns] + ) + balance.sort_index(axis=1, level=0, sort_remaining=True, inplace=True) return balance def make_transaction_frame( self, transactions: pd.Series, - detail_level: int, - zero_is_nan: bool, - with_rewards: bool, + zero_is_nan: bool = True, with_total: bool = True, text_cleaner: Callable = lambda x: x, ) -> pd.DataFrame: """Build a transaction spreadsheet.""" - columns = [transactions.rename("timestamp").map(self._extract_timestamp)] - columns.append(transactions.rename("hash").map(lambda x: x.hash)) - if with_rewards: - columns.append( - transactions.rename("reward").map( - lambda x: "True" if x.reward_amount else "False" + frame = pd.DataFrame( + data=[ + {"hash": x.hash, "message": text_cleaner(self._format_message(x))} + | ( + {} + if not self.with_rewards + else {"reward": "True" if x.reward_amount else "False"} ) - ) - columns.append( - transactions.rename("message").map( - lambda x: text_cleaner(self._format_message(x)) - ) + for x in transactions + ], + index=transactions.index, ) - balance = self.make_balance_frame(transactions, detail_level) - if not self.raw_values: - balance.columns = pd.MultiIndex.from_tuples( - [(text_cleaner(self.asset_names[c[0]]), c[1]) for c in balance.columns] - ) - balance.sort_index(axis=1, level=0, sort_remaining=True, inplace=True) - frame = pd.concat(columns, axis=1) + + balance = self.make_balance_frame(transactions, text_cleaner=text_cleaner) frame.columns = pd.MultiIndex.from_tuples( [ ("metadata", c) + (len(balance.columns[0]) - 2) * ("",) @@ -484,18 +491,22 @@ def make_transaction_frame( ] ) frame = frame.join(balance) - frame.sort_values(by=frame.columns[0], inplace=True) # Add total line at the bottom if with_total: - total = ( - [columns[0].max() + self.TRANSACTION_OFFSET, ""] - + ([""] if with_rewards else []) - + ["Total"] - ) + total = [""] + ([""] if self.with_rewards else []) + ["Total"] for column in balance.columns: total.append(balance[column].sum()) frame = pd.concat( - [frame, pd.DataFrame(data=[total], columns=frame.columns)] + [ + frame, + pd.DataFrame( + data=[total], + columns=frame.columns, + index=[ + frame.index.max() + self.TRANSACTION_OFFSET, + ], + ), + ] ) if zero_is_nan: frame.replace(np.float64(0), pd.NA, inplace=True) From 80986069435b6896087a9712ad88b093345a4de1 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Mon, 2 Oct 2023 10:44:54 +0200 Subject: [PATCH 075/124] fix duplicate timestamps --- src/cardano_account_pandas_dumper/__main__.py | 27 +++++++++---------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/src/cardano_account_pandas_dumper/__main__.py b/src/cardano_account_pandas_dumper/__main__.py index 01ab1ce..3c2eb77 100644 --- a/src/cardano_account_pandas_dumper/__main__.py +++ b/src/cardano_account_pandas_dumper/__main__.py @@ -184,23 +184,20 @@ def main(): with_rewards=not args.no_rewards, detail_level=args.detail_level, ) - transactions = ( - pd.Series( - {reporter.extract_timestamp(t): t for t in data_from_api.transactions} - | ( - {} - if args.no_rewards - else { - reporter.extract_timestamp(t): t - for t in [ - reporter.reward_transaction(r) for r in data_from_api.rewards - ] - } - ) + transactions = pd.concat( + [data_from_api.transactions] + + ( + [] + if args.no_rewards + else [ + pd.Series( + [reporter.reward_transaction(r) for r in data_from_api.rewards] + ) + ] ) - .rename("transactions") - .sort_index() ) + transactions.index = [reporter.extract_timestamp(t) for t in transactions] + transactions.sort_index(inplace=True) if args.csv_output: try: reporter.make_transaction_frame( From 6499f1a3cc26f9c35d753bc0620116e9a406e2e5 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Mon, 2 Oct 2023 10:49:27 +0200 Subject: [PATCH 076/124] remove reward column --- src/cardano_account_pandas_dumper/__main__.py | 1 - .../cardano_account_pandas_dumper.py | 11 ++--------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/src/cardano_account_pandas_dumper/__main__.py b/src/cardano_account_pandas_dumper/__main__.py index 3c2eb77..77eb737 100644 --- a/src/cardano_account_pandas_dumper/__main__.py +++ b/src/cardano_account_pandas_dumper/__main__.py @@ -181,7 +181,6 @@ def main(): truncate_length=args.truncate_length, raw_values=args.raw_values, unmute=args.unmute, - with_rewards=not args.no_rewards, detail_level=args.detail_level, ) transactions = pd.concat( diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index c29c4a6..ca04562 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -155,14 +155,12 @@ def __init__( truncate_length: int, raw_values: bool, unmute: bool, - with_rewards: bool, detail_level: int, ): self.data = data self.truncate_length = truncate_length self.raw_values = raw_values self.unmute = unmute - self.with_rewards = with_rewards self.detail_level = detail_level self.address_names = pd.Series( {a: " wallet" for a in self.data.own_addresses} @@ -396,7 +394,7 @@ def _transaction_balance(self, transaction: blockfrost.utils.Namespace) -> Any: result[(self.ADA_ASSET, self.OWN_LABEL, " deposit")] = np.longlong( transaction.deposit ) - if self.with_rewards and transaction.reward_amount: + if transaction.reward_amount: result[(self.ADA_ASSET, self.OWN_LABEL, " rewards")] = np.longlong( transaction.reward_amount ) @@ -473,11 +471,6 @@ def make_transaction_frame( frame = pd.DataFrame( data=[ {"hash": x.hash, "message": text_cleaner(self._format_message(x))} - | ( - {} - if not self.with_rewards - else {"reward": "True" if x.reward_amount else "False"} - ) for x in transactions ], index=transactions.index, @@ -493,7 +486,7 @@ def make_transaction_frame( frame = frame.join(balance) # Add total line at the bottom if with_total: - total = [""] + ([""] if self.with_rewards else []) + ["Total"] + total = ["", "Total"] for column in balance.columns: total.append(balance[column].sum()) frame = pd.concat( From 8984b9cffbf424e74debd3c629fba7e7f2915fdc Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Wed, 4 Oct 2023 11:25:42 +0200 Subject: [PATCH 077/124] fix freeze panes, better mint truncate --- src/cardano_account_pandas_dumper/__main__.py | 2 +- .../cardano_account_pandas_dumper.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/cardano_account_pandas_dumper/__main__.py b/src/cardano_account_pandas_dumper/__main__.py index 77eb737..fc6094e 100644 --- a/src/cardano_account_pandas_dumper/__main__.py +++ b/src/cardano_account_pandas_dumper/__main__.py @@ -221,7 +221,7 @@ def main(): args.xlsx_output, sheet_name=f"Transactions to block {args.to_block}", index_label="Timestamp", - freeze_panes=(3 if args.raw_values else 2, 4), + freeze_panes=(3 if args.raw_values else 2, 3), ) except OSError as exception: warnings.warn(f"Failed to write .xlsx file: {exception}") diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index ca04562..0695faa 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -246,8 +246,9 @@ def _parse_nft_mint(self, meta: blockfrost.utils.Namespace) -> str: for policy, _v in meta_dict.items(): if policy == "version": continue + result += f"{self._format_policy(policy)}:" for asset_name in _v.to_dict().keys(): - result += f"{self._format_policy(policy)}@{asset_name} " + result += f"{asset_name} " return result def _format_message(self, tx_obj: blockfrost.utils.Namespace) -> str: From 912581d8216d69fca7f74fba9871e55bfed0d6cf Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Wed, 4 Oct 2023 11:44:16 +0200 Subject: [PATCH 078/124] improve redeemer --- .../cardano_account_pandas_dumper.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 0695faa..59c5a9c 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -277,11 +277,17 @@ def _format_message(self, tx_obj: blockfrost.utils.Namespace) -> str: redeemer_scripts: Dict[str, Set] = defaultdict(set) for redeemer in tx_obj.redeemers: if redeemer.purpose == "spend": - redeemer_scripts["Spend:"].add( + redeemer_scripts[redeemer.purpose].add( self._format_script(redeemer.script_hash) ) elif redeemer.purpose == "mint": - redeemer_scripts["Mint:"].add(self._format_policy(redeemer.script_hash)) + redeemer_scripts[redeemer.purpose].add( + self._format_policy(redeemer.script_hash) + ) + else: + redeemer_scripts[redeemer.purpose].add( + self._truncate(redeemer.redeemer_data_hash) + ) for k, redeemer_script in redeemer_scripts.items(): result.extend([k, str(redeemer_script)]) if not result and all( From 5de0ccb7a92081de03d065002db1ce5f4b6b3af1 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Wed, 4 Oct 2023 12:11:46 +0200 Subject: [PATCH 079/124] cleanup frame & series building --- src/cardano_account_pandas_dumper/__main__.py | 21 ++++++-------- .../cardano_account_pandas_dumper.py | 28 +++++++++---------- 2 files changed, 23 insertions(+), 26 deletions(-) diff --git a/src/cardano_account_pandas_dumper/__main__.py b/src/cardano_account_pandas_dumper/__main__.py index fc6094e..c7ab27b 100644 --- a/src/cardano_account_pandas_dumper/__main__.py +++ b/src/cardano_account_pandas_dumper/__main__.py @@ -184,17 +184,16 @@ def main(): detail_level=args.detail_level, ) transactions = pd.concat( - [data_from_api.transactions] - + ( - [] - if args.no_rewards - else [ - pd.Series( - [reporter.reward_transaction(r) for r in data_from_api.rewards] - ) - ] - ) + [ + data_from_api.transactions, + pd.Series( + [] + if args.no_rewards + else [reporter.reward_transaction(r) for r in data_from_api.rewards] + ), + ] ) + transactions.index = [reporter.extract_timestamp(t) for t in transactions] transactions.sort_index(inplace=True) if args.csv_output: @@ -203,7 +202,6 @@ def main(): transactions, ).to_csv( args.csv_output, - index_label="Timestamp", ) except OSError as exception: warnings.warn(f"Failed to write CSV file: {exception}") @@ -220,7 +218,6 @@ def main(): ).to_excel( args.xlsx_output, sheet_name=f"Transactions to block {args.to_block}", - index_label="Timestamp", freeze_panes=(3 if args.raw_values else 2, 3), ) except OSError as exception: diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 59c5a9c..b936cb0 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -475,7 +475,7 @@ def make_transaction_frame( ) -> pd.DataFrame: """Build a transaction spreadsheet.""" - frame = pd.DataFrame( + msg_frame = pd.DataFrame( data=[ {"hash": x.hash, "message": text_cleaner(self._format_message(x))} for x in transactions @@ -483,31 +483,31 @@ def make_transaction_frame( index=transactions.index, ) - balance = self.make_balance_frame(transactions, text_cleaner=text_cleaner) - frame.columns = pd.MultiIndex.from_tuples( + balance_frame = self.make_balance_frame(transactions, text_cleaner=text_cleaner) + msg_frame.columns = pd.MultiIndex.from_tuples( [ - ("metadata", c) + (len(balance.columns[0]) - 2) * ("",) - for c in frame.columns + ("metadata", c) + (len(balance_frame.columns[0]) - 2) * ("",) + for c in msg_frame.columns ] ) - frame = frame.join(balance) + joined_frame = pd.concat(objs=[msg_frame, balance_frame], axis=1) # Add total line at the bottom if with_total: total = ["", "Total"] - for column in balance.columns: - total.append(balance[column].sum()) - frame = pd.concat( + for column in balance_frame.columns: + total.append(balance_frame[column].sum()) + joined_frame = pd.concat( [ - frame, + joined_frame, pd.DataFrame( data=[total], - columns=frame.columns, + columns=joined_frame.columns, index=[ - frame.index.max() + self.TRANSACTION_OFFSET, + joined_frame.index.max() + self.TRANSACTION_OFFSET, ], ), ] ) if zero_is_nan: - frame.replace(np.float64(0), pd.NA, inplace=True) - return frame + joined_frame.replace(np.float64(0), pd.NA, inplace=True) + return joined_frame From c50edfd2bd17cf9b612bba5ae7962777e5bd5b92 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Wed, 4 Oct 2023 19:42:07 +0200 Subject: [PATCH 080/124] consistency check, nft mint parse improvement, fine-tune other/own --- .../cardano_account_pandas_dumper.py | 45 +++++++++++++------ 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index b936cb0..6345184 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -246,9 +246,7 @@ def _parse_nft_mint(self, meta: blockfrost.utils.Namespace) -> str: for policy, _v in meta_dict.items(): if policy == "version": continue - result += f"{self._format_policy(policy)}:" - for asset_name in _v.to_dict().keys(): - result += f"{asset_name} " + result += f"{self._format_policy(policy)}:{_v.to_dict()}" return result def _format_message(self, tx_obj: blockfrost.utils.Namespace) -> str: @@ -369,6 +367,7 @@ def reward_transaction( result.index = -1 result.fees = "0" result.deposit = "0" + result.asset_mint_or_burn_count = 0 result.redeemers = [] result.hash = None result.withdrawals = [] @@ -395,26 +394,26 @@ def _column_key(self, utxo, amount): def _transaction_balance(self, transaction: blockfrost.utils.Namespace) -> Any: result: MutableMapping[Tuple, np.longlong] = defaultdict(lambda: np.longlong(0)) - result[(self.ADA_ASSET, self.OWN_LABEL, " fees")] = np.longlong( + result[(self.ADA_ASSET, self.OTHER_LABEL, " fees")] += np.longlong( transaction.fees ) - result[(self.ADA_ASSET, self.OWN_LABEL, " deposit")] = np.longlong( + result[(self.ADA_ASSET, self.OWN_LABEL, " deposit")] += np.longlong( transaction.deposit ) if transaction.reward_amount: - result[(self.ADA_ASSET, self.OWN_LABEL, " rewards")] = np.longlong( + result[(self.ADA_ASSET, self.OTHER_LABEL, " rewards")] -= np.longlong( + transaction.reward_amount + ) + result[(self.ADA_ASSET, self.OWN_LABEL, " withdrawals")] += np.longlong( transaction.reward_amount ) if transaction.withdrawals: - result[ - (self.ADA_ASSET, self.OWN_LABEL, " rewards withdrawal") - ] = np.negative( - functools.reduce( - np.add, - [np.longlong(w.amount) for w in transaction.withdrawals], - np.longlong(0), - ) + withdrawals = functools.reduce( + np.add, + [np.longlong(w.amount) for w in transaction.withdrawals], + np.longlong(0), ) + result[(self.ADA_ASSET, self.OWN_LABEL, " withdrawals")] -= withdrawals for utxo in transaction.utxos.nonref_inputs: if not utxo.collateral or not transaction.valid_contract: for amount in utxo.amount: @@ -426,6 +425,24 @@ def _transaction_balance(self, transaction: blockfrost.utils.Namespace) -> Any: for amount in utxo.amount: result[self._column_key(utxo, amount)] += np.longlong(amount.quantity) + sum_by_asset: MutableMapping[str, np.longlong] = defaultdict( + lambda: np.longlong(0) + ) + for key, value in result.items(): + if key[0] == self.ADA_DECIMALS or not transaction.asset_mint_or_burn_count: + sum_by_asset[key[0]] += value + + assert all([v == np.longlong(0) for v in sum_by_asset.values()]), ( + f"Unbalanced transaction: {transaction.hash if transaction.hash else '-'} : " + + f"{self._format_message(transaction)} : " + + str( + { + f"{self._format_policy(self.data.assets[k].policy_id)}@{self._decode_asset_name(self.data.assets[k])}": v + for k, v in sum_by_asset.items() + if v != np.longlong(0) + } + ) + ) return result def make_balance_frame( From 0dc40ff6c6beb212afc6c1cfb12345160ef990d1 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Thu, 5 Oct 2023 09:31:41 +0200 Subject: [PATCH 081/124] cleanup flags, move transactions vector to dumper --- src/cardano_account_pandas_dumper/__main__.py | 28 +++-------- .../cardano_account_pandas_dumper.py | 49 ++++++++++++++----- 2 files changed, 42 insertions(+), 35 deletions(-) diff --git a/src/cardano_account_pandas_dumper/__main__.py b/src/cardano_account_pandas_dumper/__main__.py index c7ab27b..b34bdcc 100644 --- a/src/cardano_account_pandas_dumper/__main__.py +++ b/src/cardano_account_pandas_dumper/__main__.py @@ -6,7 +6,6 @@ from json import JSONDecodeError import jstyleson -import pandas as pd from blockfrost import ApiError, BlockFrostApi from openpyxl.cell.cell import ILLEGAL_CHARACTERS_RE @@ -75,7 +74,7 @@ def _create_arg_parser(): ) result.add_argument( "--unmute", - help="Do not mute policies in the mute list and numerical-only metadata.", + help="Do not auto-mute anything, do not use muted policies.", action="store_true", ) result.add_argument( @@ -90,9 +89,10 @@ def _create_arg_parser(): action="store_true", ) result.add_argument( - "--no_rewards", + "--with_rewards", help="Do not add reward transactions.", - action="store_true", + default=True, + type=bool, ) return result @@ -149,7 +149,7 @@ def main(): api=api_instance, staking_addresses=staking_addresses_set, to_block=args.to_block, - include_rewards=not args.no_rewards, + include_rewards=not args.with_rewards, ) except ApiError as api_exception: parser.exit( @@ -183,24 +183,9 @@ def main(): unmute=args.unmute, detail_level=args.detail_level, ) - transactions = pd.concat( - [ - data_from_api.transactions, - pd.Series( - [] - if args.no_rewards - else [reporter.reward_transaction(r) for r in data_from_api.rewards] - ), - ] - ) - - transactions.index = [reporter.extract_timestamp(t) for t in transactions] - transactions.sort_index(inplace=True) if args.csv_output: try: - reporter.make_transaction_frame( - transactions, - ).to_csv( + reporter.make_transaction_frame().to_csv( args.csv_output, ) except OSError as exception: @@ -208,7 +193,6 @@ def main(): if args.xlsx_output: try: reporter.make_transaction_frame( - transactions, text_cleaner=lambda x: ILLEGAL_CHARACTERS_RE.sub( lambda y: "".join( ["\\x0" + hex(ord(y.group(0))).removeprefix("0x")] diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 6345184..5415b01 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -184,6 +184,20 @@ def __init__( self.pinned_policies = pd.Series(known_dict.get("pinned_policies", [])) self.scripts = pd.Series(known_dict.get("scripts", {})) self.labels = pd.Series(known_dict.get("labels", {})) + transactions = pd.concat( + [ + self.data.transactions, + pd.Series([self.reward_transaction(r) for r in self.data.rewards]), + ] + ) + + transactions.index = pd.Index( + [self.extract_timestamp(t) for t in transactions], + dtype="datetime64[ns]", + ) + transactions.sort_index(inplace=True) + assert len(transactions) == len(self.data.transactions) + len(self.data.rewards) + self.transactions = transactions def _truncate(self, value: str) -> str: return ( @@ -308,7 +322,9 @@ def _format_policy(self, policy: str) -> Optional[str]: ) @classmethod - def extract_timestamp(cls, transaction: blockfrost.utils.Namespace) -> Any: + def extract_timestamp( + cls, transaction: blockfrost.utils.Namespace + ) -> np.datetime64: """Returns timestamp of transaction.""" return np.datetime64( datetime.datetime.fromtimestamp(transaction.block_time) @@ -429,15 +445,20 @@ def _transaction_balance(self, transaction: blockfrost.utils.Namespace) -> Any: lambda: np.longlong(0) ) for key, value in result.items(): - if key[0] == self.ADA_DECIMALS or not transaction.asset_mint_or_burn_count: - sum_by_asset[key[0]] += value - - assert all([v == np.longlong(0) for v in sum_by_asset.values()]), ( + sum_by_asset[key[0]] += value + sum_by_asset = {k: v for k, v in sum_by_asset.items() if v} + assert ( + self.ADA_ASSET not in sum_by_asset + and len(sum_by_asset) == transaction.asset_mint_or_burn_count + ), ( f"Unbalanced transaction: {transaction.hash if transaction.hash else '-'} : " + f"{self._format_message(transaction)} : " + str( { - f"{self._format_policy(self.data.assets[k].policy_id)}@{self._decode_asset_name(self.data.assets[k])}": v + ( + f"{self._format_policy(self.data.assets[k].policy_id)}@" + + f"{self._decode_asset_name(self.data.assets[k])}" + ): v for k, v in sum_by_asset.items() if v != np.longlong(0) } @@ -447,13 +468,12 @@ def _transaction_balance(self, transaction: blockfrost.utils.Namespace) -> Any: def make_balance_frame( self, - transactions: pd.Series, text_cleaner: Callable = lambda x: x, ): """Make DataFrame with transaction balances.""" balance = pd.DataFrame( - data=[self._transaction_balance(x) for x in transactions], - index=transactions.index, + data=[self._transaction_balance(x) for x in self.transactions], + index=self.transactions.index, dtype="Int64", ) balance.columns = pd.MultiIndex.from_tuples(balance.columns) @@ -485,7 +505,6 @@ def make_balance_frame( def make_transaction_frame( self, - transactions: pd.Series, zero_is_nan: bool = True, with_total: bool = True, text_cleaner: Callable = lambda x: x, @@ -495,18 +514,22 @@ def make_transaction_frame( msg_frame = pd.DataFrame( data=[ {"hash": x.hash, "message": text_cleaner(self._format_message(x))} - for x in transactions + for x in self.transactions ], - index=transactions.index, + index=self.transactions.index, + dtype="string", ) - balance_frame = self.make_balance_frame(transactions, text_cleaner=text_cleaner) + balance_frame = self.make_balance_frame(text_cleaner=text_cleaner) msg_frame.columns = pd.MultiIndex.from_tuples( [ ("metadata", c) + (len(balance_frame.columns[0]) - 2) * ("",) for c in msg_frame.columns ] ) + assert len(msg_frame) == len( + balance_frame + ), f"Frame lengths do not match {msg_frame=!s} , {balance_frame=!s}" joined_frame = pd.concat(objs=[msg_frame, balance_frame], axis=1) # Add total line at the bottom if with_total: From 3b267d1091392d3c144028a26425ce8e31e501c7 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Thu, 5 Oct 2023 09:37:26 +0200 Subject: [PATCH 082/124] add flags --- src/cardano_account_pandas_dumper/__main__.py | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/cardano_account_pandas_dumper/__main__.py b/src/cardano_account_pandas_dumper/__main__.py index b34bdcc..bfe5afb 100644 --- a/src/cardano_account_pandas_dumper/__main__.py +++ b/src/cardano_account_pandas_dumper/__main__.py @@ -90,7 +90,19 @@ def _create_arg_parser(): ) result.add_argument( "--with_rewards", - help="Do not add reward transactions.", + help="Add synthetic transactions for rewards.", + default=True, + type=bool, + ) + result.add_argument( + "--with_total", + help="Add line with totals for each column at the bottom of the spreadsheet.", + default=True, + type=bool, + ) + result.add_argument( + "--zero_is_nan", + help="Convert zero values to NaN in spreadsheet (in order to display empty cells instead of 0).", default=True, type=bool, ) @@ -185,7 +197,9 @@ def main(): ) if args.csv_output: try: - reporter.make_transaction_frame().to_csv( + reporter.make_transaction_frame( + with_total=args.with_total, zero_is_nan=args.zero_is_nan + ).to_csv( args.csv_output, ) except OSError as exception: @@ -193,6 +207,8 @@ def main(): if args.xlsx_output: try: reporter.make_transaction_frame( + with_total=args.with_total, + zero_is_nan=args.zero_is_nan, text_cleaner=lambda x: ILLEGAL_CHARACTERS_RE.sub( lambda y: "".join( ["\\x0" + hex(ord(y.group(0))).removeprefix("0x")] From 31a86cde50f6461731336cffa34026346aa76b9e Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Thu, 5 Oct 2023 09:38:29 +0200 Subject: [PATCH 083/124] fix inverted flag error --- src/cardano_account_pandas_dumper/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cardano_account_pandas_dumper/__main__.py b/src/cardano_account_pandas_dumper/__main__.py index bfe5afb..9c13dfe 100644 --- a/src/cardano_account_pandas_dumper/__main__.py +++ b/src/cardano_account_pandas_dumper/__main__.py @@ -161,7 +161,7 @@ def main(): api=api_instance, staking_addresses=staking_addresses_set, to_block=args.to_block, - include_rewards=not args.with_rewards, + include_rewards=args.with_rewards, ) except ApiError as api_exception: parser.exit( From 6f54123a71414fed054f8684de75b09692c3ea4c Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Thu, 5 Oct 2023 09:44:40 +0200 Subject: [PATCH 084/124] do not set transaction index in checkpoint --- .../cardano_account_pandas_dumper.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 5415b01..98eff53 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -132,9 +132,7 @@ def _transaction_data( transaction.reward_amount = None result_list.append(transaction) - return pd.Series( - name="Transactions", data={t.hash: t for t in result_list} - ).sort_index() + return pd.Series(name="Transactions", data=result_list) class AccountPandasDumper: From 7ae4f7f4aa35b2ce856420ff7a2f0f0f586919fa Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Thu, 5 Oct 2023 09:55:03 +0200 Subject: [PATCH 085/124] fix line too long --- src/cardano_account_pandas_dumper/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cardano_account_pandas_dumper/__main__.py b/src/cardano_account_pandas_dumper/__main__.py index 9c13dfe..bed83e7 100644 --- a/src/cardano_account_pandas_dumper/__main__.py +++ b/src/cardano_account_pandas_dumper/__main__.py @@ -102,7 +102,7 @@ def _create_arg_parser(): ) result.add_argument( "--zero_is_nan", - help="Convert zero values to NaN in spreadsheet (in order to display empty cells instead of 0).", + help="Convert zero values to NaN in spreadsheet (to display empty cells instead of 0).", default=True, type=bool, ) From 23caf3a0ede0618be6fe64dabda92bba8e8bcdc8 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Thu, 5 Oct 2023 09:55:35 +0200 Subject: [PATCH 086/124] use asset scale instead of decimals, fix metadata column header --- .../cardano_account_pandas_dumper.py | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 98eff53..1990bbc 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -169,11 +169,17 @@ def __init__( {asset.asset: self._decode_asset_name(asset) for asset in self.data.assets} | {self.ADA_ASSET: self.ADA_ASSET} ) - self.asset_decimals = pd.Series( + self.asset_scale = pd.Series( { - asset.asset: np.longlong(asset.metadata.decimals or 0) - if hasattr(asset, "metadata") and hasattr(asset.metadata, "decimals") - else 0 + asset.asset: np.float_power( + 10, + np.negative( + np.longlong(asset.metadata.decimals or 0) + if hasattr(asset, "metadata") + and hasattr(asset.metadata, "decimals") + else 0 + ), + ) for asset in self.data.assets } | {self.ADA_ASSET: self.ADA_DECIMALS} @@ -490,10 +496,7 @@ def make_balance_frame( .T ) - balance = balance * [ - np.float_power(10, np.negative(self.asset_decimals[c[0]])) - for c in balance.columns - ] + balance = balance * [self.asset_scale[c[0]] for c in balance.columns] if not self.raw_values: balance.columns = pd.MultiIndex.from_tuples( [(text_cleaner(self.asset_names[c[0]]), c[1]) for c in balance.columns] @@ -521,7 +524,7 @@ def make_transaction_frame( balance_frame = self.make_balance_frame(text_cleaner=text_cleaner) msg_frame.columns = pd.MultiIndex.from_tuples( [ - ("metadata", c) + (len(balance_frame.columns[0]) - 2) * ("",) + (c,) + (len(balance_frame.columns[0]) - 1) * ("",) for c in msg_frame.columns ] ) From 0c203caae0865169415dc82f9da0d6ea835f177f Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Thu, 5 Oct 2023 10:05:34 +0200 Subject: [PATCH 087/124] refactor checkpoint transaction vector --- .../cardano_account_pandas_dumper.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 1990bbc..7ad5b75 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -83,7 +83,9 @@ def __init__( ) ] ) - self.transactions = self._transaction_data(api) + self.transactions = pd.Series( + name="Transactions", data=self._transaction_data(api) + ) self.assets = pd.Series( name="Assets", data={ @@ -91,7 +93,7 @@ def __init__( for a in frozenset( [ a.unit - for tx_obj in self.transactions # pylint: disable=not-an-iterable + for tx_obj in self.transactions for i in (tx_obj.utxos.inputs + tx_obj.utxos.outputs) for a in i.amount ] @@ -102,7 +104,7 @@ def __init__( def _transaction_data( self, api: BlockFrostApi, - ) -> pd.Series: + ) -> List[blockfrost.utils.Namespace]: result_list = [] for tx_hash in frozenset( [ @@ -132,7 +134,7 @@ def _transaction_data( transaction.reward_amount = None result_list.append(transaction) - return pd.Series(name="Transactions", data=result_list) + return result_list class AccountPandasDumper: From 2d2905415f319eafd1651e658a2ef982e831d834 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Thu, 5 Oct 2023 11:20:01 +0200 Subject: [PATCH 088/124] separate rewards by asset, fix ada scale --- .../cardano_account_pandas_dumper.py | 28 +++++++++++-------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 7ad5b75..bb041bb 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -184,7 +184,7 @@ def __init__( ) for asset in self.data.assets } - | {self.ADA_ASSET: self.ADA_DECIMALS} + | {self.ADA_ASSET: np.float_power(10, np.negative(self.ADA_DECIMALS))} ) self.muted_policies = pd.Series(known_dict.get("muted_policies", [])) self.pinned_policies = pd.Series(known_dict.get("pinned_policies", [])) @@ -382,6 +382,7 @@ def reward_transaction( ) ] result.reward_amount = reward[1].amount + result.reward_address = reward[0] epoch = self.data.epochs[ reward[1].epoch + 1 ] # Time is right before start of next epoch. @@ -426,16 +427,21 @@ def _transaction_balance(self, transaction: blockfrost.utils.Namespace) -> Any: result[(self.ADA_ASSET, self.OTHER_LABEL, " rewards")] -= np.longlong( transaction.reward_amount ) - result[(self.ADA_ASSET, self.OWN_LABEL, " withdrawals")] += np.longlong( - transaction.reward_amount - ) - if transaction.withdrawals: - withdrawals = functools.reduce( - np.add, - [np.longlong(w.amount) for w in transaction.withdrawals], - np.longlong(0), - ) - result[(self.ADA_ASSET, self.OWN_LABEL, " withdrawals")] -= withdrawals + result[ + ( + self.ADA_ASSET, + self.OWN_LABEL, + f" withdrawals-{self._truncate(transaction.reward_address)}", + ) + ] += np.longlong(transaction.reward_amount) + for w in transaction.withdrawals: + result[ + ( + self.ADA_ASSET, + self.OWN_LABEL, + f" withdrawals-{self._truncate(w.address)}", + ) + ] -= np.longlong(w.amount) for utxo in transaction.utxos.nonref_inputs: if not utxo.collateral or not transaction.valid_contract: for amount in utxo.amount: From 17aa80ebb9dfeed3fd9d17dff95f266661157951 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Thu, 5 Oct 2023 15:29:16 +0200 Subject: [PATCH 089/124] scaling/rounding, total construction, switch back to decimals --- .../cardano_account_pandas_dumper.py | 88 +++++++++++-------- 1 file changed, 52 insertions(+), 36 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index bb041bb..61648e6 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -1,6 +1,5 @@ """ Cardano Account To Pandas Dumper.""" import datetime -import functools import itertools from collections import defaultdict from typing import ( @@ -171,20 +170,14 @@ def __init__( {asset.asset: self._decode_asset_name(asset) for asset in self.data.assets} | {self.ADA_ASSET: self.ADA_ASSET} ) - self.asset_scale = pd.Series( + self.asset_decimals = pd.Series( { - asset.asset: np.float_power( - 10, - np.negative( - np.longlong(asset.metadata.decimals or 0) - if hasattr(asset, "metadata") - and hasattr(asset.metadata, "decimals") - else 0 - ), - ) + asset.asset: np.longlong(asset.metadata.decimals or 0) + if hasattr(asset, "metadata") and hasattr(asset.metadata, "decimals") + else 0 for asset in self.data.assets } - | {self.ADA_ASSET: np.float_power(10, np.negative(self.ADA_DECIMALS))} + | {self.ADA_ASSET: self.ADA_DECIMALS} ) self.muted_policies = pd.Series(known_dict.get("muted_policies", [])) self.pinned_policies = pd.Series(known_dict.get("pinned_policies", [])) @@ -434,14 +427,14 @@ def _transaction_balance(self, transaction: blockfrost.utils.Namespace) -> Any: f" withdrawals-{self._truncate(transaction.reward_address)}", ) ] += np.longlong(transaction.reward_amount) - for w in transaction.withdrawals: + for _w in transaction.withdrawals: result[ ( self.ADA_ASSET, self.OWN_LABEL, - f" withdrawals-{self._truncate(w.address)}", + f" withdrawals-{self._truncate(_w.address)}", ) - ] -= np.longlong(w.amount) + ] -= np.longlong(_w.amount) for utxo in transaction.utxos.nonref_inputs: if not utxo.collateral or not transaction.valid_contract: for amount in utxo.amount: @@ -480,6 +473,7 @@ def _transaction_balance(self, transaction: blockfrost.utils.Namespace) -> Any: def make_balance_frame( self, + with_total: bool, text_cleaner: Callable = lambda x: x, ): """Make DataFrame with transaction balances.""" @@ -504,12 +498,37 @@ def make_balance_frame( .T ) - balance = balance * [self.asset_scale[c[0]] for c in balance.columns] + # Scale by asset decimals + balance = balance * [ + np.float_power( + 10, + np.negative(self.asset_decimals[c[0]]), + ) + for c in balance.columns + ] + if with_total: + balance = pd.concat( + [ + balance, + pd.DataFrame( + data=[[balance[column].sum() for column in balance.columns]], + columns=balance.columns, + index=[ + balance.index.max() + self.TRANSACTION_OFFSET, + ], + ), + ] + ) + balance = pd.concat( + [balance[c].round(self.asset_decimals[c[0]]) for c in balance.columns], + axis=1, + ) + if not self.raw_values: balance.columns = pd.MultiIndex.from_tuples( [(text_cleaner(self.asset_names[c[0]]), c[1]) for c in balance.columns] ) - balance.sort_index(axis=1, level=0, sort_remaining=True, inplace=True) + balance.sort_index(axis=1, level=0, sort_remaining=True, inplace=True) return balance def make_transaction_frame( @@ -528,8 +547,22 @@ def make_transaction_frame( index=self.transactions.index, dtype="string", ) - - balance_frame = self.make_balance_frame(text_cleaner=text_cleaner) + if with_total: + msg_frame = pd.concat( + [ + msg_frame, + pd.DataFrame( + data=[["", "Total"]], + columns=msg_frame.columns, + index=[ + msg_frame.index.max() + self.TRANSACTION_OFFSET, + ], + ), + ] + ) + balance_frame = self.make_balance_frame( + with_total=with_total, text_cleaner=text_cleaner + ) msg_frame.columns = pd.MultiIndex.from_tuples( [ (c,) + (len(balance_frame.columns[0]) - 1) * ("",) @@ -540,23 +573,6 @@ def make_transaction_frame( balance_frame ), f"Frame lengths do not match {msg_frame=!s} , {balance_frame=!s}" joined_frame = pd.concat(objs=[msg_frame, balance_frame], axis=1) - # Add total line at the bottom - if with_total: - total = ["", "Total"] - for column in balance_frame.columns: - total.append(balance_frame[column].sum()) - joined_frame = pd.concat( - [ - joined_frame, - pd.DataFrame( - data=[total], - columns=joined_frame.columns, - index=[ - joined_frame.index.max() + self.TRANSACTION_OFFSET, - ], - ), - ] - ) if zero_is_nan: joined_frame.replace(np.float64(0), pd.NA, inplace=True) return joined_frame From fa7780cdc40b3e1d3b024c69143e920ee5a20dad Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Thu, 5 Oct 2023 15:40:29 +0200 Subject: [PATCH 090/124] revert nft mint change, combine scaling and rounding --- .../cardano_account_pandas_dumper.py | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 61648e6..95a676f 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -259,7 +259,9 @@ def _parse_nft_mint(self, meta: blockfrost.utils.Namespace) -> str: for policy, _v in meta_dict.items(): if policy == "version": continue - result += f"{self._format_policy(policy)}:{_v.to_dict()}" + result += f"{self._format_policy(policy)}:" + for asset_name in _v.to_dict().keys(): + result += f"{asset_name} " return result def _format_message(self, tx_obj: blockfrost.utils.Namespace) -> str: @@ -497,15 +499,6 @@ def make_balance_frame( .sum(numeric_only=True) .T ) - - # Scale by asset decimals - balance = balance * [ - np.float_power( - 10, - np.negative(self.asset_decimals[c[0]]), - ) - for c in balance.columns - ] if with_total: balance = pd.concat( [ @@ -519,8 +512,19 @@ def make_balance_frame( ), ] ) + balance = pd.concat( - [balance[c].round(self.asset_decimals[c[0]]) for c in balance.columns], + [ + balance[c] + .mul( + np.float_power( + 10, + np.negative(self.asset_decimals[c[0]]), + ) + ) + .round(self.asset_decimals[c[0]]) + for c in balance.columns + ], axis=1, ) From e47467d2c2d24fe959692e9a3b1108d630a6f17e Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Mon, 9 Oct 2023 10:35:47 +0200 Subject: [PATCH 091/124] fix freeze frames --- src/cardano_account_pandas_dumper/__main__.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/cardano_account_pandas_dumper/__main__.py b/src/cardano_account_pandas_dumper/__main__.py index bed83e7..0be6a4b 100644 --- a/src/cardano_account_pandas_dumper/__main__.py +++ b/src/cardano_account_pandas_dumper/__main__.py @@ -206,7 +206,7 @@ def main(): warnings.warn(f"Failed to write CSV file: {exception}") if args.xlsx_output: try: - reporter.make_transaction_frame( + frame = reporter.make_transaction_frame( with_total=args.with_total, zero_is_nan=args.zero_is_nan, text_cleaner=lambda x: ILLEGAL_CHARACTERS_RE.sub( @@ -215,10 +215,16 @@ def main(): ), x, ), - ).to_excel( + ) + frame.to_excel( args.xlsx_output, sheet_name=f"Transactions to block {args.to_block}", - freeze_panes=(3 if args.raw_values else 2, 3), + freeze_panes=( + len(frame.columns[0]) + 1 + if isinstance(type(frame.columns[0]), tuple) + else 2, + 3, + ), ) except OSError as exception: warnings.warn(f"Failed to write .xlsx file: {exception}") From 781f47c9886a8556f03876564a3773b4aa081e0c Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Mon, 9 Oct 2023 12:18:09 +0200 Subject: [PATCH 092/124] fix detail level 1 --- .../cardano_account_pandas_dumper.py | 41 +++++++++++-------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 95a676f..6113b4b 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -12,6 +12,7 @@ Optional, Set, Tuple, + cast, ) import blockfrost.utils @@ -488,17 +489,14 @@ def make_balance_frame( balance.sort_index(axis=1, level=0, sort_remaining=True, inplace=True) if not self.unmute: self._drop_muted_assets(balance) + if self.detail_level == 1: - balance.drop(labels=self.OTHER_LABEL, axis=1, level=1, inplace=True) - balance = ( - balance.T.groupby( - level=(0, 2) - if not (self.raw_values and self.detail_level > 1) - else (0, 1, 2) - ) - .sum(numeric_only=True) - .T - ) + group: Tuple = (0, 1) + elif self.raw_values: + group = (0, 1, 2) + else: + group = (0, 2) + balance = balance.T.groupby(level=group).sum(numeric_only=True).T if with_total: balance = pd.concat( [ @@ -530,10 +528,16 @@ def make_balance_frame( if not self.raw_values: balance.columns = pd.MultiIndex.from_tuples( - [(text_cleaner(self.asset_names[c[0]]), c[1]) for c in balance.columns] + [ + (text_cleaner(self.asset_names[c[0]]),) + cast(tuple, c)[1:] + for c in balance.columns + ] ) balance.sort_index(axis=1, level=0, sort_remaining=True, inplace=True) - return balance + if self.detail_level == 1: + return balance.xs(self.OWN_LABEL, level=1, axis=1) + else: + return balance def make_transaction_frame( self, @@ -567,12 +571,13 @@ def make_transaction_frame( balance_frame = self.make_balance_frame( with_total=with_total, text_cleaner=text_cleaner ) - msg_frame.columns = pd.MultiIndex.from_tuples( - [ - (c,) + (len(balance_frame.columns[0]) - 1) * ("",) - for c in msg_frame.columns - ] - ) + if self.detail_level > 1: + msg_frame.columns = pd.MultiIndex.from_tuples( + [ + (c,) + (len(balance_frame.columns[0]) - 1) * ("",) + for c in msg_frame.columns + ] + ) assert len(msg_frame) == len( balance_frame ), f"Frame lengths do not match {msg_frame=!s} , {balance_frame=!s}" From 53e74f96af541833b71faa6a07e6be519d217744 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Mon, 9 Oct 2023 12:46:29 +0200 Subject: [PATCH 093/124] first version with plot --- pyproject.toml | 16 +++++++++++++--- src/cardano_account_pandas_dumper/__main__.py | 14 ++++++++++++-- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2b2d164..fbb647b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,12 +5,22 @@ description = "Create a spreadsheet with the owned amount of any Cardano asset a readme = "README.md" requires-python = ">=3.11" license = { file = "LICENSE" } -keywords = ["Cardano", "Pandas", "report", "wallet"] +keywords = [ + "asset", + "balance", + "Cardano", + "graphic", + "Pandas", + "report", + "transaction", + "wallet", +] dependencies = [ - "jstyleson", - "pandas", "blockfrost-python", + "jstyleson", + "matplotlib", "openpyxl", + "pandas", "types-openpyxl", ] diff --git a/src/cardano_account_pandas_dumper/__main__.py b/src/cardano_account_pandas_dumper/__main__.py index 0be6a4b..0ea74be 100644 --- a/src/cardano_account_pandas_dumper/__main__.py +++ b/src/cardano_account_pandas_dumper/__main__.py @@ -6,6 +6,7 @@ from json import JSONDecodeError import jstyleson +import matplotlib.pyplot as plt from blockfrost import ApiError, BlockFrostApi from openpyxl.cell.cell import ILLEGAL_CHARACTERS_RE @@ -66,6 +67,11 @@ def _create_arg_parser(): help="Path to CSV output file.", type=argparse.FileType("wb"), ) + result.add_argument( + "--plot", + help="Draw a plot of balance over time.", + action="store_true", + ) result.add_argument( "--detail_level", help="Level of detail of report (1=only own addresses, 2=other addresses as well).", @@ -122,11 +128,11 @@ def main(): message="Following addresses do not look like valid staking addresses: " + " ".join(invalid_staking_addresses), ) - if not any([args.checkpoint_output, args.csv_output, args.xlsx_output]): + if not any([args.checkpoint_output, args.csv_output, args.xlsx_output, args.plot]): parser.exit( status=1, message="No output specified, neeed at least one of --checkpoint_output," - + " --csv_output, --xlsx_output.\n", + + " --csv_output, --xlsx_output, --plot.\n", ) known_dict_from_file = jstyleson.load(args.known_file) if args.known_file else {} staking_addresses_set = frozenset(args.staking_address) @@ -228,6 +234,10 @@ def main(): ) except OSError as exception: warnings.warn(f"Failed to write .xlsx file: {exception}") + if args.plot: + plt.figure(layout="constrained") + balance = reporter.make_balance_frame(with_total=False).cumsum() + balance.plot(logy=True) print("Done.") From a510fd64a693606523019ba70b418525ffdff7af Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Mon, 9 Oct 2023 17:31:52 +0200 Subject: [PATCH 094/124] fix NA handling, improve graph --- src/cardano_account_pandas_dumper/__main__.py | 15 ++++++--------- .../cardano_account_pandas_dumper.py | 4 +--- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/src/cardano_account_pandas_dumper/__main__.py b/src/cardano_account_pandas_dumper/__main__.py index 0ea74be..ae18638 100644 --- a/src/cardano_account_pandas_dumper/__main__.py +++ b/src/cardano_account_pandas_dumper/__main__.py @@ -9,7 +9,6 @@ import matplotlib.pyplot as plt from blockfrost import ApiError, BlockFrostApi from openpyxl.cell.cell import ILLEGAL_CHARACTERS_RE - from .cardano_account_pandas_dumper import AccountData, AccountPandasDumper # Error codes due to project key rate limiting or capping @@ -106,12 +105,6 @@ def _create_arg_parser(): default=True, type=bool, ) - result.add_argument( - "--zero_is_nan", - help="Convert zero values to NaN in spreadsheet (to display empty cells instead of 0).", - default=True, - type=bool, - ) return result @@ -214,7 +207,6 @@ def main(): try: frame = reporter.make_transaction_frame( with_total=args.with_total, - zero_is_nan=args.zero_is_nan, text_cleaner=lambda x: ILLEGAL_CHARACTERS_RE.sub( lambda y: "".join( ["\\x0" + hex(ord(y.group(0))).removeprefix("0x")] @@ -235,9 +227,14 @@ def main(): except OSError as exception: warnings.warn(f"Failed to write .xlsx file: {exception}") if args.plot: - plt.figure(layout="constrained") balance = reporter.make_balance_frame(with_total=False).cumsum() balance.plot(logy=True) + plt.legend( + bbox_to_anchor=(1, 1), + fontsize="small", + ) + plt.suptitle(f"Asset balances until block {args.to_block}.") + plt.show() print("Done.") diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 6113b4b..5eabd46 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -497,6 +497,7 @@ def make_balance_frame( else: group = (0, 2) balance = balance.T.groupby(level=group).sum(numeric_only=True).T + balance[balance == 0] = pd.NA if with_total: balance = pd.concat( [ @@ -541,7 +542,6 @@ def make_balance_frame( def make_transaction_frame( self, - zero_is_nan: bool = True, with_total: bool = True, text_cleaner: Callable = lambda x: x, ) -> pd.DataFrame: @@ -582,6 +582,4 @@ def make_transaction_frame( balance_frame ), f"Frame lengths do not match {msg_frame=!s} , {balance_frame=!s}" joined_frame = pd.concat(objs=[msg_frame, balance_frame], axis=1) - if zero_is_nan: - joined_frame.replace(np.float64(0), pd.NA, inplace=True) return joined_frame From 3b7541df8d31d0c6dbd712161460924328f7d878 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Tue, 10 Oct 2023 11:03:57 +0200 Subject: [PATCH 095/124] raw_values = param, add legend --- src/cardano_account_pandas_dumper/__main__.py | 16 +++++-- .../cardano_account_pandas_dumper.py | 45 +++++++++++-------- 2 files changed, 39 insertions(+), 22 deletions(-) diff --git a/src/cardano_account_pandas_dumper/__main__.py b/src/cardano_account_pandas_dumper/__main__.py index ae18638..a6bebf5 100644 --- a/src/cardano_account_pandas_dumper/__main__.py +++ b/src/cardano_account_pandas_dumper/__main__.py @@ -6,6 +6,7 @@ from json import JSONDecodeError import jstyleson +from matplotlib.patches import Rectangle import matplotlib.pyplot as plt from blockfrost import ApiError, BlockFrostApi from openpyxl.cell.cell import ILLEGAL_CHARACTERS_RE @@ -190,14 +191,14 @@ def main(): data=data_from_api, known_dict=known_dict_from_file, truncate_length=args.truncate_length, - raw_values=args.raw_values, unmute=args.unmute, detail_level=args.detail_level, ) if args.csv_output: try: reporter.make_transaction_frame( - with_total=args.with_total, zero_is_nan=args.zero_is_nan + with_total=args.with_total, + raw_values=args.raw_values, ).to_csv( args.csv_output, ) @@ -213,6 +214,7 @@ def main(): ), x, ), + raw_values=args.raw_values, ) frame.to_excel( args.xlsx_output, @@ -227,9 +229,17 @@ def main(): except OSError as exception: warnings.warn(f"Failed to write .xlsx file: {exception}") if args.plot: - balance = reporter.make_balance_frame(with_total=False).cumsum() + balance = reporter.make_balance_frame( + with_total=False, raw_values=True + ).cumsum() balance.plot(logy=True) + plt.legend( + [ + Rectangle(xy=(0, 0), width=10, height=10, color=(0.9, 0.9, 0.7)) + for l in balance.columns + ], + [reporter.asset_names.get(c, c) for c in balance.columns], bbox_to_anchor=(1, 1), fontsize="small", ) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 5eabd46..5d4fea1 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -153,13 +153,11 @@ def __init__( data: AccountData, known_dict: Any, truncate_length: int, - raw_values: bool, unmute: bool, detail_level: int, ): self.data = data self.truncate_length = truncate_length - self.raw_values = raw_values self.unmute = unmute self.detail_level = detail_level self.address_names = pd.Series( @@ -169,7 +167,6 @@ def __init__( self.policy_names = pd.Series(known_dict.get("policies", {})) self.asset_names = pd.Series( {asset.asset: self._decode_asset_name(asset) for asset in self.data.assets} - | {self.ADA_ASSET: self.ADA_ASSET} ) self.asset_decimals = pd.Series( { @@ -314,14 +311,10 @@ def _format_message(self, tx_obj: blockfrost.utils.Namespace) -> str: return " ".join(result) def _format_script(self, script: str) -> str: - return ({} if self.raw_values else self.scripts).get( - script, self._truncate(script) - ) + return self.scripts.get(script, self._truncate(script)) def _format_policy(self, policy: str) -> Optional[str]: - return ({} if self.raw_values else self.policy_names).get( - policy, self._truncate(policy) - ) + return self.policy_names.get(policy, self._truncate(policy)) @classmethod def extract_timestamp( @@ -396,7 +389,12 @@ def reward_transaction( result.utxos.nonref_inputs = [] return result - def _column_key(self, utxo, amount): + def _column_key( + self, + utxo, + amount, + raw_values: bool, + ): # Index: (asset_id, own, address_name) return ( amount.unit if amount.unit != self.data.LOVELACE_ASSET else self.ADA_ASSET, @@ -404,14 +402,18 @@ def _column_key(self, utxo, amount): if utxo.address in self.data.own_addresses else self.OTHER_LABEL, self._truncate(utxo.address) - if self.raw_values + if raw_values else self.address_names.get( utxo.address, self.OTHER_LABEL, ), ) - def _transaction_balance(self, transaction: blockfrost.utils.Namespace) -> Any: + def _transaction_balance( + self, + transaction: blockfrost.utils.Namespace, + raw_values: bool, + ) -> Any: result: MutableMapping[Tuple, np.longlong] = defaultdict(lambda: np.longlong(0)) result[(self.ADA_ASSET, self.OTHER_LABEL, " fees")] += np.longlong( transaction.fees @@ -441,13 +443,15 @@ def _transaction_balance(self, transaction: blockfrost.utils.Namespace) -> Any: for utxo in transaction.utxos.nonref_inputs: if not utxo.collateral or not transaction.valid_contract: for amount in utxo.amount: - result[self._column_key(utxo, amount)] -= np.longlong( + result[self._column_key(utxo, amount, raw_values)] -= np.longlong( amount.quantity ) for utxo in transaction.utxos.outputs: for amount in utxo.amount: - result[self._column_key(utxo, amount)] += np.longlong(amount.quantity) + result[self._column_key(utxo, amount, raw_values)] += np.longlong( + amount.quantity + ) sum_by_asset: MutableMapping[str, np.longlong] = defaultdict( lambda: np.longlong(0) @@ -477,11 +481,12 @@ def _transaction_balance(self, transaction: blockfrost.utils.Namespace) -> Any: def make_balance_frame( self, with_total: bool, + raw_values: bool, text_cleaner: Callable = lambda x: x, ): """Make DataFrame with transaction balances.""" balance = pd.DataFrame( - data=[self._transaction_balance(x) for x in self.transactions], + data=[self._transaction_balance(x, raw_values) for x in self.transactions], index=self.transactions.index, dtype="Int64", ) @@ -492,7 +497,7 @@ def make_balance_frame( if self.detail_level == 1: group: Tuple = (0, 1) - elif self.raw_values: + elif raw_values: group = (0, 1, 2) else: group = (0, 2) @@ -527,10 +532,11 @@ def make_balance_frame( axis=1, ) - if not self.raw_values: + if not raw_values: balance.columns = pd.MultiIndex.from_tuples( [ - (text_cleaner(self.asset_names[c[0]]),) + cast(tuple, c)[1:] + (text_cleaner(self.asset_names.get(c[0], c[0])),) + + cast(tuple, c)[1:] for c in balance.columns ] ) @@ -542,6 +548,7 @@ def make_balance_frame( def make_transaction_frame( self, + raw_values: bool, with_total: bool = True, text_cleaner: Callable = lambda x: x, ) -> pd.DataFrame: @@ -569,7 +576,7 @@ def make_transaction_frame( ] ) balance_frame = self.make_balance_frame( - with_total=with_total, text_cleaner=text_cleaner + with_total=with_total, text_cleaner=text_cleaner, raw_values=raw_values ) if self.detail_level > 1: msg_frame.columns = pd.MultiIndex.from_tuples( From 0786f09f8eb0bc7662352e1774e187430f0e462a Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Wed, 11 Oct 2023 13:01:49 +0200 Subject: [PATCH 096/124] move graph to reporter, add ADA logo --- pyproject.toml | 3 +- src/cardano_account_pandas_dumper/__main__.py | 21 ++------ .../ada_logo.webp | Bin 0 -> 8330 bytes .../cardano_account_pandas_dumper.py | 46 ++++++++++++++++++ 4 files changed, 51 insertions(+), 19 deletions(-) create mode 100644 src/cardano_account_pandas_dumper/ada_logo.webp diff --git a/pyproject.toml b/pyproject.toml index fbb647b..f16bb50 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,7 @@ dependencies = [ "openpyxl", "pandas", "types-openpyxl", + "types-Pillow", ] [project.scripts] @@ -31,7 +32,7 @@ cardano_account_pandas_dumper = "cardano_account_pandas_dumper.__main__:main" where = ["src"] [tool.setuptools.package-data] -cardano_account_pandas_dumper = ["*.jsonc"] +cardano_account_pandas_dumper = ["*.jsonc", "*.webp"] [tool.mypy] follow_imports = "normal" diff --git a/src/cardano_account_pandas_dumper/__main__.py b/src/cardano_account_pandas_dumper/__main__.py index a6bebf5..76c6625 100644 --- a/src/cardano_account_pandas_dumper/__main__.py +++ b/src/cardano_account_pandas_dumper/__main__.py @@ -6,8 +6,7 @@ from json import JSONDecodeError import jstyleson -from matplotlib.patches import Rectangle -import matplotlib.pyplot as plt +import matplotlib as mpl from blockfrost import ApiError, BlockFrostApi from openpyxl.cell.cell import ILLEGAL_CHARACTERS_RE from .cardano_account_pandas_dumper import AccountData, AccountPandasDumper @@ -229,22 +228,8 @@ def main(): except OSError as exception: warnings.warn(f"Failed to write .xlsx file: {exception}") if args.plot: - balance = reporter.make_balance_frame( - with_total=False, raw_values=True - ).cumsum() - balance.plot(logy=True) - - plt.legend( - [ - Rectangle(xy=(0, 0), width=10, height=10, color=(0.9, 0.9, 0.7)) - for l in balance.columns - ], - [reporter.asset_names.get(c, c) for c in balance.columns], - bbox_to_anchor=(1, 1), - fontsize="small", - ) - plt.suptitle(f"Asset balances until block {args.to_block}.") - plt.show() + reporter.plot_balance() + mpl.pyplot.show() print("Done.") diff --git a/src/cardano_account_pandas_dumper/ada_logo.webp b/src/cardano_account_pandas_dumper/ada_logo.webp new file mode 100644 index 0000000000000000000000000000000000000000..be149031daafbed464650dd1fd8d5fb83919e7f2 GIT binary patch literal 8330 zcmYjW1yCJ2vp%?c(c;3=x=Dkdk{j#(3 z&1|xp>`c^TrKCbh0RU}DF%>Np9vwsg0D$**FhKm%MHN(JQV{?Ei2CsNo?@-Vo}4Xv ziDGArz6hs4W#5Z0cc0MrFq7AZB#*`tpx4)PcOr;kxDg%oix2xG(O721MUDz^LPQ`j zF_T4VPEau61vQc(Djf>BHN|Gt!I}~HeT^&^k1}!%iI%@&W>3*YE=`;TQc4}N&J3tO!7oM-F`h;u#C?aFoZ z$a+hvc`^$EWFx@}b+pbTVQ~?Dpr!utvKtjjt-}iV9`80Zq5* z1qorQjHM6ptlLsS(LELow5>n0;tHs#nyuGjmjshNO2_X?*L@X{^|T+p_oTrmT@j3O z>He945*2X+k|Q{nTq zBmVImGzvf(#T~t3$t>r2oNK!ILIn|hH)9 zQyxjl#NvI#*Tc75sLL~cVleY5mcE}jbca65} z3YiP(sf&2@w#WM*u7QXZJ9$|YJT_FdiE2@^NEF$W-@}ru6DLnOZnv8j-2=y5Xj#5fzMu$EFc$`oaFgvo{Le z1}>qv-*#%m@_>VHl3mk3?j4t%=+8$IUd>FV$(S)#NK+~xq2SR z<(Q7xnC6xZFS7>P=4QtcIXWaB5sS}s9Vup!1tPO^JE**6+OPjMhF@(XD>aQK=wxOe zOg1^_2u&dK-0sMZBXc~@p3Oj(>_Ik=MRqn%Zu{Cfm@=`R*7-^L=pc>TfY4qiQFYX9Thf{ZI)HCr#CfDUZI`P zY<#)@lU*nsmc9#cD9dEr{vX%?93n6qf5KE}W&!StjeH6SrVE2o;5{;ttBa%i(-KGH z=(pB~W!0&09iEPaldBUkf-JyFJn@FBe5bI3r<{m2N1S^mX#%J039w~OZ{rO1Q)kTD zfKnOOVbcPzi8JC`0!A^?2{Hcq$@r`W8T_x;%f&om+;w}*E_-ZVRII%zqC0X&@`TT9 z?l88@k!_r9EyiZi25tZDXJ#y!?HQl7u?&>sdj1pZN*f*fV3Nr}=@D_@T_LyV)DUVb zteX($aN@Q4Eu~V!E(a@`dwmg<8!k;7E8TQ!>272hM<(Z|}2=>5&n)JA`-V4xvTYsc54Cy)9Df_?%`TdiPXv9k8g=N$r=;xH> zgvGvJ+vyapN(^UAI{X;z-#D*)T^JpA$-RZ)+E8CBqbp=}7Dgg>IQ@~eTJ2-fH6yVTJqR0I`5H7Kb+H*VBNqZ?aOuEgI`^HW zJXu~Q`{HX@s@e8YT4?{?O;_UWG*W{2Z>)ZdVk5d&@{+hD1`r>h+p42CU%0nBA9ef5 z!#-_sO?mTb+fgIZn_e8;S5fFUF0H)i@nT}A4=P1^>L%IVB-yxFcn6FttNFqmQ9bCl zYz~@2RvXN#OPxPBNS(w`!buB^DfPCjSC7sfl&D3vcyEUE2GRYJvY`ORBh!3bQaqB= zIFS!l`0=yc0l#TOoKW_ahUe)EN-~)-Fvd*61s1$kMfvr2tyEHHH(@^igSX5uQl%C~ z;-6n+;SWY{`GSsTq)F>ZP86SQSN-5a1+E{clnuIn1RH1EL@~3pNBJN?3NLUxkJ^_8 z!-+?TM+M~!h{5x)E&#nptc!z5O3Hjwt*kHU5Zc(fORnQuu;DHyTTp~pS9P~kWmlS? z239pGw=^2tm7hZ|a1osCLF;)p15HMWP>17xpG*=3wt`)9jb0zG-mrz2Lxk;Jnqfu!M27i#P-Efne@WbVw$^cU$b&uX) z>IO^xRLd25Q1fe^uZQ;M&Z@)2=0DnbU#ZYH{#$s|EHq4`dD6!cLb_0A9k%cZ+eV)I zjy0p4k4!JR;X{%WE4~XF*|;zCIh(6mx%~fnM)WQwN0V!Otwy<9^VRC*xh~QqJM#Lp zJ#bOG2VC52u%Xh(20;w{w^}2G#Q^}W%mF_j=oujxA%q;b(q$+q$jPYYJ7j`UVoV+1 zte$L!f`+qSg4Di!P=HS#USEQ~-XmO+tOZ&2Fba88^{fPade?Oq&bw6TdVS~qGt!Hu z5;Pbz@iF}N@d6DN`-phgdwnz(1_sT2%)T%B4f*oFAD=*gxxpD9@BK+4dF#qbEGs?l zB-BCeMr}QS&9RTJkLQoH3y6=8)z>FD^&ZKdia=U#{CDn`yj`#4fPyKiouu>D6XDk9 z&JE!mqi$b_K;E~rQKk!C?ZVZ&ubtGF5ON^5d=K`Z|jpJly4;#R%xzI%N@cLQaNPTqi!z9n@(&O zC4g|<@TeiJ1t$=UD_qc~JX`g5X+nYTyqXdSK4Vc4!Kvzl?9AN-sd32P9SLE%gGN%f z6fvXr(Iy$tT3NqYW~v(VK}Pl1d|<((5)CzstKw%pCu}J49z?(}w0bKsTZ7Aqu$#q^ zxIzgwt4Q<*7jS*Cox6vr+_DPD<=>1FO^{f;P@&Y64AGJiSS}OM$RL$BnHRwdDJ~c6 zPqLn!Hwk?jEPbI^}FL*0Q z*l#I`CAE>igui)zqKVlZm^Y;d<4i>Y>kwJ37tC63801_UvLi!e>%N#9y|PH;E%hLzdXsepNL`K#EA7PN`#$Yp5o*IoJx0NAMUCkhf~& zs$Ug!pu%cFXCzc`S~*!uD>Ie373;C;;gGu9N`lSA4!KtGxA(5w@6mwE%U?77OGLJ6J3b2+%A)qDQ2FN7UI zCdUy%bX8_M;6;y-wW*47nI!LRj0fU{=+hAv*VUPH{4O~ygn1L^aR3}}2g-QWi2p}t zL*9kL@okN05c>Tx@58)4Jed{gtFlDENmKN}&-ggew@BXh*IY3UNw@*cTpXzU*bvY= zg%X_gbf;Cz($hKsLT!yr-I6*(n(}0D?q|&9wV4WDrxL-_11VoRxucxTm$N3qZGk$-=A%#3z<1dr;+r*Xp=e|!ErDZMyXC?`^nrSS#nZ8WL>A&Pt_JT;?k2%euZ;G ztpFH$P#Waeg@jtj0wVV%y*m<3I~q>I1g%&_cci+7gWdbj;4=YQ&o&{l3yQ-(mn4MT zD{>WA5Hu(1k7W%4e>Pj$q$N>enKmToie1;u6#YZE>4!U?Q#U9yQsaS^MOdL|TqK?* zjWtwMn+V_qypyD%`s!RHG)YJCpSgMtBDq8jzbA~#0;`uZOwZc$-h3Tso!D>rm3HUP zMX_nVtoZ;EmepnYU);j)tU}$hY%Uo>_yc<4DKq!|harXTG`Re|KiY1do26TJf;_|% z-dB|gb0^DAgFY^k#TMSwLpsV4{J~IRB`@v@>=oyA0r?cdrJvfUpjpZAlU2*m?Ss~XzpSsaTB;yxY)jBLz{+L_1;t+AvJ#;pgzs0Y1}rb z@tTl%lu%PIyZQ}3b)pUJHiCpFf9I0zS-|&u%bRkq+Mi9Y z>k-E$_U(-mu<1h53Jr9Kd#BTC7iCa6vh)no0Pmdoas@^ z?_>cRa!Mu_cnyY{M95F*t~$|9u2TBYoRv_w=-<9<2G3(!@3TvpSplvfvbAD@t1q0o zQR=_RC(o?nw!#VN?#CoZh^sB>mnHcel2vX?HN3>#sbbuI84pOSwBj80lRSXn60u1c zA0d|*eHpN1KN1w~V9v|^`2J4gJNmWF1H->d8&543Ivc~mZeJ-gp$i#5i!|dS6MfI* z=htwZH@-QqXC(KIo2Xu=06aRBo#C&9g9Wo`ms_Jx*^OO^e$8ySE*skTcu5ocrXx?n zR%Y*~d7c`sST}Y&!yOkon2R1T@mjW1_g-(HN(bqnDFP8Tyx;=HdI2L;8La$n9ZnyO z!T6GGsrS*^MjXHs)J}$lrkV}dGH3^D)_{R@1m0ME_an%=e@2xFO4vT6>^Ik_0zoXy z)8hoClFAW+cfBzu)sft@4v`2SH)G5cplLCpNG^KZ|qVDAFOgc>Fq$CDdsYFD^DS zuSTYND|skcjM_L;HBJW`R=@i_lXVlHc;(pO5w?GBBdzw*`;#C$uuXfro!ZfyvQuf- z7F57O0MdL%9u`^rGI<{a{Y@mB@BNykas&tWg4^{qd!*8{w#ti_Kg~->QUTQ>t$g_f zhZhok9GysBy1PoY107h035!YakU&PycyH|H{)?J#`~Kz^30X>Vzs&+k$xa_0=hfI~ zhvez^hyHniaa`_w;m=@|&dJpb>8%aDDbfcNZvKy-BiXO-)Sn|aU0u-gVf2WGH|1eg z5<|868DhZ|=<0>rSo9|;-w)c|v|%bLA*Td=FGNT~`Zjl*9F~DT@gYG--JEL&#BV;zw%|y3Y0QY?TgRD(oB8t@$bSS|bxtig z--c6webtLMnF`b#2k9*1cNI?vmEPBaA-9;O5DGQs99bMeT^!8lL_}mQX6PNr?HLpf zna_kl1l(H|@u)u#Fbhzel(RIHwV6HOM1(jM^(lzvGuz(ZazL#cV}j6NfHjZRvY#|} zB$B|-JQBnR!r<|3{t^%M9UR&IoBRO%t@yRt`r{tb-kvZ=hdHkOWE*7YP2E!Ios-UE zfQ5bPSu^b>W8p4FpGPq4nTo18kI9C9TRj#0&WKp(l3Q8Cl7!URtUx;wCxc60i?u>Y z;;2nII{R*Y0x7GTV4;bZa_!wfv1DXsyzR}hU*Dka*~ROQ_m(9s2V3fR%5AeCcPEa@ zDTx|QBM)b<0?eC@7=Qrt*J;d3+;7ayNj8a~)-=?(%%g8P;|Hd0Qy87n=AWnEZsR9u z<65;CTTobCa6Trm4VhJP*6(j5G%Evb#E|EO;qEhN=U1$F7idEgy}MHJK6&t{Lhl-l z%RIu~x348hpc|s1k6usnOw;8q*EFPU}H)|`rkY>#eitP?Va zy+M@+#awi(H+Gn)xie|imSv^pelV?2K*@{#|X&jj9*6DvfvDu-2NOt$HEFPU#%z|Vjx z#?LaJT56^b`n-9eKi5JHa@C{C`t#&aL|vSx-%5LlZhN zTp(dRn9cM>VAMy(xjd67j+e6ief!K5#GZXc{qYBw{qlR*b|htjK?GoaM;cryz<*zB zV}6}fxvmO%JS*}m)-K26<#CpueqF$0QaP^u&=!c*BM3&dnRO`J)I61ps;sT$F=Lho zJvar?qz|#!jE(B-y=K!MW?M>+=cn-#*kY9r*I2%53jR%oh4|xaxH|W{VzQ;O9BCdHLICa_%XJQ3m)EHe6z1p2Z&$kRSU}eEENYPrp=Y z*8+J-)MlY#F*geqlzAI#>4ng4`6zAIUf?*lT{fucc4&A$y3wh{tN9lY-@RL$GMXrU zbx)%#-CeLc_Ys?NEwN%H7Ug=r(^ET=<)rjYriby8s7+rfWEE#2OmSb5GsSb=+2g!%K^+Q$-2TCYM zgTjwV2}IM-vA1#TxZVCPgdRQr;~RTMw0pUQ*3c*&o*6+zwuH;Q;LRkGI|TlKl}t=m zT%>o7>#`#pgMoY5U-j{llDtoGEf1Rp41R7=Ej!`N5bP*QE&&m%TQBSeI>+~)zz3#G zPDy-2juZVqV-_s@nA;Q@iNDmof@pC=k%F-)5Jt7Fpm=3qqF%JBJu>R9_zygWJ|yXm z5Jnn$@zy82_@L3lZ=2%^_GZB6DcoyF(i#bR5*s+kXRBLc0|{6@WAv@ladZP1F>}eL zWo}JkRJv5&Ew)BOMikz6X`U*Rc~uq{oqM#suSF=R;)f^tb)`p$Cv74bGb$k7-z6Co zWn0R#Mos3(SOo|3l(^C(UFLsS>4J|&y7Hi7V+V^mMUX;}*q(c_DEz+qY@fYpYGkhA zZ~Iv~{y`5}AATg-^M>yO1acTyGoG(qWJ|%-h~DC`-#i^d2ffYl<5=w+iVWr|xcTAB{%Db7LlU#beAkPAv$+pf2jvFU>OsNZotJ2xl2UQepc zBa5a$njd|00heg~aHXOZIi39^0~%J{%)#aQh3!9#qrJE6ZO>cJ6}Xk#m#KL7JANY( zbj#j=)j#-b9l_CL$WS0tJ1l4_ANcR4>9YUqAW{fm5&x494()-$0-oKtQ{&>EaENeU_?Uu0EtVf~~C`M^I8v_S5OP0WM@8J@Fg z9Zfz!z?rJ+xh}3P32UzP-KuIxd-1c)$YuYfzIh5q3&&CKIiLIX_)u9e#MHw3UeCAY z-8DT}u!$w6URb_gMHv5xR1)Es1LPt?&*|x1^0RY98)!`7u*mr9xu!S!MSER2JXbW` zA8(VYBj3i5TXs zNN9g0N}sI{8mAiDOp@OX6eGIi%GgUOhw%m2bRKEo2N-=~rfCNDbI)8p0=AYe=(4^y zYeeAbCeSK4I?so^?X-N4L6covwMBdtMc+cBo5_h}m{fJFiBUgGt~<^D`&|!^)RC3y z_(yu%SONgDvh)D>zbjY(CIl1!;xGDV5J15EM{51WpZ~>%e=+U9HjsZY7Ucig+eAVC z$Nn$jf11qS`5&JDm9PLnG8_QOSV N<6j!6|A+sf{~x}v)A#@Y literal 0 HcmV?d00001 diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 5d4fea1..5928f20 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -1,7 +1,10 @@ """ Cardano Account To Pandas Dumper.""" import datetime import itertools +from base64 import b64decode from collections import defaultdict +from io import BytesIO +import os from typing import ( Any, Callable, @@ -16,6 +19,7 @@ ) import blockfrost.utils +import matplotlib as mpl import numpy as np import pandas as pd from blockfrost import BlockFrostApi @@ -590,3 +594,45 @@ def make_transaction_frame( ), f"Frame lengths do not match {msg_frame=!s} , {balance_frame=!s}" joined_frame = pd.concat(objs=[msg_frame, balance_frame], axis=1) return joined_frame + + def plot_balance(self): + balance = self.make_balance_frame(with_total=False, raw_values=True).cumsum() + balance.sort_index( + axis=1, + level=0, + sort_remaining=True, + inplace=True, + key=lambda i: [self.asset_names.get(x, x) for x in i], + ) + + balance.plot( + logy=True, title=f"Asset balances until block {self.data.to_block}." + ) + assets = [self.data.assets.get(c, None) for c in balance.columns] + logos = [ + BytesIO(b64decode(a.metadata.logo)) + if ( + a + and hasattr(a, "metadata") + and hasattr(a.metadata, "logo") + and a.metadata.logo + ) + else None + for a in assets + ] + assert ( + balance.columns[0] == self.ADA_ASSET + ), f"ADA not the first asset in {balance.columns}" + logos[0] = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "ada_logo.webp" + ) + + mpl.pyplot.legend( + [ + mpl.patches.Rectangle(xy=(0, 0), width=10, height=10, color=f"C{i}") + for i in range(len(balance.columns)) + ], + [self.asset_names.get(c, c) for c in balance.columns], + bbox_to_anchor=(1, 1), + fontsize="small", + ) From 9e9e1ef39c9274c4116f91482d34e226a0855e8c Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Wed, 11 Oct 2023 13:08:19 +0200 Subject: [PATCH 097/124] simplify logos --- .../cardano_account_pandas_dumper.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 5928f20..bc119ab 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -617,15 +617,15 @@ def plot_balance(self): and hasattr(a.metadata, "logo") and a.metadata.logo ) - else None + else ( + os.path.join( + os.path.dirname(os.path.abspath(__file__)), "ada_logo.webp" + ) + if a is None # ADA + else None + ) for a in assets ] - assert ( - balance.columns[0] == self.ADA_ASSET - ), f"ADA not the first asset in {balance.columns}" - logos[0] = os.path.join( - os.path.dirname(os.path.abspath(__file__)), "ada_logo.webp" - ) mpl.pyplot.legend( [ From 818b1e514e06d714a0e06ccf331db3ea2eb625de Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Thu, 12 Oct 2023 11:21:56 +0200 Subject: [PATCH 098/124] factor out logos creation --- .../cardano_account_pandas_dumper.py | 95 ++++++++++++++----- 1 file changed, 71 insertions(+), 24 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index bc119ab..6472202 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -20,6 +20,10 @@ import blockfrost.utils import matplotlib as mpl +from matplotlib.image import BboxImage +from matplotlib.legend_handler import HandlerBase +from matplotlib.patches import Rectangle +from matplotlib.transforms import TransformedBbox import numpy as np import pandas as pd from blockfrost import BlockFrostApi @@ -164,6 +168,7 @@ def __init__( self.truncate_length = truncate_length self.unmute = unmute self.detail_level = detail_level + self.logos = None # Created lazily on plot self.address_names = pd.Series( {a: " wallet" for a in self.data.own_addresses} | known_dict.get("addresses", {}) @@ -595,6 +600,26 @@ def make_transaction_frame( joined_frame = pd.concat(objs=[msg_frame, balance_frame], axis=1) return joined_frame + def _make_logos_vector(self): + if self.logos is None: + self.logos = pd.Series( + { + a.asset: BytesIO(b64decode(a.metadata.logo)) + if ( + hasattr(a, "metadata") + and hasattr(a.metadata, "logo") + and a.metadata.logo + ) + else None + for a in self.data.assets + } + | { + self.ADA_ASSET: os.path.join( + os.path.dirname(os.path.abspath(__file__)), "ada_logo.webp" + ) + } + ) + def plot_balance(self): balance = self.make_balance_frame(with_total=False, raw_values=True).cumsum() balance.sort_index( @@ -606,33 +631,55 @@ def plot_balance(self): ) balance.plot( - logy=True, title=f"Asset balances until block {self.data.to_block}." + logy=True, + title=f"Asset balances in wallet until block {self.data.to_block}.", ) - assets = [self.data.assets.get(c, None) for c in balance.columns] - logos = [ - BytesIO(b64decode(a.metadata.logo)) - if ( - a - and hasattr(a, "metadata") - and hasattr(a.metadata, "logo") - and a.metadata.logo - ) - else ( - os.path.join( - os.path.dirname(os.path.abspath(__file__)), "ada_logo.webp" - ) - if a is None # ADA - else None - ) - for a in assets - ] + self._make_logos_vector() + + class _ImageHandler(HandlerBase): + def __init__(self, data: Any) -> None: + self.image = mpl.image.imread(data) if data is not None else None + super().__init__(10, 10) + + def create_artists( + self, + legend, + orig_handle, + xdescent, + ydescent, + width, + height, + fontsize, + trans, + ): + if self.image is not None: + image = BboxImage( + TransformedBbox( + orig_handle.get_bbox().expanded(0.7, 0.7), transform=trans + ), + interpolation="antialiased", + resample=True, + ) + image.set_data(self.image) + + self.update_prop(image, orig_handle, legend) + return [orig_handle, image] + else: + return [orig_handle] + + legends = [ + Rectangle(xy=(0, 0), width=10, height=10, color=f"C{i}") + for i in range(len(balance.columns)) + ] mpl.pyplot.legend( - [ - mpl.patches.Rectangle(xy=(0, 0), width=10, height=10, color=f"C{i}") - for i in range(len(balance.columns)) - ], + legends, [self.asset_names.get(c, c) for c in balance.columns], + handler_map={ + legends[i]: _ImageHandler(self.logos[balance.columns[i]]) + for i in range(len(balance.columns)) + }, bbox_to_anchor=(1, 1), - fontsize="small", + labelcolor="linecolor", + shadow=True, ) From 2b16aaba256c565282c077f7674d6fd82427a1d6 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Thu, 12 Oct 2023 15:42:59 +0200 Subject: [PATCH 099/124] some README fixes --- README.md | 34 ++++------------------------------ 1 file changed, 4 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 4ea1894..9fe317f 100644 --- a/README.md +++ b/README.md @@ -119,12 +119,11 @@ transaction message columns 3-...: transaction input (positive) or output (negative) for each asset and address. -row 0: asset policy -row 1: asset name -row 2: address +row 0: asset name +row 1: address -If the `--raw_values` flag is passed, row 3 is inserted, with a value of `own`for own addresses (belonging to the specified staking addresses) -and `other` for other addresses (if `--raw_values`is not passed, this information is on row 2). +If the `--raw_values` flag is passed, row 2 is inserted, with a value of `own`for own addresses (belonging to the specified staking addresses) +and `other` for other addresses. Addresses belonging to one of the specified staking addresses are labeled as `own`. With `--detail_level=2`, known addresses are listed with their name, other addresses are labeled as `other`. @@ -155,28 +154,3 @@ or purchasing one of our cool [PixelSoup NFTs](https://www.jpg.store/PixelSoup?t Donations and NFT purchases are both really appreciated, the advantage of an NFT purchase is that there is a nonzero probability of financial upside. If you think this tool can be useful to others, please retweet [the announcement](https://twitter.com/PixelSoup42/status/1697305462721396957) - -## Comparison with [cardano-accointing-exporter](https://github.com/pabstma/cardano-accointing-exporter) - -After finishing this tool, I was made aware that another comparable project existed: [cardano-accointing-exporter](https://github.com/pabstma/cardano-accointing-exporter) - -Here is a comparison table for both projects (please submit corrections if you think anything is wrong): - -| Feature | [cardano_account_pandas_dumper](https://github.com/pixelsoup42/cardano_account_pandas_dumper) | [cardano-accointing-exporter](https://github.com/pabstma/cardano-accointing-exporter) | -| ------- | --------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------| -| CSV output |✔️|✔️| -| .xlsx output |✔️|✔️| -| coingecko integration for fiat price |❌|✔️[^1]| -| Knows about assets other than ADA |✔️|❌| -| Knows about DeFI contract addresses |✔️[^2]|❌| -| Extracts useful information from tx metadata |✔️|❌| -| Ready to use after one-liner install command |✔️|❌| -| Code is [Mypy](https://mypy-lang.org/) clean |✔️|❌| -| Lines of Python code in repo (2023-09-01)| 529 | 1011| -| Has a cool logo 😉 |✔️|❌| - - - -[^1]: Could not get this to work - -[^2]: With `--detail_level=2` From 0638abf5f33c7f45112e804368c815f1e4282bf6 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Thu, 12 Oct 2023 15:46:50 +0200 Subject: [PATCH 100/124] refactor legend --- .../cardano_account_pandas_dumper.py | 109 +++++++++++------- 1 file changed, 65 insertions(+), 44 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 6472202..c0a7f45 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -20,6 +20,9 @@ import blockfrost.utils import matplotlib as mpl +import matplotlib.pyplot as pyplot + +from matplotlib.font_manager import FontProperties from matplotlib.image import BboxImage from matplotlib.legend_handler import HandlerBase from matplotlib.patches import Rectangle @@ -620,6 +623,41 @@ def _make_logos_vector(self): } ) + class _ImageHandler(HandlerBase): + def __init__(self, size, color, data: Any) -> None: + self.size=size + self.image = mpl.image.imread(data) if data is not None else None + self.color=color + super().__init__() + + def create_artists( + self, + legend, + orig_handle, + xdescent, + ydescent, + width, + height, + fontsize, + trans, + ): + rectangle = Rectangle(xy=(0, 0), width=self.size, height=self.size, color=self.color) + if self.image is not None: + image = BboxImage( + TransformedBbox( + rectangle.get_bbox().expanded(0.7, 0.7), transform=trans + ), + interpolation="antialiased", + resample=True, + ) + image.set_data(self.image) + + self.update_prop(image, orig_handle, legend) + + return [rectangle, image] + else: + return [rectangle] + def plot_balance(self): balance = self.make_balance_frame(with_total=False, raw_values=True).cumsum() balance.sort_index( @@ -629,57 +667,40 @@ def plot_balance(self): inplace=True, key=lambda i: [self.asset_names.get(x, x) for x in i], ) + font_properties = FontProperties(size="small") + fig = pyplot.figure() + plot_ax=pyplot.subplot2grid(shape=(1,3),loc=(0,0),colspan=2, fig=fig) + + legend_ax=pyplot.subplot2grid(shape=(1,3),loc=(0,2),colspan=1, fig=fig) - balance.plot( + plot=balance.plot( + ax=plot_ax, logy=True, title=f"Asset balances in wallet until block {self.data.to_block}.", + legend=False, ) + text=pyplot.text(x=0,y=0,s="M", font_properties=font_properties) + text_bbox=text.get_window_extent() + text.remove() self._make_logos_vector() - - class _ImageHandler(HandlerBase): - def __init__(self, data: Any) -> None: - self.image = mpl.image.imread(data) if data is not None else None - super().__init__(10, 10) - - def create_artists( - self, - legend, - orig_handle, - xdescent, - ydescent, - width, - height, - fontsize, - trans, - ): - if self.image is not None: - image = BboxImage( - TransformedBbox( - orig_handle.get_bbox().expanded(0.7, 0.7), transform=trans - ), - interpolation="antialiased", - resample=True, - ) - image.set_data(self.image) - - self.update_prop(image, orig_handle, legend) - - return [orig_handle, image] - else: - return [orig_handle] - - legends = [ - Rectangle(xy=(0, 0), width=10, height=10, color=f"C{i}") - for i in range(len(balance.columns)) - ] - mpl.pyplot.legend( - legends, + LEGEND_FONT_SCALE=2 + ASSETS_PER_COLUMN=16 + legend_ax.axis("off") + for text in legend_ax.legend( + plot.get_lines(), [self.asset_names.get(c, c) for c in balance.columns], handler_map={ - legends[i]: _ImageHandler(self.logos[balance.columns[i]]) + plot.get_lines()[i]: self._ImageHandler(size=LEGEND_FONT_SCALE*text_bbox.width,color=f"C{i}",data=self.logos[balance.columns[i]]) for i in range(len(balance.columns)) }, - bbox_to_anchor=(1, 1), labelcolor="linecolor", - shadow=True, - ) + prop=font_properties, + handleheight=LEGEND_FONT_SCALE, + handlelength=LEGEND_FONT_SCALE, + labelspacing=LEGEND_FONT_SCALE+.1, + ncols=max(len(plot.get_lines())/ASSETS_PER_COLUMN,1), + frameon=False + + ).get_texts(): + text.set(y=text.get_window_extent().y0 + LEGEND_FONT_SCALE * text_bbox.height / 2) + fig.subplots_adjust() From c5781d2290ad13ea4a789ea2ed5634e8fe06d35a Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Thu, 12 Oct 2023 16:05:29 +0200 Subject: [PATCH 101/124] more legend tweaking --- .../cardano_account_pandas_dumper.py | 28 +++++++++---------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index c0a7f45..5213500 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -624,8 +624,7 @@ def _make_logos_vector(self): ) class _ImageHandler(HandlerBase): - def __init__(self, size, color, data: Any) -> None: - self.size=size + def __init__(self, color, data: Any) -> None: self.image = mpl.image.imread(data) if data is not None else None self.color=color super().__init__() @@ -641,7 +640,8 @@ def create_artists( fontsize, trans, ): - rectangle = Rectangle(xy=(0, 0), width=self.size, height=self.size, color=self.color) + rectangle = Rectangle(xy=(xdescent, ydescent), + width=width, height=height, color=self.color) if self.image is not None: image = BboxImage( TransformedBbox( @@ -669,9 +669,8 @@ def plot_balance(self): ) font_properties = FontProperties(size="small") fig = pyplot.figure() - plot_ax=pyplot.subplot2grid(shape=(1,3),loc=(0,0),colspan=2, fig=fig) - - legend_ax=pyplot.subplot2grid(shape=(1,3),loc=(0,2),colspan=1, fig=fig) + plot_ax=fig.add_subplot(1,2,1) + legend_ax=fig.add_subplot(1,2,2) plot=balance.plot( ax=plot_ax, @@ -683,24 +682,23 @@ def plot_balance(self): text_bbox=text.get_window_extent() text.remove() self._make_logos_vector() - LEGEND_FONT_SCALE=2 - ASSETS_PER_COLUMN=16 + legend_font_scale=2 + assets_per_column=24 legend_ax.axis("off") for text in legend_ax.legend( plot.get_lines(), [self.asset_names.get(c, c) for c in balance.columns], handler_map={ - plot.get_lines()[i]: self._ImageHandler(size=LEGEND_FONT_SCALE*text_bbox.width,color=f"C{i}",data=self.logos[balance.columns[i]]) + plot.get_lines()[i]: self._ImageHandler(color=f"C{i}",data=self.logos[balance.columns[i]]) for i in range(len(balance.columns)) }, labelcolor="linecolor", prop=font_properties, - handleheight=LEGEND_FONT_SCALE, - handlelength=LEGEND_FONT_SCALE, - labelspacing=LEGEND_FONT_SCALE+.1, - ncols=max(len(plot.get_lines())/ASSETS_PER_COLUMN,1), + handleheight=legend_font_scale, + handlelength=legend_font_scale, + ncols=max(len(plot.get_lines())/assets_per_column,1), frameon=False ).get_texts(): - text.set(y=text.get_window_extent().y0 + LEGEND_FONT_SCALE * text_bbox.height / 2) - fig.subplots_adjust() + text.set(y=text.get_window_extent().y0 + legend_font_scale * text_bbox.height / 2) + pyplot.tight_layout() From 55fd7830b79f3a997c50cf8c6326a59eff6c0195 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Thu, 12 Oct 2023 17:04:41 +0200 Subject: [PATCH 102/124] tweak ncols and loc --- .../cardano_account_pandas_dumper.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 5213500..b6b810e 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -668,7 +668,7 @@ def plot_balance(self): key=lambda i: [self.asset_names.get(x, x) for x in i], ) font_properties = FontProperties(size="small") - fig = pyplot.figure() + fig = pyplot.figure(constrained_layout=True) plot_ax=fig.add_subplot(1,2,1) legend_ax=fig.add_subplot(1,2,2) @@ -696,9 +696,9 @@ def plot_balance(self): prop=font_properties, handleheight=legend_font_scale, handlelength=legend_font_scale, - ncols=max(len(plot.get_lines())/assets_per_column,1), - frameon=False + ncols=int((len(plot.get_lines())+assets_per_column)/assets_per_column), + frameon=False, + loc="center right" ).get_texts(): text.set(y=text.get_window_extent().y0 + legend_font_scale * text_bbox.height / 2) - pyplot.tight_layout() From 98c93d2c75ddc7c1bb0a3b192aff72701202aa14 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Thu, 12 Oct 2023 19:26:17 +0200 Subject: [PATCH 103/124] change flag to svg_output --- src/cardano_account_pandas_dumper/__main__.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/cardano_account_pandas_dumper/__main__.py b/src/cardano_account_pandas_dumper/__main__.py index 76c6625..b1a4e99 100644 --- a/src/cardano_account_pandas_dumper/__main__.py +++ b/src/cardano_account_pandas_dumper/__main__.py @@ -6,7 +6,7 @@ from json import JSONDecodeError import jstyleson -import matplotlib as mpl +import matplotlib.pyplot as plt from blockfrost import ApiError, BlockFrostApi from openpyxl.cell.cell import ILLEGAL_CHARACTERS_RE from .cardano_account_pandas_dumper import AccountData, AccountPandasDumper @@ -67,9 +67,9 @@ def _create_arg_parser(): type=argparse.FileType("wb"), ) result.add_argument( - "--plot", - help="Draw a plot of balance over time.", - action="store_true", + "--svg_output", + help="Path to SVG output file.", + type=argparse.FileType("wb"), ) result.add_argument( "--detail_level", @@ -121,11 +121,11 @@ def main(): message="Following addresses do not look like valid staking addresses: " + " ".join(invalid_staking_addresses), ) - if not any([args.checkpoint_output, args.csv_output, args.xlsx_output, args.plot]): + if not any([args.checkpoint_output, args.csv_output, args.xlsx_output, args.svg_output]): parser.exit( status=1, message="No output specified, neeed at least one of --checkpoint_output," - + " --csv_output, --xlsx_output, --plot.\n", + + " --csv_output, --xlsx_output, --svg_output.\n", ) known_dict_from_file = jstyleson.load(args.known_file) if args.known_file else {} staking_addresses_set = frozenset(args.staking_address) @@ -227,9 +227,12 @@ def main(): ) except OSError as exception: warnings.warn(f"Failed to write .xlsx file: {exception}") - if args.plot: + if args.svg_output: reporter.plot_balance() - mpl.pyplot.show() + try: + plt.savefig(args.svg_output,format='svg') + except OSError as exception: + warnings.warn(f"Failed to write .svg file: {exception}") print("Done.") From 6c9c0ddc451feb0a50f42c9d129c4928e820bbf2 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Thu, 12 Oct 2023 19:54:22 +0200 Subject: [PATCH 104/124] improve legend layout --- .../cardano_account_pandas_dumper.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index b6b810e..53738fe 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -623,7 +623,7 @@ def _make_logos_vector(self): } ) - class _ImageHandler(HandlerBase): + class _ImageLegendHandler(HandlerBase): def __init__(self, color, data: Any) -> None: self.image = mpl.image.imread(data) if data is not None else None self.color=color @@ -668,9 +668,9 @@ def plot_balance(self): key=lambda i: [self.asset_names.get(x, x) for x in i], ) font_properties = FontProperties(size="small") - fig = pyplot.figure(constrained_layout=True) - plot_ax=fig.add_subplot(1,2,1) - legend_ax=fig.add_subplot(1,2,2) + fig = pyplot.figure(constrained_layout=True,figsize=(20,14)) + plot_ax=pyplot.subplot2grid(fig=fig,shape=(1,8),loc=(0,0),colspan=7) + legend_ax=pyplot.subplot2grid(fig=fig,shape=(1,8),loc=(0,7),colspan=1) plot=balance.plot( ax=plot_ax, @@ -683,20 +683,18 @@ def plot_balance(self): text.remove() self._make_logos_vector() legend_font_scale=2 - assets_per_column=24 legend_ax.axis("off") for text in legend_ax.legend( plot.get_lines(), [self.asset_names.get(c, c) for c in balance.columns], handler_map={ - plot.get_lines()[i]: self._ImageHandler(color=f"C{i}",data=self.logos[balance.columns[i]]) + plot.get_lines()[i]: self._ImageLegendHandler(color=f"C{i}",data=self.logos[balance.columns[i]]) for i in range(len(balance.columns)) }, labelcolor="linecolor", prop=font_properties, handleheight=legend_font_scale, handlelength=legend_font_scale, - ncols=int((len(plot.get_lines())+assets_per_column)/assets_per_column), frameon=False, loc="center right" From b737fffd43fdbb856268377fd09e8e67f564370e Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Fri, 13 Oct 2023 10:47:25 +0200 Subject: [PATCH 105/124] add .svg --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 15ad5c3..4addd16 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ *.pickle *.csv *.xlsx +*.svg *.bak build *.egg-info From d0d334003d214e3bb1de858a3dc422888870595f Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Fri, 13 Oct 2023 10:50:15 +0200 Subject: [PATCH 106/124] update --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f16bb50..745f72b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ keywords = [ "report", "transaction", "wallet", + "graph", ] dependencies = [ "blockfrost-python", @@ -22,7 +23,6 @@ dependencies = [ "openpyxl", "pandas", "types-openpyxl", - "types-Pillow", ] [project.scripts] From f09dd2891fe4dc9a796aaefc79510cd07d4d0746 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Fri, 13 Oct 2023 11:07:34 +0200 Subject: [PATCH 107/124] switch to graph_output, add metadata --- src/cardano_account_pandas_dumper/__main__.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/cardano_account_pandas_dumper/__main__.py b/src/cardano_account_pandas_dumper/__main__.py index b1a4e99..6d1ee03 100644 --- a/src/cardano_account_pandas_dumper/__main__.py +++ b/src/cardano_account_pandas_dumper/__main__.py @@ -14,6 +14,7 @@ # Error codes due to project key rate limiting or capping PROJECT_KEY_ERROR_CODES = frozenset([402, 403, 418, 429]) +CREATOR_STRING="https://github.com/pixelsoup42/cardano_account_pandas_dumper" def _create_arg_parser(): result = argparse.ArgumentParser( @@ -67,9 +68,10 @@ def _create_arg_parser(): type=argparse.FileType("wb"), ) result.add_argument( - "--svg_output", - help="Path to SVG output file.", - type=argparse.FileType("wb"), + "--graph_output", + help="Path to graphics output file.", + type=str, + ) result.add_argument( "--detail_level", @@ -121,11 +123,11 @@ def main(): message="Following addresses do not look like valid staking addresses: " + " ".join(invalid_staking_addresses), ) - if not any([args.checkpoint_output, args.csv_output, args.xlsx_output, args.svg_output]): + if not any([args.checkpoint_output, args.csv_output, args.xlsx_output, args.graph_output]): parser.exit( status=1, message="No output specified, neeed at least one of --checkpoint_output," - + " --csv_output, --xlsx_output, --svg_output.\n", + + " --csv_output, --xlsx_output, --graph_output.\n", ) known_dict_from_file = jstyleson.load(args.known_file) if args.known_file else {} staking_addresses_set = frozenset(args.staking_address) @@ -227,12 +229,13 @@ def main(): ) except OSError as exception: warnings.warn(f"Failed to write .xlsx file: {exception}") - if args.svg_output: + if args.graph_output: reporter.plot_balance() try: - plt.savefig(args.svg_output,format='svg') + plt.savefig(args.graph_output, + metadata={"Creator":CREATOR_STRING,"Software":CREATOR_STRING}) except OSError as exception: - warnings.warn(f"Failed to write .svg file: {exception}") + warnings.warn(f"Failed to write graph file: {exception}") print("Done.") From 4dbe806bafcc6211c735355a1503ea2384d0356a Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Fri, 13 Oct 2023 12:10:11 +0200 Subject: [PATCH 108/124] move metadata creation to reporter --- src/cardano_account_pandas_dumper/__main__.py | 4 +-- .../cardano_account_pandas_dumper.py | 29 +++++++++++++++++-- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/src/cardano_account_pandas_dumper/__main__.py b/src/cardano_account_pandas_dumper/__main__.py index 6d1ee03..671cf34 100644 --- a/src/cardano_account_pandas_dumper/__main__.py +++ b/src/cardano_account_pandas_dumper/__main__.py @@ -14,7 +14,6 @@ # Error codes due to project key rate limiting or capping PROJECT_KEY_ERROR_CODES = frozenset([402, 403, 418, 429]) -CREATOR_STRING="https://github.com/pixelsoup42/cardano_account_pandas_dumper" def _create_arg_parser(): result = argparse.ArgumentParser( @@ -232,8 +231,7 @@ def main(): if args.graph_output: reporter.plot_balance() try: - plt.savefig(args.graph_output, - metadata={"Creator":CREATOR_STRING,"Software":CREATOR_STRING}) + plt.savefig(args.graph_output,metadata=reporter.get_graph_metadata(args.graph_output),pad_inches=0.5) except OSError as exception: warnings.warn(f"Failed to write graph file: {exception}") print("Done.") diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 53738fe..6bc7200 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -11,6 +11,7 @@ Dict, FrozenSet, List, + Mapping, MutableMapping, Optional, Set, @@ -31,6 +32,9 @@ import pandas as pd from blockfrost import BlockFrostApi +CREATOR_STRING="https://github.com/pixelsoup42/cardano_account_pandas_dumper" + + class AccountData: """Hold data retrieved from the API to allow checkpointing it.""" @@ -624,6 +628,7 @@ def _make_logos_vector(self): ) class _ImageLegendHandler(HandlerBase): + # TODO: pass asset instead of data, move code above to here, move outside class def __init__(self, color, data: Any) -> None: self.image = mpl.image.imread(data) if data is not None else None self.color=color @@ -658,7 +663,11 @@ def create_artists( else: return [rectangle] + def _plot_title(self): + return f"Asset balances in wallet until block {self.data.to_block}." + def plot_balance(self): + """ Create a Matplotlib plot with the asset balance over time.""" balance = self.make_balance_frame(with_total=False, raw_values=True).cumsum() balance.sort_index( axis=1, @@ -675,12 +684,14 @@ def plot_balance(self): plot=balance.plot( ax=plot_ax, logy=True, - title=f"Asset balances in wallet until block {self.data.to_block}.", + title=self._plot_title(), legend=False, ) + # Get font size text=pyplot.text(x=0,y=0,s="M", font_properties=font_properties) text_bbox=text.get_window_extent() text.remove() + self._make_logos_vector() legend_font_scale=2 legend_ax.axis("off") @@ -688,7 +699,8 @@ def plot_balance(self): plot.get_lines(), [self.asset_names.get(c, c) for c in balance.columns], handler_map={ - plot.get_lines()[i]: self._ImageLegendHandler(color=f"C{i}",data=self.logos[balance.columns[i]]) + plot.get_lines()[i]: + self._ImageLegendHandler(color=f"C{i}",data=self.logos[balance.columns[i]]) for i in range(len(balance.columns)) }, labelcolor="linecolor", @@ -700,3 +712,16 @@ def plot_balance(self): ).get_texts(): text.set(y=text.get_window_extent().y0 + legend_font_scale * text_bbox.height / 2) + pyplot.tight_layout() + + def get_graph_metadata(self, filename:str) -> Mapping : + """Return graph metadata depending on file extension.""" + extension=os.path.splitext(filename)[1] + if extension in (".svg",".pdf"): + return { "Creator": CREATOR_STRING, "Title": self._plot_title() } + elif extension==".png": + return {"Software" : CREATOR_STRING,"Title": self._plot_title()} + elif extension in (".ps",".eps"): + return { "Creator": CREATOR_STRING } + else: + return {} From 704f88477de21d03345486bd3cd787c8630ae0a6 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Fri, 13 Oct 2023 12:25:48 +0200 Subject: [PATCH 109/124] add pad_inches --- src/cardano_account_pandas_dumper/__main__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/cardano_account_pandas_dumper/__main__.py b/src/cardano_account_pandas_dumper/__main__.py index 671cf34..e0e8a22 100644 --- a/src/cardano_account_pandas_dumper/__main__.py +++ b/src/cardano_account_pandas_dumper/__main__.py @@ -231,7 +231,9 @@ def main(): if args.graph_output: reporter.plot_balance() try: - plt.savefig(args.graph_output,metadata=reporter.get_graph_metadata(args.graph_output),pad_inches=0.5) + plt.savefig(args.graph_output, + metadata=reporter.get_graph_metadata(args.graph_output), + pad_inches=0.5) except OSError as exception: warnings.warn(f"Failed to write graph file: {exception}") print("Done.") From ba565a311c5a0f92524db23bcfc485c9d4049704 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Fri, 13 Oct 2023 12:28:20 +0200 Subject: [PATCH 110/124] move logo creation to legend handler --- .../cardano_account_pandas_dumper.py | 47 +++++++------------ 1 file changed, 18 insertions(+), 29 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 6bc7200..e778fa9 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -175,7 +175,6 @@ def __init__( self.truncate_length = truncate_length self.unmute = unmute self.detail_level = detail_level - self.logos = None # Created lazily on plot self.address_names = pd.Series( {a: " wallet" for a in self.data.own_addresses} | known_dict.get("addresses", {}) @@ -607,30 +606,11 @@ def make_transaction_frame( joined_frame = pd.concat(objs=[msg_frame, balance_frame], axis=1) return joined_frame - def _make_logos_vector(self): - if self.logos is None: - self.logos = pd.Series( - { - a.asset: BytesIO(b64decode(a.metadata.logo)) - if ( - hasattr(a, "metadata") - and hasattr(a.metadata, "logo") - and a.metadata.logo - ) - else None - for a in self.data.assets - } - | { - self.ADA_ASSET: os.path.join( - os.path.dirname(os.path.abspath(__file__)), "ada_logo.webp" - ) - } - ) - class _ImageLegendHandler(HandlerBase): - # TODO: pass asset instead of data, move code above to here, move outside class - def __init__(self, color, data: Any) -> None: - self.image = mpl.image.imread(data) if data is not None else None + def __init__(self, color, asset_id: str, + asset_obj: Optional[blockfrost.utils.Namespace]) -> None: + self.asset_id=asset_id + self.asset_obj=asset_obj self.color=color super().__init__() @@ -647,7 +627,16 @@ def create_artists( ): rectangle = Rectangle(xy=(xdescent, ydescent), width=width, height=height, color=self.color) - if self.image is not None: + if self.asset_id == AccountPandasDumper.ADA_ASSET: + image_data=os.path.join(os.path.dirname(os.path.abspath(__file__)), "ada_logo.webp") + elif (self.asset_obj is not None + and hasattr(self.asset_obj , "metadata") + and hasattr(self.asset_obj.metadata, "logo") + and self.asset_obj.metadata.logo): + image_data= BytesIO(b64decode(self.asset_obj.metadata.logo)) + else: + image_data = None + if image_data is not None: image = BboxImage( TransformedBbox( rectangle.get_bbox().expanded(0.7, 0.7), transform=trans @@ -655,7 +644,7 @@ def create_artists( interpolation="antialiased", resample=True, ) - image.set_data(self.image) + image.set_data(mpl.image.imread(image_data)) self.update_prop(image, orig_handle, legend) @@ -691,8 +680,6 @@ def plot_balance(self): text=pyplot.text(x=0,y=0,s="M", font_properties=font_properties) text_bbox=text.get_window_extent() text.remove() - - self._make_logos_vector() legend_font_scale=2 legend_ax.axis("off") for text in legend_ax.legend( @@ -700,7 +687,9 @@ def plot_balance(self): [self.asset_names.get(c, c) for c in balance.columns], handler_map={ plot.get_lines()[i]: - self._ImageLegendHandler(color=f"C{i}",data=self.logos[balance.columns[i]]) + self._ImageLegendHandler(color=f"C{i}", + asset_id=balance.columns[i], + asset_obj=self.data.assets.get(balance.columns[i],None)) for i in range(len(balance.columns)) }, labelcolor="linecolor", From 608a4236de6446fc68d249c5801a9f2f36d734d9 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Fri, 13 Oct 2023 12:41:05 +0200 Subject: [PATCH 111/124] switch legend to medium font --- .../cardano_account_pandas_dumper.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index e778fa9..ae4b308 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -665,10 +665,10 @@ def plot_balance(self): inplace=True, key=lambda i: [self.asset_names.get(x, x) for x in i], ) - font_properties = FontProperties(size="small") + font_properties = FontProperties(size="medium") fig = pyplot.figure(constrained_layout=True,figsize=(20,14)) - plot_ax=pyplot.subplot2grid(fig=fig,shape=(1,8),loc=(0,0),colspan=7) - legend_ax=pyplot.subplot2grid(fig=fig,shape=(1,8),loc=(0,7),colspan=1) + plot_ax=pyplot.subplot2grid(fig=fig,shape=(1,7),loc=(0,0),colspan=6) + legend_ax=pyplot.subplot2grid(fig=fig,shape=(1,7),loc=(0,6),colspan=1) plot=balance.plot( ax=plot_ax, From c91c908ff71dc221aabaaa43c73b9b75773faa11 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Fri, 13 Oct 2023 14:27:55 +0200 Subject: [PATCH 112/124] switch to matplotlib.rc --- pyproject.toml | 2 +- src/cardano_account_pandas_dumper/__main__.py | 21 ++++++++++++------- .../cardano_account_pandas_dumper.py | 17 ++++----------- .../matplotlib.rc | 9 ++++++++ 4 files changed, 28 insertions(+), 21 deletions(-) create mode 100644 src/cardano_account_pandas_dumper/matplotlib.rc diff --git a/pyproject.toml b/pyproject.toml index 745f72b..a56881e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,7 @@ cardano_account_pandas_dumper = "cardano_account_pandas_dumper.__main__:main" where = ["src"] [tool.setuptools.package-data] -cardano_account_pandas_dumper = ["*.jsonc", "*.webp"] +cardano_account_pandas_dumper = ["*.jsonc", "*.webp", "*.rc"] [tool.mypy] follow_imports = "normal" diff --git a/src/cardano_account_pandas_dumper/__main__.py b/src/cardano_account_pandas_dumper/__main__.py index e0e8a22..719704b 100644 --- a/src/cardano_account_pandas_dumper/__main__.py +++ b/src/cardano_account_pandas_dumper/__main__.py @@ -6,6 +6,7 @@ from json import JSONDecodeError import jstyleson +import matplotlib as mpl import matplotlib.pyplot as plt from blockfrost import ApiError, BlockFrostApi from openpyxl.cell.cell import ILLEGAL_CHARACTERS_RE @@ -72,6 +73,12 @@ def _create_arg_parser(): type=str, ) + result.add_argument( + "--matplotlib_rc", + help="Path to matplotlib defaults file.", + type=str, + default=os.path.join(os.path.dirname(os.path.abspath(__file__)), "matplotlib.rc"), + ) result.add_argument( "--detail_level", help="Level of detail of report (1=only own addresses, 2=other addresses as well).", @@ -229,13 +236,13 @@ def main(): except OSError as exception: warnings.warn(f"Failed to write .xlsx file: {exception}") if args.graph_output: - reporter.plot_balance() - try: - plt.savefig(args.graph_output, - metadata=reporter.get_graph_metadata(args.graph_output), - pad_inches=0.5) - except OSError as exception: - warnings.warn(f"Failed to write graph file: {exception}") + with mpl.rc_context(fname=args.matplotlib_rc): + try: + reporter.plot_balance() + plt.savefig(args.graph_output, + metadata=reporter.get_graph_metadata(args.graph_output)) + except OSError as exception: + warnings.warn(f"Failed to write graph file: {exception}") print("Done.") diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index ae4b308..b8ce063 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -665,10 +665,9 @@ def plot_balance(self): inplace=True, key=lambda i: [self.asset_names.get(x, x) for x in i], ) - font_properties = FontProperties(size="medium") - fig = pyplot.figure(constrained_layout=True,figsize=(20,14)) - plot_ax=pyplot.subplot2grid(fig=fig,shape=(1,7),loc=(0,0),colspan=6) - legend_ax=pyplot.subplot2grid(fig=fig,shape=(1,7),loc=(0,6),colspan=1) + font_properties = FontProperties(size=mpl.rcParams['legend.fontsize']) + plot_ax=pyplot.subplot2grid(shape=(1,7),loc=(0,0),colspan=6) + legend_ax=pyplot.subplot2grid(shape=(1,7),loc=(0,6),colspan=1) plot=balance.plot( ax=plot_ax, @@ -680,7 +679,6 @@ def plot_balance(self): text=pyplot.text(x=0,y=0,s="M", font_properties=font_properties) text_bbox=text.get_window_extent() text.remove() - legend_font_scale=2 legend_ax.axis("off") for text in legend_ax.legend( plot.get_lines(), @@ -692,16 +690,9 @@ def plot_balance(self): asset_obj=self.data.assets.get(balance.columns[i],None)) for i in range(len(balance.columns)) }, - labelcolor="linecolor", - prop=font_properties, - handleheight=legend_font_scale, - handlelength=legend_font_scale, - frameon=False, - loc="center right" ).get_texts(): - text.set(y=text.get_window_extent().y0 + legend_font_scale * text_bbox.height / 2) - pyplot.tight_layout() + text.set(y=text.get_window_extent().y0 + mpl.rcParams['legend.handleheight'] * text_bbox.height / 2) def get_graph_metadata(self, filename:str) -> Mapping : """Return graph metadata depending on file extension.""" diff --git a/src/cardano_account_pandas_dumper/matplotlib.rc b/src/cardano_account_pandas_dumper/matplotlib.rc new file mode 100644 index 0000000..4444d99 --- /dev/null +++ b/src/cardano_account_pandas_dumper/matplotlib.rc @@ -0,0 +1,9 @@ +savefig.pad_inches:0.5 +legend.fontsize:medium +legend.labelcolor:linecolor +legend.handleheight:2 +legend.handlelength:2 +legend.frameon:False +legend.loc:center +figure.autolayout: True +figure.figsize: 20,14 From 1de353099e2c405f09b703057e3db10e661e5b68 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Fri, 13 Oct 2023 14:35:55 +0200 Subject: [PATCH 113/124] use default format if unspecified --- .../cardano_account_pandas_dumper.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index b8ce063..01f2954 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -692,16 +692,19 @@ def plot_balance(self): }, ).get_texts(): - text.set(y=text.get_window_extent().y0 + mpl.rcParams['legend.handleheight'] * text_bbox.height / 2) + text.set(y=text.get_window_extent().y0 + + mpl.rcParams['legend.handleheight'] * text_bbox.height / 2) def get_graph_metadata(self, filename:str) -> Mapping : """Return graph metadata depending on file extension.""" - extension=os.path.splitext(filename)[1] - if extension in (".svg",".pdf"): + save_format=os.path.splitext(filename)[1].removeprefix('.') + if not save_format: + save_format= mpl.rcParams["savefig.format"] + if save_format in ("svg","pdf"): return { "Creator": CREATOR_STRING, "Title": self._plot_title() } - elif extension==".png": + elif save_format=="png": return {"Software" : CREATOR_STRING,"Title": self._plot_title()} - elif extension in (".ps",".eps"): + elif save_format in ("ps","eps"): return { "Creator": CREATOR_STRING } else: return {} From dbfbcfbeff549d8f2267de39a7e90d6d69810e40 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Fri, 13 Oct 2023 16:59:08 +0200 Subject: [PATCH 114/124] add log scale param --- src/cardano_account_pandas_dumper/__main__.py | 8 +++- .../cardano_account_pandas_dumper.py | 41 +++++++++++++++---- 2 files changed, 41 insertions(+), 8 deletions(-) diff --git a/src/cardano_account_pandas_dumper/__main__.py b/src/cardano_account_pandas_dumper/__main__.py index 719704b..778398c 100644 --- a/src/cardano_account_pandas_dumper/__main__.py +++ b/src/cardano_account_pandas_dumper/__main__.py @@ -79,6 +79,12 @@ def _create_arg_parser(): type=str, default=os.path.join(os.path.dirname(os.path.abspath(__file__)), "matplotlib.rc"), ) + result.add_argument( + "--log_scale", + help="Use log scale on graph, default=automatic.", + type=bool, + default=None, + ) result.add_argument( "--detail_level", help="Level of detail of report (1=only own addresses, 2=other addresses as well).", @@ -238,7 +244,7 @@ def main(): if args.graph_output: with mpl.rc_context(fname=args.matplotlib_rc): try: - reporter.plot_balance() + reporter.plot_balance(log_scale=args.log_scale) plt.savefig(args.graph_output, metadata=reporter.get_graph_metadata(args.graph_output)) except OSError as exception: diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 01f2954..2244937 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -32,7 +32,6 @@ import pandas as pd from blockfrost import BlockFrostApi -CREATOR_STRING="https://github.com/pixelsoup42/cardano_account_pandas_dumper" @@ -155,7 +154,9 @@ def _transaction_data( class AccountPandasDumper: """Hold logic to convert an instance of AccountData to a Pandas dataframe.""" + # Transaction timestamp = block time + transaction index in block * TRANSACTION_OFFSET TRANSACTION_OFFSET = np.timedelta64(1000, "ns") + OWN_LABEL = " own" OTHER_LABEL = "other" ADA_ASSET = " ADA" @@ -163,6 +164,11 @@ class AccountPandasDumper: METADATA_MESSAGE_LABEL = "674" METADATA_NFT_MINT_LABEL = "721" + # Constants for graph output + CREATOR_STRING="https://github.com/pixelsoup42/cardano_account_pandas_dumper" + # Switch to log scale if difference between min and max is more than this order of magnitude + AUTO_LOG_SCALE_THRESHOLD=8 + def __init__( self, data: AccountData, @@ -655,7 +661,28 @@ def create_artists( def _plot_title(self): return f"Asset balances in wallet until block {self.data.to_block}." - def plot_balance(self): + def _auto_log_scale(self,balance : pd.DataFrame) -> bool: + # Ignore values that are lower than asset precision or NA when calculating min + min_mask = pd.concat( + [ + balance[c] + .abs() + .ge( + np.float_power( + 10, + np.negative(self.asset_decimals[c]), + ) + ) + for c in balance.columns + ], + axis=1, + ) & balance.notna() + + balance_min=balance.where(min_mask,np.infty).abs().min(skipna=True,numeric_only=True).min() + balance_max=balance.abs().max(skipna=True,numeric_only=True).max() + return bool((np.log10(balance_max) - np.log10(balance_min)) > self.AUTO_LOG_SCALE_THRESHOLD) + + def plot_balance(self, log_scale: Optional[bool]): """ Create a Matplotlib plot with the asset balance over time.""" balance = self.make_balance_frame(with_total=False, raw_values=True).cumsum() balance.sort_index( @@ -671,7 +698,7 @@ def plot_balance(self): plot=balance.plot( ax=plot_ax, - logy=True, + logy=log_scale if log_scale is not None else self._auto_log_scale(balance), title=self._plot_title(), legend=False, ) @@ -682,7 +709,7 @@ def plot_balance(self): legend_ax.axis("off") for text in legend_ax.legend( plot.get_lines(), - [self.asset_names.get(c, c) for c in balance.columns], + cast(List[str],[self.asset_names.get(c, c) for c in balance.columns]), handler_map={ plot.get_lines()[i]: self._ImageLegendHandler(color=f"C{i}", @@ -701,10 +728,10 @@ def get_graph_metadata(self, filename:str) -> Mapping : if not save_format: save_format= mpl.rcParams["savefig.format"] if save_format in ("svg","pdf"): - return { "Creator": CREATOR_STRING, "Title": self._plot_title() } + return { "Creator": self.CREATOR_STRING, "Title": self._plot_title() } elif save_format=="png": - return {"Software" : CREATOR_STRING,"Title": self._plot_title()} + return {"Software" : self.CREATOR_STRING,"Title": self._plot_title()} elif save_format in ("ps","eps"): - return { "Creator": CREATOR_STRING } + return { "Creator": self.CREATOR_STRING } else: return {} From fde3074b016211f406adbd9e2d0d98ded607228d Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Fri, 13 Oct 2023 18:08:52 +0200 Subject: [PATCH 115/124] more matplotlib.rc properties --- .../cardano_account_pandas_dumper.py | 2 -- src/cardano_account_pandas_dumper/matplotlib.rc | 14 +++++++++----- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 2244937..9372a3d 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -647,8 +647,6 @@ def create_artists( TransformedBbox( rectangle.get_bbox().expanded(0.7, 0.7), transform=trans ), - interpolation="antialiased", - resample=True, ) image.set_data(mpl.image.imread(image_data)) diff --git a/src/cardano_account_pandas_dumper/matplotlib.rc b/src/cardano_account_pandas_dumper/matplotlib.rc index 4444d99..b12fabe 100644 --- a/src/cardano_account_pandas_dumper/matplotlib.rc +++ b/src/cardano_account_pandas_dumper/matplotlib.rc @@ -1,9 +1,13 @@ -savefig.pad_inches:0.5 +axes.formatter.limits: -10, 12 +axes.grid: True +axes.grid.which: both +axes.titleweight: bold +figure.autolayout: True +figure.figsize: 20,14 legend.fontsize:medium -legend.labelcolor:linecolor +legend.frameon:False legend.handleheight:2 legend.handlelength:2 -legend.frameon:False +legend.labelcolor:linecolor legend.loc:center -figure.autolayout: True -figure.figsize: 20,14 +savefig.pad_inches:0.5 From e483c43623c67f53bff87af347e04e4de53fdf82 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Fri, 13 Oct 2023 19:58:09 +0200 Subject: [PATCH 116/124] switch to subplots --- src/cardano_account_pandas_dumper/__main__.py | 8 +- .../cardano_account_pandas_dumper.py | 135 ++---------------- .../matplotlib.rc | 6 +- 3 files changed, 12 insertions(+), 137 deletions(-) diff --git a/src/cardano_account_pandas_dumper/__main__.py b/src/cardano_account_pandas_dumper/__main__.py index 778398c..719704b 100644 --- a/src/cardano_account_pandas_dumper/__main__.py +++ b/src/cardano_account_pandas_dumper/__main__.py @@ -79,12 +79,6 @@ def _create_arg_parser(): type=str, default=os.path.join(os.path.dirname(os.path.abspath(__file__)), "matplotlib.rc"), ) - result.add_argument( - "--log_scale", - help="Use log scale on graph, default=automatic.", - type=bool, - default=None, - ) result.add_argument( "--detail_level", help="Level of detail of report (1=only own addresses, 2=other addresses as well).", @@ -244,7 +238,7 @@ def main(): if args.graph_output: with mpl.rc_context(fname=args.matplotlib_rc): try: - reporter.plot_balance(log_scale=args.log_scale) + reporter.plot_balance() plt.savefig(args.graph_output, metadata=reporter.get_graph_metadata(args.graph_output)) except OSError as exception: diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 9372a3d..b2b27f1 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -1,40 +1,20 @@ """ Cardano Account To Pandas Dumper.""" import datetime import itertools -from base64 import b64decode -from collections import defaultdict -from io import BytesIO import os -from typing import ( - Any, - Callable, - Dict, - FrozenSet, - List, - Mapping, - MutableMapping, - Optional, - Set, - Tuple, - cast, -) +from collections import defaultdict +from typing import (Any, Callable, Dict, FrozenSet, List, Mapping, + MutableMapping, Optional, Set, Tuple, cast) import blockfrost.utils import matplotlib as mpl -import matplotlib.pyplot as pyplot - -from matplotlib.font_manager import FontProperties -from matplotlib.image import BboxImage -from matplotlib.legend_handler import HandlerBase -from matplotlib.patches import Rectangle -from matplotlib.transforms import TransformedBbox +from matplotlib import pyplot import numpy as np import pandas as pd from blockfrost import BlockFrostApi - class AccountData: """Hold data retrieved from the API to allow checkpointing it.""" @@ -166,8 +146,6 @@ class AccountPandasDumper: # Constants for graph output CREATOR_STRING="https://github.com/pixelsoup42/cardano_account_pandas_dumper" - # Switch to log scale if difference between min and max is more than this order of magnitude - AUTO_LOG_SCALE_THRESHOLD=8 def __init__( self, @@ -612,113 +590,16 @@ def make_transaction_frame( joined_frame = pd.concat(objs=[msg_frame, balance_frame], axis=1) return joined_frame - class _ImageLegendHandler(HandlerBase): - def __init__(self, color, asset_id: str, - asset_obj: Optional[blockfrost.utils.Namespace]) -> None: - self.asset_id=asset_id - self.asset_obj=asset_obj - self.color=color - super().__init__() - - def create_artists( - self, - legend, - orig_handle, - xdescent, - ydescent, - width, - height, - fontsize, - trans, - ): - rectangle = Rectangle(xy=(xdescent, ydescent), - width=width, height=height, color=self.color) - if self.asset_id == AccountPandasDumper.ADA_ASSET: - image_data=os.path.join(os.path.dirname(os.path.abspath(__file__)), "ada_logo.webp") - elif (self.asset_obj is not None - and hasattr(self.asset_obj , "metadata") - and hasattr(self.asset_obj.metadata, "logo") - and self.asset_obj.metadata.logo): - image_data= BytesIO(b64decode(self.asset_obj.metadata.logo)) - else: - image_data = None - if image_data is not None: - image = BboxImage( - TransformedBbox( - rectangle.get_bbox().expanded(0.7, 0.7), transform=trans - ), - ) - image.set_data(mpl.image.imread(image_data)) - - self.update_prop(image, orig_handle, legend) - - return [rectangle, image] - else: - return [rectangle] - def _plot_title(self): return f"Asset balances in wallet until block {self.data.to_block}." - def _auto_log_scale(self,balance : pd.DataFrame) -> bool: - # Ignore values that are lower than asset precision or NA when calculating min - min_mask = pd.concat( - [ - balance[c] - .abs() - .ge( - np.float_power( - 10, - np.negative(self.asset_decimals[c]), - ) - ) - for c in balance.columns - ], - axis=1, - ) & balance.notna() - - balance_min=balance.where(min_mask,np.infty).abs().min(skipna=True,numeric_only=True).min() - balance_max=balance.abs().max(skipna=True,numeric_only=True).max() - return bool((np.log10(balance_max) - np.log10(balance_min)) > self.AUTO_LOG_SCALE_THRESHOLD) - - def plot_balance(self, log_scale: Optional[bool]): + def plot_balance(self): """ Create a Matplotlib plot with the asset balance over time.""" - balance = self.make_balance_frame(with_total=False, raw_values=True).cumsum() - balance.sort_index( - axis=1, - level=0, - sort_remaining=True, - inplace=True, - key=lambda i: [self.asset_names.get(x, x) for x in i], - ) - font_properties = FontProperties(size=mpl.rcParams['legend.fontsize']) - plot_ax=pyplot.subplot2grid(shape=(1,7),loc=(0,0),colspan=6) - legend_ax=pyplot.subplot2grid(shape=(1,7),loc=(0,6),colspan=1) - - plot=balance.plot( - ax=plot_ax, - logy=log_scale if log_scale is not None else self._auto_log_scale(balance), + balance = self.make_balance_frame(with_total=False,raw_values=False).cumsum() + balance.plot( title=self._plot_title(), - legend=False, + subplots=True ) - # Get font size - text=pyplot.text(x=0,y=0,s="M", font_properties=font_properties) - text_bbox=text.get_window_extent() - text.remove() - legend_ax.axis("off") - for text in legend_ax.legend( - plot.get_lines(), - cast(List[str],[self.asset_names.get(c, c) for c in balance.columns]), - handler_map={ - plot.get_lines()[i]: - self._ImageLegendHandler(color=f"C{i}", - asset_id=balance.columns[i], - asset_obj=self.data.assets.get(balance.columns[i],None)) - for i in range(len(balance.columns)) - }, - - ).get_texts(): - text.set(y=text.get_window_extent().y0 + - mpl.rcParams['legend.handleheight'] * text_bbox.height / 2) def get_graph_metadata(self, filename:str) -> Mapping : """Return graph metadata depending on file extension.""" diff --git a/src/cardano_account_pandas_dumper/matplotlib.rc b/src/cardano_account_pandas_dumper/matplotlib.rc index b12fabe..29e42a4 100644 --- a/src/cardano_account_pandas_dumper/matplotlib.rc +++ b/src/cardano_account_pandas_dumper/matplotlib.rc @@ -2,12 +2,12 @@ axes.formatter.limits: -10, 12 axes.grid: True axes.grid.which: both axes.titleweight: bold -figure.autolayout: True -figure.figsize: 20,14 +figure.constrained_layout.use: True +figure.figsize: 20,40 legend.fontsize:medium legend.frameon:False legend.handleheight:2 legend.handlelength:2 legend.labelcolor:linecolor -legend.loc:center +legend.loc:upper left savefig.pad_inches:0.5 From 5358d71f46ca526eb128f6ac185fbffe15ff0f0d Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Sat, 14 Oct 2023 02:26:08 +0200 Subject: [PATCH 117/124] subplots with redundant xticks --- .../cardano_account_pandas_dumper.py | 19 +++++++++++++------ .../matplotlib.rc | 16 +++++++--------- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index b2b27f1..815fb2b 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -595,14 +595,21 @@ def _plot_title(self): def plot_balance(self): """ Create a Matplotlib plot with the asset balance over time.""" - balance = self.make_balance_frame(with_total=False,raw_values=False).cumsum() - balance.plot( - title=self._plot_title(), - subplots=True - ) + balance = self.make_balance_frame(with_total=False,raw_values=False).cumsum().replace(pd.NA,0) + fig,ax=pyplot.subplots(len(balance.columns),2, + width_ratios=(7,1), + figsize=(11.69,2.0675*len(balance.columns))) + fig.suptitle(self._plot_title()) + for i in range(len((balance.columns))): + ax[i][1].axis("off") + balance.plot( + y=balance.columns[i], + ax=ax[i][0], + legend=False, + ) def get_graph_metadata(self, filename:str) -> Mapping : - """Return graph metadata depending on file extension.""" + """Return graph metadata for file name.""" save_format=os.path.splitext(filename)[1].removeprefix('.') if not save_format: save_format= mpl.rcParams["savefig.format"] diff --git a/src/cardano_account_pandas_dumper/matplotlib.rc b/src/cardano_account_pandas_dumper/matplotlib.rc index 29e42a4..fed8052 100644 --- a/src/cardano_account_pandas_dumper/matplotlib.rc +++ b/src/cardano_account_pandas_dumper/matplotlib.rc @@ -1,13 +1,11 @@ + axes.formatter.limits: -10, 12 axes.grid: True axes.grid.which: both -axes.titleweight: bold +axes.spines.bottom: False +axes.spines.left: True # display axis spines +axes.spines.right: False +axes.spines.top: False figure.constrained_layout.use: True -figure.figsize: 20,40 -legend.fontsize:medium -legend.frameon:False -legend.handleheight:2 -legend.handlelength:2 -legend.labelcolor:linecolor -legend.loc:upper left -savefig.pad_inches:0.5 +figure.titleweight: bold +savefig.pad_inches:0.5 \ No newline at end of file From 37de47988a5ed93549e763739f53477d883a8584 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Sat, 14 Oct 2023 13:59:59 +0200 Subject: [PATCH 118/124] fix subplot layout --- .../cardano_account_pandas_dumper.py | 37 ++++++++++++++++--- .../matplotlib.rc | 12 +++--- 2 files changed, 38 insertions(+), 11 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 815fb2b..98b2aff 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -539,7 +539,6 @@ def make_balance_frame( for c in balance.columns ] ) - balance.sort_index(axis=1, level=0, sort_remaining=True, inplace=True) if self.detail_level == 1: return balance.xs(self.OWN_LABEL, level=1, axis=1) else: @@ -577,6 +576,8 @@ def make_transaction_frame( balance_frame = self.make_balance_frame( with_total=with_total, text_cleaner=text_cleaner, raw_values=raw_values ) + if not raw_values: + balance_frame.sort_index(axis=1, level=0, sort_remaining=True, inplace=True) if self.detail_level > 1: msg_frame.columns = pd.MultiIndex.from_tuples( [ @@ -593,20 +594,44 @@ def make_transaction_frame( def _plot_title(self): return f"Asset balances in wallet until block {self.data.to_block}." - def plot_balance(self): + def plot_balance(self, order:str="alpha"): """ Create a Matplotlib plot with the asset balance over time.""" - balance = self.make_balance_frame(with_total=False,raw_values=False).cumsum().replace(pd.NA,0) + balance = self.make_balance_frame(with_total=False,raw_values=True).cumsum() + if order=="alpha": + balance.sort_index( + axis=1, + level=0, + sort_remaining=True, + inplace=True, + key=lambda i: [self.asset_names.get(x, x) for x in i], + ) + elif order=="appearance": + pass + else: + raise ValueError(f"Unkown ordering: {order}") fig,ax=pyplot.subplots(len(balance.columns),2, width_ratios=(7,1), figsize=(11.69,2.0675*len(balance.columns))) - fig.suptitle(self._plot_title()) - for i in range(len((balance.columns))): - ax[i][1].axis("off") + fig.suptitle("\n"+self._plot_title()+"\n") + for i in range(len((balance.columns))): # pylint: disable=consider-using-enumerate + ax[i][1].xaxis.set_visible(False) + ax[i][1].yaxis.set_visible(False) + ax[i][0].spines.right.set_visible(False) + ax[i][1].spines.left.set_visible(False) balance.plot( y=balance.columns[i], + xlim=(min(balance.index),max(balance.index)), ax=ax[i][0], legend=False, ) + if i==0 and len(balance.columns)>1: + ax[i][0].xaxis.set_ticks_position("top") + elif i Mapping : """Return graph metadata for file name.""" diff --git a/src/cardano_account_pandas_dumper/matplotlib.rc b/src/cardano_account_pandas_dumper/matplotlib.rc index fed8052..ac9c4e1 100644 --- a/src/cardano_account_pandas_dumper/matplotlib.rc +++ b/src/cardano_account_pandas_dumper/matplotlib.rc @@ -2,10 +2,12 @@ axes.formatter.limits: -10, 12 axes.grid: True axes.grid.which: both -axes.spines.bottom: False -axes.spines.left: True # display axis spines -axes.spines.right: False -axes.spines.top: False figure.constrained_layout.use: True +figure.constrained_layout.h_pad:0 +figure.constrained_layout.w_pad:0 +figure.constrained_layout.hspace: 0 +figure.constrained_layout.wspace: 0 figure.titleweight: bold -savefig.pad_inches:0.5 \ No newline at end of file +savefig.bbox:tight +savefig.pad_inches:0.5 + From ddd8a86bf92e36e71c1851864682e36908a5192f Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Sat, 14 Oct 2023 14:10:37 +0200 Subject: [PATCH 119/124] add sort by appearance --- .../cardano_account_pandas_dumper.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 98b2aff..faa275d 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -594,7 +594,7 @@ def make_transaction_frame( def _plot_title(self): return f"Asset balances in wallet until block {self.data.to_block}." - def plot_balance(self, order:str="alpha"): + def plot_balance(self, order:str="appearance"): """ Create a Matplotlib plot with the asset balance over time.""" balance = self.make_balance_frame(with_total=False,raw_values=True).cumsum() if order=="alpha": @@ -606,7 +606,13 @@ def plot_balance(self, order:str="alpha"): key=lambda i: [self.asset_names.get(x, x) for x in i], ) elif order=="appearance": - pass + balance.sort_index( + axis=1, + level=0, + sort_remaining=True, + inplace=True, + key=lambda i: [balance[x].first_valid_index() for x in i], + ) else: raise ValueError(f"Unkown ordering: {order}") fig,ax=pyplot.subplots(len(balance.columns),2, From 4f8fa0c4b266346088063abff9e713cdaa51aa9a Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Sat, 14 Oct 2023 16:07:21 +0200 Subject: [PATCH 120/124] first version with legend, black --- .../cardano_account_pandas_dumper.py | 112 +++++++++++++----- 1 file changed, 85 insertions(+), 27 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index faa275d..0414f45 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -1,20 +1,36 @@ """ Cardano Account To Pandas Dumper.""" +from base64 import b64decode import datetime +from io import BytesIO import itertools import os from collections import defaultdict -from typing import (Any, Callable, Dict, FrozenSet, List, Mapping, - MutableMapping, Optional, Set, Tuple, cast) +from textwrap import wrap +from typing import ( + Any, + Callable, + Dict, + FrozenSet, + List, + Mapping, + MutableMapping, + Optional, + Set, + Tuple, + cast, +) import blockfrost.utils import matplotlib as mpl from matplotlib import pyplot +from matplotlib.axes import Axes +from matplotlib.image import AxesImage, BboxImage +from matplotlib.transforms import TransformedBbox import numpy as np import pandas as pd from blockfrost import BlockFrostApi - class AccountData: """Hold data retrieved from the API to allow checkpointing it.""" @@ -145,7 +161,7 @@ class AccountPandasDumper: METADATA_NFT_MINT_LABEL = "721" # Constants for graph output - CREATOR_STRING="https://github.com/pixelsoup42/cardano_account_pandas_dumper" + CREATOR_STRING = "https://github.com/pixelsoup42/cardano_account_pandas_dumper" def __init__( self, @@ -594,10 +610,47 @@ def make_transaction_frame( def _plot_title(self): return f"Asset balances in wallet until block {self.data.to_block}." - def plot_balance(self, order:str="appearance"): - """ Create a Matplotlib plot with the asset balance over time.""" - balance = self.make_balance_frame(with_total=False,raw_values=True).cumsum() - if order=="alpha": + def _draw_asset_legend(self, ax: Axes, asset_id: str): + ticker = None + name = str(self.asset_names.get(asset_id, asset_id)) + image_data: Any = None + if asset_id == self.ADA_ASSET: + ticker = "ADA" + image_data = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "ada_logo.webp" + ) + else: + asset_obj = self.data.assets[asset_id] + if hasattr(asset_obj, "metadata"): + if hasattr(asset_obj.metadata, "logo"): + image_data = BytesIO(b64decode(asset_obj.metadata.logo)) + if hasattr(asset_obj.metadata, "ticker"): + ticker = asset_obj.metadata.ticker + if ticker: + ax.text( + 0.5, + 0.9, + ticker, + horizontalalignment="center", + transform=ax.transAxes, + fontsize="large", + fontweight="bold", + ) + ax.text( + 0.5, + 0.8, + name, + horizontalalignment="center", + transform=ax.transAxes, + fontsize="xx-small", + ) + if image_data: + ax.imshow(mpl.image.imread(image_data), aspect="auto") + + def plot_balance(self, order: str = "appearance"): + """Create a Matplotlib plot with the asset balance over time.""" + balance = self.make_balance_frame(with_total=False, raw_values=True).cumsum() + if order == "alpha": balance.sort_index( axis=1, level=0, @@ -605,7 +658,7 @@ def plot_balance(self, order:str="appearance"): inplace=True, key=lambda i: [self.asset_names.get(x, x) for x in i], ) - elif order=="appearance": + elif order == "appearance": balance.sort_index( axis=1, level=0, @@ -615,40 +668,45 @@ def plot_balance(self, order:str="appearance"): ) else: raise ValueError(f"Unkown ordering: {order}") - fig,ax=pyplot.subplots(len(balance.columns),2, - width_ratios=(7,1), - figsize=(11.69,2.0675*len(balance.columns))) - fig.suptitle("\n"+self._plot_title()+"\n") - for i in range(len((balance.columns))): # pylint: disable=consider-using-enumerate + fig, ax = pyplot.subplots( + len(balance.columns), + 2, + width_ratios=(6, 1), + figsize=(11.69, 2.0675 * len(balance.columns)), + ) + fig.suptitle("\n" + self._plot_title() + "\n") + for i in range( # pylint: disable=consider-using-enumerate + len((balance.columns)) + ): ax[i][1].xaxis.set_visible(False) ax[i][1].yaxis.set_visible(False) ax[i][0].spines.right.set_visible(False) ax[i][1].spines.left.set_visible(False) balance.plot( y=balance.columns[i], - xlim=(min(balance.index),max(balance.index)), + xlim=(min(balance.index), max(balance.index)), ax=ax[i][0], legend=False, ) - if i==0 and len(balance.columns)>1: + if i == 0 and len(balance.columns) > 1: ax[i][0].xaxis.set_ticks_position("top") - elif i Mapping : + def get_graph_metadata(self, filename: str) -> Mapping: """Return graph metadata for file name.""" - save_format=os.path.splitext(filename)[1].removeprefix('.') + save_format = os.path.splitext(filename)[1].removeprefix(".") if not save_format: - save_format= mpl.rcParams["savefig.format"] - if save_format in ("svg","pdf"): - return { "Creator": self.CREATOR_STRING, "Title": self._plot_title() } - elif save_format=="png": - return {"Software" : self.CREATOR_STRING,"Title": self._plot_title()} - elif save_format in ("ps","eps"): - return { "Creator": self.CREATOR_STRING } + save_format = mpl.rcParams["savefig.format"] + if save_format in ("svg", "pdf"): + return {"Creator": self.CREATOR_STRING, "Title": self._plot_title()} + elif save_format == "png": + return {"Software": self.CREATOR_STRING, "Title": self._plot_title()} + elif save_format in ("ps", "eps"): + return {"Creator": self.CREATOR_STRING} else: return {} From f8c999be4343bc5d739767f811de793cb8798af2 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Sat, 14 Oct 2023 17:12:32 +0200 Subject: [PATCH 121/124] fix images --- .../cardano_account_pandas_dumper.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 0414f45..e3f2e3a 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -608,7 +608,7 @@ def make_transaction_frame( return joined_frame def _plot_title(self): - return f"Asset balances in wallet until block {self.data.to_block}." + return f"Asset balances in wallet until block {self.data.to_block}" def _draw_asset_legend(self, ax: Axes, asset_id: str): ticker = None @@ -643,9 +643,17 @@ def _draw_asset_legend(self, ax: Axes, asset_id: str): horizontalalignment="center", transform=ax.transAxes, fontsize="xx-small", + clip_on=True, ) if image_data: - ax.imshow(mpl.image.imread(image_data), aspect="auto") + ax.set_adjustable("datalim") + ax.imshow( + mpl.image.imread(image_data), + aspect="equal", + extent=(0.3, 0.7, 0.8, 0.4), + ) + ax.set_xlim((0.0, 1.0)) + ax.set_ylim((1.0, 0.0)) def plot_balance(self, order: str = "appearance"): """Create a Matplotlib plot with the asset balance over time.""" From 226822b759b430e4c232beea2127f8fffc0d5eb0 Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Sat, 14 Oct 2023 17:42:50 +0200 Subject: [PATCH 122/124] arguments cleanup, add url --- src/cardano_account_pandas_dumper/__main__.py | 25 ++++---- .../cardano_account_pandas_dumper.py | 58 +++++++++---------- 2 files changed, 41 insertions(+), 42 deletions(-) diff --git a/src/cardano_account_pandas_dumper/__main__.py b/src/cardano_account_pandas_dumper/__main__.py index 719704b..b670744 100644 --- a/src/cardano_account_pandas_dumper/__main__.py +++ b/src/cardano_account_pandas_dumper/__main__.py @@ -9,7 +9,6 @@ import matplotlib as mpl import matplotlib.pyplot as plt from blockfrost import ApiError, BlockFrostApi -from openpyxl.cell.cell import ILLEGAL_CHARACTERS_RE from .cardano_account_pandas_dumper import AccountData, AccountPandasDumper # Error codes due to project key rate limiting or capping @@ -71,13 +70,14 @@ def _create_arg_parser(): "--graph_output", help="Path to graphics output file.", type=str, - ) result.add_argument( "--matplotlib_rc", help="Path to matplotlib defaults file.", type=str, - default=os.path.join(os.path.dirname(os.path.abspath(__file__)), "matplotlib.rc"), + default=os.path.join( + os.path.dirname(os.path.abspath(__file__)), "matplotlib.rc" + ), ) result.add_argument( "--detail_level", @@ -129,7 +129,9 @@ def main(): message="Following addresses do not look like valid staking addresses: " + " ".join(invalid_staking_addresses), ) - if not any([args.checkpoint_output, args.csv_output, args.xlsx_output, args.graph_output]): + if not any( + [args.checkpoint_output, args.csv_output, args.xlsx_output, args.graph_output] + ): parser.exit( status=1, message="No output specified, neeed at least one of --checkpoint_output," @@ -199,11 +201,11 @@ def main(): known_dict=known_dict_from_file, truncate_length=args.truncate_length, unmute=args.unmute, - detail_level=args.detail_level, ) if args.csv_output: try: reporter.make_transaction_frame( + detail_level=args.detail_level, with_total=args.with_total, raw_values=args.raw_values, ).to_csv( @@ -214,13 +216,8 @@ def main(): if args.xlsx_output: try: frame = reporter.make_transaction_frame( + detail_level=args.detail_level, with_total=args.with_total, - text_cleaner=lambda x: ILLEGAL_CHARACTERS_RE.sub( - lambda y: "".join( - ["\\x0" + hex(ord(y.group(0))).removeprefix("0x")] - ), - x, - ), raw_values=args.raw_values, ) frame.to_excel( @@ -239,8 +236,10 @@ def main(): with mpl.rc_context(fname=args.matplotlib_rc): try: reporter.plot_balance() - plt.savefig(args.graph_output, - metadata=reporter.get_graph_metadata(args.graph_output)) + plt.savefig( + args.graph_output, + metadata=reporter.get_graph_metadata(args.graph_output), + ) except OSError as exception: warnings.warn(f"Failed to write graph file: {exception}") print("Done.") diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index e3f2e3a..6ae8656 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -1,14 +1,12 @@ """ Cardano Account To Pandas Dumper.""" -from base64 import b64decode import datetime -from io import BytesIO import itertools import os +from base64 import b64decode from collections import defaultdict -from textwrap import wrap +from io import BytesIO from typing import ( Any, - Callable, Dict, FrozenSet, List, @@ -22,13 +20,12 @@ import blockfrost.utils import matplotlib as mpl -from matplotlib import pyplot -from matplotlib.axes import Axes -from matplotlib.image import AxesImage, BboxImage -from matplotlib.transforms import TransformedBbox import numpy as np import pandas as pd from blockfrost import BlockFrostApi +from matplotlib import pyplot +from matplotlib.axes import Axes +from openpyxl.cell.cell import ILLEGAL_CHARACTERS_RE class AccountData: @@ -169,12 +166,10 @@ def __init__( known_dict: Any, truncate_length: int, unmute: bool, - detail_level: int, ): self.data = data self.truncate_length = truncate_length self.unmute = unmute - self.detail_level = detail_level self.address_names = pd.Series( {a: " wallet" for a in self.data.own_addresses} | known_dict.get("addresses", {}) @@ -227,7 +222,11 @@ def _decode_asset_name(self, asset: blockfrost.utils.Namespace) -> str: return asset.metadata.name asset_hex_name = asset.asset.removeprefix(asset.policy_id) try: - return bytes.fromhex(asset_hex_name).decode() + decoded = bytes.fromhex(asset_hex_name).decode() + return ILLEGAL_CHARACTERS_RE.sub( + lambda y: "".join(["\\x0" + hex(ord(y.group(0))).removeprefix("0x")]), + decoded, + ) except UnicodeDecodeError: return f"{self._format_policy(asset.policy_id)}@{self._truncate(asset_hex_name)}" @@ -493,12 +492,7 @@ def _transaction_balance( ) return result - def make_balance_frame( - self, - with_total: bool, - raw_values: bool, - text_cleaner: Callable = lambda x: x, - ): + def make_balance_frame(self, with_total: bool, raw_values: bool, detail_level: int): """Make DataFrame with transaction balances.""" balance = pd.DataFrame( data=[self._transaction_balance(x, raw_values) for x in self.transactions], @@ -510,7 +504,7 @@ def make_balance_frame( if not self.unmute: self._drop_muted_assets(balance) - if self.detail_level == 1: + if detail_level == 1: group: Tuple = (0, 1) elif raw_values: group = (0, 1, 2) @@ -550,27 +544,23 @@ def make_balance_frame( if not raw_values: balance.columns = pd.MultiIndex.from_tuples( [ - (text_cleaner(self.asset_names.get(c[0], c[0])),) - + cast(tuple, c)[1:] + (self.asset_names.get(c[0], c[0]),) + cast(tuple, c)[1:] for c in balance.columns ] ) - if self.detail_level == 1: + if detail_level == 1: return balance.xs(self.OWN_LABEL, level=1, axis=1) else: return balance def make_transaction_frame( - self, - raw_values: bool, - with_total: bool = True, - text_cleaner: Callable = lambda x: x, + self, detail_level: int, raw_values: bool, with_total: bool ) -> pd.DataFrame: """Build a transaction spreadsheet.""" msg_frame = pd.DataFrame( data=[ - {"hash": x.hash, "message": text_cleaner(self._format_message(x))} + {"hash": x.hash, "message": self._format_message(x)} for x in self.transactions ], index=self.transactions.index, @@ -590,11 +580,11 @@ def make_transaction_frame( ] ) balance_frame = self.make_balance_frame( - with_total=with_total, text_cleaner=text_cleaner, raw_values=raw_values + detail_level=detail_level, with_total=with_total, raw_values=raw_values ) if not raw_values: balance_frame.sort_index(axis=1, level=0, sort_remaining=True, inplace=True) - if self.detail_level > 1: + if detail_level > 1: msg_frame.columns = pd.MultiIndex.from_tuples( [ (c,) + (len(balance_frame.columns[0]) - 1) * ("",) @@ -614,11 +604,13 @@ def _draw_asset_legend(self, ax: Axes, asset_id: str): ticker = None name = str(self.asset_names.get(asset_id, asset_id)) image_data: Any = None + url = None if asset_id == self.ADA_ASSET: ticker = "ADA" image_data = os.path.join( os.path.dirname(os.path.abspath(__file__)), "ada_logo.webp" ) + url = "https://cardano.org/" else: asset_obj = self.data.assets[asset_id] if hasattr(asset_obj, "metadata"): @@ -626,6 +618,8 @@ def _draw_asset_legend(self, ax: Axes, asset_id: str): image_data = BytesIO(b64decode(asset_obj.metadata.logo)) if hasattr(asset_obj.metadata, "ticker"): ticker = asset_obj.metadata.ticker + if hasattr(asset_obj.metadata, "url"): + url = asset_obj.metadata.url if ticker: ax.text( 0.5, @@ -635,6 +629,8 @@ def _draw_asset_legend(self, ax: Axes, asset_id: str): transform=ax.transAxes, fontsize="large", fontweight="bold", + clip_on=True, + url=url, ) ax.text( 0.5, @@ -644,6 +640,7 @@ def _draw_asset_legend(self, ax: Axes, asset_id: str): transform=ax.transAxes, fontsize="xx-small", clip_on=True, + url=url, ) if image_data: ax.set_adjustable("datalim") @@ -651,13 +648,16 @@ def _draw_asset_legend(self, ax: Axes, asset_id: str): mpl.image.imread(image_data), aspect="equal", extent=(0.3, 0.7, 0.8, 0.4), + url=url, ) ax.set_xlim((0.0, 1.0)) ax.set_ylim((1.0, 0.0)) def plot_balance(self, order: str = "appearance"): """Create a Matplotlib plot with the asset balance over time.""" - balance = self.make_balance_frame(with_total=False, raw_values=True).cumsum() + balance = self.make_balance_frame( + detail_level=1, with_total=False, raw_values=True + ).cumsum() if order == "alpha": balance.sort_index( axis=1, From 5bed9e05d22b78be3bdc07f06497352207be4aba Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Sat, 14 Oct 2023 18:24:08 +0200 Subject: [PATCH 123/124] add graph flags, fix order --- src/cardano_account_pandas_dumper/__main__.py | 29 ++++++++++++++++++- .../cardano_account_pandas_dumper.py | 15 ++++++---- 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/src/cardano_account_pandas_dumper/__main__.py b/src/cardano_account_pandas_dumper/__main__.py index b670744..2e6af40 100644 --- a/src/cardano_account_pandas_dumper/__main__.py +++ b/src/cardano_account_pandas_dumper/__main__.py @@ -71,6 +71,13 @@ def _create_arg_parser(): help="Path to graphics output file.", type=str, ) + result.add_argument( + "--graph_order", + help="Graph order of assets: appearance=order of appearance (default), alpha=alphabetical.", + type=str, + choices=["alpha", "appearance"], + default="appearance", + ) result.add_argument( "--matplotlib_rc", help="Path to matplotlib defaults file.", @@ -79,6 +86,21 @@ def _create_arg_parser(): os.path.dirname(os.path.abspath(__file__)), "matplotlib.rc" ), ) + result.add_argument( + "--graph_width", help="Width of graph, in inches.", type=float, default=11.69 + ) + result.add_argument( + "--graph_height", + help="Height of graph for one asset, in inches.", + type=float, + default=2.0675, + ) + result.add_argument( + "--width_ratio", + help="Ratio of plot width to legend with for an asset .", + type=int, + default=6, + ) result.add_argument( "--detail_level", help="Level of detail of report (1=only own addresses, 2=other addresses as well).", @@ -235,7 +257,12 @@ def main(): if args.graph_output: with mpl.rc_context(fname=args.matplotlib_rc): try: - reporter.plot_balance() + reporter.plot_balance( + order=args.graph_order, + graph_width=args.graph_width, + graph_height=args.graph_height, + width_ratio=args.width_ratio, + ) plt.savefig( args.graph_output, metadata=reporter.get_graph_metadata(args.graph_output), diff --git a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py index 6ae8656..df91d96 100644 --- a/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py +++ b/src/cardano_account_pandas_dumper/cardano_account_pandas_dumper.py @@ -511,7 +511,6 @@ def make_balance_frame(self, with_total: bool, raw_values: bool, detail_level: i else: group = (0, 2) balance = balance.T.groupby(level=group).sum(numeric_only=True).T - balance[balance == 0] = pd.NA if with_total: balance = pd.concat( [ @@ -581,7 +580,7 @@ def make_transaction_frame( ) balance_frame = self.make_balance_frame( detail_level=detail_level, with_total=with_total, raw_values=raw_values - ) + ).replace(0, pd.NA) if not raw_values: balance_frame.sort_index(axis=1, level=0, sort_remaining=True, inplace=True) if detail_level > 1: @@ -653,7 +652,9 @@ def _draw_asset_legend(self, ax: Axes, asset_id: str): ax.set_xlim((0.0, 1.0)) ax.set_ylim((1.0, 0.0)) - def plot_balance(self, order: str = "appearance"): + def plot_balance( + self, order: str, graph_width: float, graph_height: float, width_ratio: int + ): """Create a Matplotlib plot with the asset balance over time.""" balance = self.make_balance_frame( detail_level=1, with_total=False, raw_values=True @@ -672,15 +673,17 @@ def plot_balance(self, order: str = "appearance"): level=0, sort_remaining=True, inplace=True, - key=lambda i: [balance[x].first_valid_index() for x in i], + key=lambda i: [ + balance[x].replace(0, pd.NA).first_valid_index() for x in i + ], ) else: raise ValueError(f"Unkown ordering: {order}") fig, ax = pyplot.subplots( len(balance.columns), 2, - width_ratios=(6, 1), - figsize=(11.69, 2.0675 * len(balance.columns)), + width_ratios=(width_ratio, 1), + figsize=(graph_width, graph_height * len(balance.columns)), ) fig.suptitle("\n" + self._plot_title() + "\n") for i in range( # pylint: disable=consider-using-enumerate From c30a918353939dcbfb79e5ef45e281549f17b12b Mon Sep 17 00:00:00 2001 From: pixelsoup42 Date: Sat, 28 Oct 2023 19:26:24 +0200 Subject: [PATCH 124/124] update readme, fix typo --- README.md | 46 ++++++++++++++---- sample_graph.png | Bin 0 -> 97064 bytes src/cardano_account_pandas_dumper/__main__.py | 2 +- 3 files changed, 38 insertions(+), 10 deletions(-) create mode 100644 sample_graph.png diff --git a/README.md b/README.md index 9fe317f..30cd962 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,8 @@ Create a spreadsheet with the owned amount of any Cardano asset at the end of a Also, provide a reusable module that lets you turn the transaction history of specified staking addresses into a [Pandas](https://pandas.pydata.org/) dataframe for further analysis and processing. +By default, it will also add synthetic transactions with the staking rewards received at the end of each epoch. + ## Requirements * Python 3.11, possibly works with lower versions, not tested. @@ -35,6 +37,16 @@ If you get a [blockfrost.io](https://blockfrost.io) API error, or if execution i This basic usage just lists all transactions that affect the specified staking addresses, with the total of each owned asset at the end of the specified `to_block`. +You can also generate graphics output: + +```sh +cardano_account_pandas_dumper --csv_output report.csv --graph_output report.svg ... +``` + +that looks like this: + +![Sample graphics output](sample_graph.png) + ## Advanced usage ```sh @@ -71,12 +83,31 @@ The checkpoint must have been created with the `--checkpoint_output` flag. `--xlsx_output XLSX_OUTPUT` : Path to Excel spreadsheet output file. -If you want to further process the data with [Pandas](https://pandas.pydata.org/), you can serialize the generated `DataFrame` into a file. `--csv_output CSV_OUTPUT` : Path to CSV output file. Specifies the CSV file to write the output to. +`--graph_output CSV_OUTPUT` +: Path to graph output file. +Specifies the graphics file to write. +The format is inferred from the extension, supports all matplotlib formats. + +`--graph_order alpha | appearance` +: Graph order of assets: appearance=order of appearance (default), alpha=alphabetical. + +`--matplotlib_rc MATPLOTLIB_RC_PATH` +: Path to custom matplotlib defaults file. + +`--graph_width WIDTH` +: Width of graph, in inches. + +`--graph_height HEIGHT` +: Height of graph for one asset, in inches. + +`--width_ratio FLOAT` +: Ratio of plot width to legend with for an asset. + `--detail_level DETAIL_LEVEL` : Level of detail of report (1=only own addresses, 2=other addresses as well). By default (`--detail_level=1`), only addresses linked to the specified staking addresses will be shown. @@ -92,16 +123,13 @@ The muted policies are listed in the `known.jsonc` file. This flag disables muti When a policy, address or asset is not known, it is listed as a numerical hex value. For legibility, those values are truncated to a specific number of digits (6 by default). This flag lets you specify another truncation length. +0 means do not truncate. -`--no_truncate` -: Do not truncate numerical identifiers. -If you need numerical hex values to not be truncated at all (see `--truncate_length`above), specify this flag. - -`--raw_values` -: Do not translate policies, assets and addresses to their names, keep them as hex. +`--with_rewards` +: Add synthetic transactions for staking rewards (default=True). -`--no_rewards` -: Do not add pseudo-transactions with rewards for each epoch. +`--with_total` +: dd line with totals for each column at the bottom of the spreadsheet (default=True). ## Output format diff --git a/sample_graph.png b/sample_graph.png new file mode 100644 index 0000000000000000000000000000000000000000..a06bd83880a40c03701fe32567f1dcbaf5637f25 GIT binary patch literal 97064 zcmeFZcRZH;|37?6r7c8~QA%ZGgbJY|A|pFwRmk3ZwUJPWkWu!^US%XJgk&XKAr#r0 z`*~d7@9+Qn{`0;cx6kK#TrTN6kK=v3-mlkly-(lk3etObG3+9dNPA_jN-C2`TTMwM za@8H%@skPt1~n3i?2(Ct#B~`7i6hpwRz@b~h9uId7alJ}uU1}UtkzShR_0)J+@JFN zjd0hMD~x8ePp=gLKed4`|^2c;|W2L@_ywO3SYYEWog+P_A!aj;Rge?h$& zMyFNfY~2&+w8-odO08c?HzKT7;ex@pZN4nU@A8)X?OF`) z8$W53bB#VP$3^$y#@H?OGlD}VL&>RQAv|}q({1~P($!4UJrp-SUDwsq91(8Z5p<)9 z`nb>a>k@}(zx%YL2H!d=zq*rh(`O>=bHekqJyHSNRNq{3Tp#fI_dz?2SFlTV7u&tQ zgSE@__hn3EsuWRmY9=XSa`cv9l5 zp~g(>fnD1M)}NKJQ|#&8QjxE2_1ELI80pYBhXXkyU9^Lonk)}dNt>*u9Drq zX`4RY_~-~8%l2E>q$Nq4#Q$PTlY;S+oz_=1?eGLUiGRp0pNe$G4=L+!Nw2kw*J-aBAI zC)vhn`!Z9OMpbgF>i+9ggRBqipHmO8GEv2MK6>;hbicApcgXtM#z5C>!yPAy0qVy2 zGQH+t=`MB=yAghVV=<9Y#=}(MTUr12Yo(#f$0v*4=YPL8Tno6i{O=#VO}+2S@PEGN zc<}#!kF|LF|F6S;74iR`b@&v0<)Y-=BK6_JepOYHqh1BP@>5*sJzDq_%#(BP7gN$@cIzTc;%wxl`*7V~&vd?vO6}JOLpDgyd zQIh@zdaaRXXJ;p?rO}nGjB}2!Y}$hdX!f=x=VU7rD%ycpVj`l%&9vXW7ee_Uze+ z<)EM->!GTh@$vB=>*{tI7#Q4bPkq_tPP%^MhL1{;!cl&H8q)qlhkWndBl-1v&5x_y zd~d&#bqggWW!~MEPvZqA*w_X?-X|}z8{Jn@TG|pXL(`U_eUS7#Jlr=LZZ{>?Uvr+M_2xXgN0 zpUu=O*R_~^?=T@L`RhwK|9EaVzj5E9aKFna6Ri^)8=F^GnZxg|m+-oVSZt%#3!|AbNfj{KYyyQuNIkioykA_!SV0= zsbS;B2sYs;g^bF%&MX6oV*9b62p(bKALh)YZ&-k0>%j*`Q7tVkL%)8-hg)^!7#j7L zy3U*K+_B^9mxHvlhBhL*ckQ~T{-OQ?_X_IzOF8ODA2GW)L`aUo!6M7;hj52#Kf3M z*Hu-k$kpm11WX=uffq=AJ09 z?0t&j7_9Q8p%-&MzmNGsatAI{v#7ZEC>PfrVhdp`c{E+k5#s-}kZ+_}#2 zl4DZOK4Nd|ApLGnJIL(zhp(sDj)ISmZ$B03-iHq&5rSsw3*Jh&HWpH>OXSqQ!Ag1| z+vB)0+{xE`GmZ3{#ngD+i&K52HXCa$ORKB1Z>1#X#Ghc;ye)sh;<-e|y5;cU!)<9A zlm~fqtG*>%<1_hoTt4^SpFieYI`0qE1~5rDI-c$6?N!Ru6}>QBMUxt?Q_%x@~(-_OmBbxz0q!yLXy(bX3RBbEq*-0+8yQO-2Lv|yB#}s zUg|INXk-qpm%Vna@+mDR@!Ii;iCaji8kqt6n1qiA3v0d9R8|f$z4a|gk>$XF1NwDA zY^S)nr8?i~@ZI^mztMH?c5?N1H+}WI)@e00HB%2&e|~Xn@b~W>1D3Ns3D?4guy%OB z#M64R_wW-be}@lA@JaZ-V2K5ZBH%?Q1qQ=#dMuSs_U692cP5I0-?&LqL17;w6B8dU!`am}qVeG2!@J{Jc95Lc=R5gs zRqY^sMD;&^{`{WbL7r5XTK~h|c%jtWvlSF9v5tR#T$Puvek0-iJSK+n;K75q|Lfbg zZ8P{;;~!xrrTK9C-cQL&al{i9b{M}QKglFyE6HZEF}1lt*_x!N(VD0j^KN0VMVdv> zY$eZOLfw9u39hS#HfWg4b%+I2`;OP1D^q3vhDvF|ypFP{db+c$c%}Iq{jYRX= ztH;60y5h#m;IEvfewyqE$|uxib|j_q_qFoNdN*%!DJdz@>D95Ew~qDk`86^kyS_4i zx@q=@S*A(Esh1X7q>Jtz9%?GjMj9ivZz^J0INRn@>{Ym1`_TZ}OM*|x=M;HrYiloe zmT#z8wkDOjEoxar^P0Au%KwAAZX31ukw=An-B=Riu@D}yh}WS$bjZXeik+REmDLul zCq=jPeV4tPo112dbGBCJhp93%8_~YLzQx|Tv8Cys2 zRZ-j{Sw5yd`Xq+#4F2n7_uLpJno+mzik7}9x4N=2D^V=x`xpDYtg^>~EA*f1mb&gF z+1T1D*R}=Wc}&mDC=cWD^CT%mQKP=dj#!mUOBE0Ar=vSTs%mdh{3em9>mj z2@0x}y4)UjUizMWyU2QQ%e@a#kNV0y4rAw47WR9l2`9=cDAb@K8Sv77E%R{0I2gj; zzWY7vsAYed|51r27bSTOK5d;FZ@)I%FpEFp>|x}@{!CNP_=IVGz0}oVp3_xJi-8nt z(wcaq*p?0L;@RAs{cCiO!NEb)Wvaw2GxV7!#df1Zf4Xw7qOZQ>)cDt9*Gxt#Tb&fS z`N6SqD07^EZp3_@U#RKS^U6pJWERyD%q4~xhNO3th|{gP-z{0Wx#Xlhht6z8^YSh) z7bg&1chMJMXvx$Qzgg-^D;>yu<>EyWH8nM_(HDBsM`BWA zz$6dTaoxE0c=y4f{7&7puZ!P*sHZ=*pB<5JOHrXn)5xp=DBMZ%@$-|AliTy)!GkR8 z!ONKGTS(*WY2Ns2w+25w{{16=LPB&?MOit^r1b=sUKyinDKqN>JjBBo1L|Qe%fn$P zc}i^6FA;*fDJYpGG04bBoa*TxgV?UM73;5TCMTc5NZAHd^YPgs{xHx^!#h!DNioU;zBuJ#&s}z%qgYol{0g5bKh-Crh5AH>A#x8R>=xaPiq!hGTpj; zn~W4E<)8Y`#%+IM;dToPiJ@};Jt;NY+(!GOv?o_(E zy5klnTD#Q~8d|!#4hcu?BdON@la!n*e=Hw^%03}iM~^$0o9LtjewV<8+D}9C5S9EB z7xzdZNu?2kq~-0EBHQ6z1eW^!`|eBUQ=-jy?fU^tB5!4aPyFsLFE_Q#a+x*yH9Sm4 z`jKby@Mmoxc@efXI@)==kv-hHB|c&vD+B1U#I4`^5Z8mopzd5=6UY*XM@C1oUSDx2 zS@>c0xv8lJdy9&R$sc=ukoXHFF8t^pkB5eC32MpSyvew0_ii=2xSa9MOv+N-K>5hC zgDdkBpPQSdZEX1NeUO*ozFBk+v%K+)#FITrjT&dI`ko1zbsXX2qXO#1@71COs0q0* z_By35`>@-fm8Pwv-tEd^d#6_(0JIXP6emS&t3BJv38>N%C!O{aYeKH5s3>6aZEIuX zIgQ3=%x!kfqTVQF7E+eSsv}Vy1Qxq=i43ITG@HxH9G;ECm+biPX=4+&^C9DSdQ^-)I0^d&izVRrpU;_R&pj2iza!%j=j044AP`fcrEV zOHXlfUK5o0(^H&l`edo`T)#d>82ZW*h5#|1*XCLkQ{_-?NliOj`MAx{CW!LDSi^-C zxShSU^&bA6^2-{~QCUUhMRat)t5@{T==m0XU;}SA&j<+#-T2^mniLB>Ha^(aW?~3*i>vg$^l%sRMAmWKR_JlEf^c(z{u zYN&&jGe6O}8Mttoi|fnK=1DfT6z@;lcGKOsSx{6&5Ad0+o)J+!MxeMSlq{C>2EVc3 zMlmm$9lwWs=Ky9!kt!=ItG-P_^QK#MFShylWaLt7$VU!s%a6UD>Qc|>@1yhG9&bw_ zkuc;jGAYndvLf!iKX`5R2rFQ$I*s6It-O2Q-V`~xx!C}KjN+b*3kwTNqvD&sWge@y zKnOULS`+0D=STCDEp4$Xou`wPmF0TZIcKDL-dyP{CnhfA1?T;>fh@I&@{!hme+ZUr ztW40KHGgG#hMa`ID21A6ENh)hlyr$|>$@f^`*dXRTUCEQi;0N|hYSt+6__YpSy`F$ zSVGu8?3o_BUf{Ds=vGR(M)FlvRk3JjOJhkfyw(F3(;7k)0$xR&3I9Fs`u+R&i;@7l z`Zg|BR#x+y&0?$RuG-0Y|NdFy*4@Y2QfL^L**MQoS^LwF?;MMI`#$x1o-&+Djf?5?_=I_Pf=`nE7~A2 zUx?;LY`MvvV&~b$b1ZkeuhkOT0oRE^X8VxW-wX|0-G)%TpTB&`doMTk>tkvCx49G7 z4Npy7DAo`P)h&pPz4$kiU0r^&)MNE5UL{WEs^+&h5)^ckti4bfL$e#R3x&i6@aI)?`p~(e+EJ2ozvfj zHQuwUV0ShuTQxm9d`^MCv7&;Eii+yI%gk|1L?vbAEg~W!=$0vutk76o9v)*|VW|=` z#bnza%B_>ivGJeg(w?kD{;^jt1ENR(f#)@Iwr*@}(DU8eI?@#7H~q7gyl9o~l9beT zVg|5ytx@Hhwtw>DDf3yL9sT)ILsyB?7Tx7{M@FP6vn>i5oPi0?iECwUj2ye+Pj?|D z85tQVuY6Y~C@+Y0nsxtfC~e8&!BeOjs`h*F^5tV6pB)Zz1bjlD zOL-)J=gx7Am2{~Z&73?5mZqkL6j;%?(Bt` z-oAaCW!A|(Pv^_71c39lptZS~^61f{c$c=haQkxEbF1$GUR@U_{fY4m{pRbK{dtG+ z5ZBo^_T?-Mp{FaeZ`WD_P^C?9uNQpAmDw!M7|N@>q%iyu@csLZKi{*-u?(w$(k>ik zHO~$a*pNU0z*bv|R%@OloRCQTIsH2>a7&TMk9H78eO`J(6-=#qV4}+1dKZK9it}{! z!52Y6TOL;21MW^sG)|CzeO`%eNNo8R%}dqnMEe{I%q7xffUq&c{f5d)2|;p3exqkv zdO%5Ea}2K$tksY1^ryF1`~=pk0RFd%I8CZH&H;W@{p~4M;eY02YkLO0#ur#ORV*0& zq}@32+{DV_RA!LP;j`xVe2$zwO9$jmPLhwgn&^NRyb&dmIK$l-a-q4t{?llTR{+Ey z`Irldi8_E(;!oDuX{gqnKDf@Oe(&nqNg9LlRDA4|$4uSHpjWSwH~vmd@t{t|`!UCT zpg=7D3>0ts%y<9({Rk@iAB9$#9ilcvlJmt1Nx@qAroIm!J|sPljXfgRk20(Vfg4Yj z9k#Huw$AFh$`2R;vi9~ZYvXTKLDRPV5DZe3;-0Mh!NgOS_HacTYBL6;qXBMd%)t!d zJv?tx@nD;&h_pXLf*8-KQ`h4-!=o+KR zWJjPvD5Hfrymk@2peP7xmg5NkGViDL{CC1puDDfhN5P(n&KjGeH1r z`EvI5c@cDw?JO-VBhK`J#(D&<2-yF=K3|-almx#eIfu_*mW?$B+EMOp;^AxcmP2OF zRr&(O0$aLzJXtY8nuk+IjNje0-4TAA+nh z*ibDcE_OiKN5#ciq~fSR_ncS8lK4E=oZA;4qosh(ZYA`VcRGjBB=5E+`ZHWgO^4oh z$JiLU;TEVpPw;EszI_{*nDFDfT|+bqP`rm38K()6k6?CLS^F`@UI{yJ2s{4SfwGd6 zl#I3NFSE9_y(}xc3t#)0nF9bcKK1r8K}=GX7rm*Y!-okj_llRyX|m^%q~umG)hExN z@BcMAdQ3>@5UzzFajM3j0g{2cFs`=(v{RsUZ{50e0I*WSiUke!4Wzl>UAZUU#>EW| z4BW$uyj6@51Li?%lu%Nlf?WF*1EyrYL;G%b!RbGL{~B0XQBqJ)U@sk@qhp0?qj+I@ zE4ZKjy!=iQx|yV@X{s>YrAwDiojUc=|M0o}R8;o?9@1_Hzka>vQ(-6=8zen}`M}dVO{EMZfrR1E6mN*K4cAN1w zcA{c1ag^iU>6AEA5in-1MY`edd0|j+v|?gC9&FpKZ*1(1@u`}r%S3QafW6EJrkdK? z%Em@{regz^o5IZ(X7`{%u3x`Sic;{#GbW~I1v;>5?I4f{{h2#E=3Gl%%{X{@_e003 zxJO35fB*j2fb6kXE32zN0gt2r+D+S%@BjLB2kwc+e56_gWXC6a4sF2;Vp|-brLDjs z<6=MkstcYbWVMB3r%vrajnQqQaMW7Ui3QlcUF&tGeOG9o<25fu(fN=clVGc?cm(0$ z;h!5Duk^lmAV?@Fa2r$y9^{JoVi&w9FAP#HaEJ#k?q4DnKC0AG1$Au~Q4`?@^>+5sQiwX+T!sMeX z92K_#WPtp%m-Uv7D@5R-p&^ut{@KR60z+^XffAln8~RHfd#79a6a;sdRiF=H=D?Mq zOTrdBOuS-U-E!*?+p%NhJ9q9p#l@A!ph*05OIv{I|H%S;{+#u`5js2sD$l#M3m`gI z<>b~HniJgqMpo!wgoXgLdfUbo-3)KEE3Ref;StB{eQVef3ud-vs>b~6(wa{zV^ zfUqc@DqaIww;8T~$w4Cy%?QM!&~+c#N35g`7=~XfMx`}i6B@ecb$2ZhOWcThL+^p$ z?GFbPKO})%YKnTxW!vIc{P?^w-5d=Jj^ed%%`;e8gclW^M^o1>J=8X1w&oH~E&zFP7Y&n;GGL(PlP7mZ+$DifqUs`&og}rien_P8^fv&iawL?=Nm5R|y>_Ps46LfQR=T95giuQv&RKH7*=qU158{NY zet}+3pbth-S5^K-KrygI$)r{p@HsX7&o#!;A z3or0L+yT^s5q?#Eb-q(C0g?|W{&7!FPvs0PLCs>@VXvF}%sSGEcWN24hYxuSsOLEv zzF!gK;+0UL;hoZy-C^j@c&H*Pn1XNY96mG-Y*0Ffk`ElZEh#%v@%KE*8F;O2E zD8>QaP5nO*=M2Px#tY?}8y~h(P^9I!Pn9mVF5%L3dGmvREoPpAZ5d2Ishj*1^NLvk}~e!@T?wChJg9^>m;9+yY9aY?S6kfnxa=; z_TwCgqUL-16r*V9EWTjvZK-P4FocJp*{QaAE)O3}QA-OgDk}2H%*@=nZCf2U?IfpI z101$s&_2{*Abgj4h@kA4x~3}-;ZKtUyux_&kT0tH@+Abfow8)(=oosg-(AJ-0_4@> zXU-^BcRoKVp(=Os#0go@==K@dzG7?tDp#L2=RA{&dj|Q~UN{PGCv)S*tJmUQy8O*G z{)gF4ow`a?4T{2b?)UW9h7(7RUcw@_6?}OA-t?UtD=SEM7~rl9aP4n<;X~&vUmI!N zxf2g8boJuJd&GqTj85V5&(QDr2Hn@j&W;A8%j8F1%FS|59eK)~JNf(o(_&;x^{z-u z`^Ck@VgC8xUvpFade~zEd<#oI1N{mM3qiM`N#;zQ;tb|-stMQ^H51cml+3r-OXSmYb0H9XR80@b$B5}*nBg^XMZrdm z5p|d*0(_ezkyfd0(s%DBzZQ0omEWut-#7{gOq4OPQ)*efZd&#fy-6%zn`uC8RW&y& z0t&{Pb-uGjG{Cmd1lNTUO`9A8TI-OD5q3z0mn;Kch1h_e8_T1z?2+gCoGa7Cf?mH) zPlStJhfzgrfGu0L#6!J|Sowk^0FQ!#0#S6RJzRdqP#WvBrnO0ecS|GhX19Xti@PasNi;xKw6+0JK$Y7noDS+1RcC zaj@8PCmir>-o6hpzv0i)|CWcXwaID^=fK?RDz+h4fZT`w|k2ET}RxK52~c z1X#Z_->&h5H!e1NZKwcFs+;&m5Dg3+BYweRd1iQMVBiP_ywa#T_NDvk-$GMQKenfoj-$Sqo+#&p;>bg1^v?#T~ zB`z}nIdtGy>ira#BhxRgLFce`t3sSB_LVfPk z=OX>`F`A)l2a~wxP2^sV!TM)q%`vFlo}HT;3;{>&l-TLznFgK&FtMPQFWK;tD+_-) zVJ&0}du8gD-T?!6xl#FmoyzTJ&EB26cL&1|F<}k@%;rRC?%Dnj=^`!BqT%7;I)nj^ z^Yg1nO5&oR9O|(ta8Sn>MhV$T!82&jCq~nMJbUlzpWNo-TwHQFIXP`E=HPRD;cL7s zdGfWQ;^KP1)F)BAUwbA98+psEwWaBq@3#X}#&#d%R$cr9v^w@Z`@ha7A9Kzs8YY>V zq$J$Uc!C^=8OkUry&xt6cIjzWR@i5>c!cxTWiah*oSZb~eefWCp#a1uC5g}FXlCi( zf4U5t&IB&;9tMG{@bps0`wF^ z2$4CEsHbOV`LDIT8fV(vSS-IHc?bI#Q#nPZNZcIkM5FTyHX+gY!7Oxo&mql3JmK~I z`LN}7?%AW-84XQNNhOI<*gjbRw4rASuC)iy!`&8EL=gP&5GrrIHA3DY2hIo-Qv++}m*%7_jCO%}!`3`dxiJWPFkI7mmfX=^8+sF= z+ccn-;;;D88Tm$dtS;2SNokvhS#yPwz(jIwU-j;E7g*p)zTFUoC2~m2y=cUsp|4LD z!knee@>KbzE}~Tp8EfZE+dhxaLBR4$OG|Urr=l8bO$sC!EZry%7uNw$;I2Zeeu>lG z@$tvWVwvN`(Iy-BYD&gqpyaqO+on-Y!Y5bvlIItd4E2;qG~d?P(V-5LD^?8F{Nk^= zlQDVk8&MIwtF&b~?dUOt#Y;sc0;hyo{~ z1v+A^td~VeK0dH}RA5S~CMZe%LN4j~=g(KWO5L0i*i7rsF!&Gs`NIX2uR+;_F_8M{ zIP13*m6JplY>(egYUJr1u1-pLloS>gHj?q0?ON#FAa5_nw>3fV+Lik}7ImnKSGPYW z%<|u~RLlP=DTfhjDtzw_$<`2MPYa|B>L%D33jGHuKc$z$5a;hQ|JslPLYryV&`XKG z9~BjaWM1+Z(r`KO3cioe;7W3^O1r9)*wFuHAS0xCA$D;Q+eyv#eI6L34J0i{(d&w* z1hp%6P=2>Zh!NMmw|>~Fmfpk$~IC_hEV=Eb`qH* z!F6Z`TR#5LSgOx6YGjz6pI6(QgzE`gs^*XG?Y|jM5CD+@W@(>qe#IvNFk|Sjg6m|( z5S=YBF0}0Zg$U0p~>NbW}&cDZq90g_SVZYiKWJu(3LI4U&Z5q6r{jSBXn* z_3-Umx2hmPjC(l!{gF@WNi&6YDSVa8M~IQw7WF@{VT#_rPxYcad{z#AlYzP(w}&F9 z1qOM9o2{mv`*gLa*H?aG$7{Y#*I5(kP*yc4?;a@t*FwS+S(0C#*om_)u1`)S_%PC?b0r^tOoP9 zLH-iCgkOsPfpJ#YyTl$hJU~sog@-3|7t;sq+Bfm>Tao(AJ}-LW#CF)~5lfui0b<9kAyJ5=i=p#z3T~5C|n4Od=Hs#r(#YVK9PF zuuoDv{QUhNBZ2{Sltc9_k$g~Brz2v5ZjLb23_nH zDkSw8+A+c33mORm_?@K@v)3BhY1(31&a~!>C8TRE1Gfuv!{{O|J+6_3fnL@Jl7gQ~IbuK1U`-#xEJ#S*!bGF7jZ+n=u6ewKF;F>P7UthfVN*UIP|pzun+piqulo*UFwC}oeotUV4%N{$ zE89%FYnh$Uc9MH?*sc1f94vb?Vcep2#pYJdP^}yP(cG#^g*vGKO!Gs2#DpUabT48V zbc4&$^241vYQly%sqi|hXal>E8o;^|mc8C&Wh4M+*P`aQ{UlTe#260%_rKwwqO8Oo zA=Ff4R#ZiABqbyu2v&4?ILm9((;Y%9jI?amTu>V^h`<__LLsM}pswdmX_tG75cV^~P3^gaDP(t70b^LQfjjuT7w$nt%={-!s2jF{OK{TohnTyQ++3$UbePT+f zD1czVb-v0-P5l6+5QqAO$eD5C9j6L}3L>}&MX#d2zq}_Wd+saC!GqqNx^8>*JeLzr z7-;IcYst2ezPbYepF~9TTWc#(N_dQ{*ko7~B5)k-wc)b1HrtHK(nttXf)h=I`H1v0 z{GHTi;BF)Lo0PwhUxY5XeGmOTBo4IFKWja0ej06@&^!TxGKge%Mft`&kAPXn9^mqe z=xo-LJwm_j--DANIJSqLmxNsFbzjPB(?7??Os0Ee6%`2=_#YZo9C|e^Z~+kE^Qfo{ z_QQ}?A#$@q9YXuy|FBR#Q_tx&zCMnG_>mENKYruy#Z7}{1VF9VmTbUt zDI@HX?7Pkj3x5Jx94c^!JwU`&kf`H45JSm&`t)901g)_8%RylIS7!`&5x}`CF~?WR ztUC|ggG(!K2a-T2Mxfe=>QOI0JiKjC5UCMiD+|2 zIfB}9QNpyu2R5ST6}qZ#Z~F z9{81*$NR(wmKGoCLo1OJAp}#<#eLvL1+S{c`pd=D({J3DX7M;cf|r}d?v0G(4K83% z^W3S(^ERvtlf8a$`UeIo9^#c;6KZ93b(x8v666#E+|nXfTuv=F6lOxSkM>+K@q5`5 z5^}(PvgZ(Xb9Gu~S*jQjqvVJK4y%CMuy5bK2-`FMr=b2kB=Xo8S(m8O^#=Wu$aP~2Ebpo{#`IL}GXI`M22YI?yWE5;Mj@tF zY2Ysr&LI(@1)Zoyjuf>JBpb+rcnXLX5lDlR_BBMw)3eOz&aP{GR!H#`TIpdh5NTW5 zfad==Ma$|`x!#+JNDM~cRASlS0uUiFL^PLB-5d$#5XrxOaI%6>F#w!`ti+S%ooLE0XWU(GTB*&-_Fa*;ik88Q>c!7vCK#BC5*3vwo82t?P zvOe^-ad{y#K0Q|k>&4U0HS|1a0jH3P$>`Gd?-;V%SX)kg_GBNE)|=H&xW+)-gy^<053EFq#BaV z>Vb9N6^GH%(e=&#=q#|%cE@23BB~Cb8ES>G{n)qaTU#~_AXu?OE|LIrX9E6jy`UP# z9SQ#uf$mo?8;Rf^5#q-iTRS*pTz9wZE75$}0E1Nb&ENFj9&u-g*{>+RE&!`65HO<8 zN#`z-E`)i8X=ym!SPF7?``70e5leD1GA&ijG0P)g*T0v#or6B`3@tCR^peU8JF~_J zL84}|!EbX}NDVbfDP`_x3-Vn(&B?dw>gs+Yq=@KF04%_8$_tWed$qlkT(HzUsFet% zXL4W@@nc8MfHXt3`Gv`&5@XR*q=a4AUiKgP7ZVdRT0e;3Z#`H>o1TqLwt07fJZ6%B z$Fn_*=dZUWZh$8tvdeB&wmbshk7o!e`vvHDgnT4+oTj%Ec6O_MpBr=nLJuT*4_&9% z(mw;}Cua}0yp4~SRM`jRE|oIqTa!M%0TJM}ZE`AXDsT(V?QE!UlD(&-v%xT$3W~gd4pc!&jYK{z;KlBhtxDhYj5FL8wzI_Gn*HsebUf?&?OxHp} zLO!E^CIC?zt&2gN0P68i)Jw$jmo?Sni_y@~s2vKI`r!b}{2FDNtgR;~dG#nIBBHDdMrv0exm}e-QLqWp@q$}?V3pX}3h2nuU zpi7-z@KDoruR)q5htDCD;n1O!v6H_LlZ%xVL8JE5hQi)6SB97@s{eFIa)rc4C=ppuZ8gw?> zNOUiQz;kpqHPb^t>O>z$p){InVetgl&m<%z`GN8n-H8Tq_l0&jBBu8Pxzc#B3!`HP zff}Ukc=#Foa}WlaUR>;~x&y!c8H;$|xZqo?WDe1{*dwtNb+*S+Dyu?J|oXwlGbpE)N0}ypd-Sc5*>K5FC97 zJ=SRJbrp{;t@5WGL#@gl#jGA}J?|Z^!6yI3vbjXT;^>ol>V)a%RCFa@2-ijeZ@v|n zcd^0)NDW&e4nckT^hx!K7hWH7QTu03!wj7d>NvO5@OOW~AAez9-s@~4P$;XRfBhGU zoQo7RGn&7&c?keq21{%Rvs;}dh>3|wt=b+H1oc(5wis!oFyLAF+JXc^1%j$Fl}7Us z0545R7u@t1K*tR7T*;55#$HIkIVY5Gv892BvVOF!5AWb*8kvt@)5wx%ip2y_0XSDR z36kjDz%l*--)}K{&1@B^p9W@Yhp2kr__UYkI`b&!2!gnEAAu z=SEK<;&YA9Cvjp2LAQhR)*-K7bD>bOewZwj4z&gW#nkgRUpNNsg%Is91!oAkj!i&T z{3=9P35X8uJ~}Te?h4dnesW;k7YM($!d0pTaYKd!?NRft1%5fB|B++@cv-EaI7u55 z%B5W(XWIELgSkFQF8u9Si5*NULVgvb{)%1;KB>tsH(!Z^aHpUdgn|ug|GigYQ2B&I z#%XoIf;e!%kb3UiISpp7H5-?aaASd}0(AjMXVBf*I5=cq4r9nAkI{G9>)6`a<-XdV zyeGzMqZm$!Hj|6lmsfn>fWRR6!NX#ORML6jd+F-r2W1>fYP4=$q=_+P5_7)+O5T1- z2A11FadB}$hVI=wB}XtURVdXkajfJIYoNtNM(~O&b9u%mW?bPD;h`z7D{qF=!+peP4GXfr| zhWkc9OV;iw{6_m85F!PF=uFT~niI(;z@w^Gt05cwY!~}oCok`jI?faFR_Ny)-b zflF|LqYpJ#)zYGbb5#krd-Q}~NMd9!h1OTv;v^c`5nOJB$Eo$*&>T*(p~pbUjz>;~ z(7)zM`V9GL=4;_nO<0>&(Vwl#58@FfpgwTmIN(?a49Pm;$j%%l9^i60A-v+zbL;Br z4hOK#$2$J`ZeY`W?c|8#RNraf3qr{CEiW$zjk?H2f`oE{MEGX8CG6RRDca8u53KZ! z3J&48AfAzctB5F!lAPRU%*e=SzA#~Fc#4CAnnY}cF7Iz~lSDrt+(+Ul3}DMuNy$em zF9yMQw_F6-GjeG?>_zLVcum)9-2uQNl&Avm$8_CNE%|G5av`9cvIvdDgRkPT+ceLw zC*kj&gym_}xl3gxxv1y{Ax0z6_AOP7b9#EZ~@HU013bK)&flR!<{@(s<+2hlPJ8xO}nC1AOwxR4JTq(Vd~o_~QZIQI3;!?tuy z73`W5z;B($4RO_)YvFy)DOcUK;GYs6mAQol;e26uj{F?~ZT9o__C7^ZxBLHO0rJc` z6;NNN%Z6uo{ptBW-O?FB2HZ3nnNL5cl|(zfaN&Z0OgM3rDShHq(6vQ~kpO_)cyU+- zYJXk^ycvylQ@}ewPEDT4e8$4`-tn)gO}D&n4Ix`!aL$53ld@tvR;YDY(DaVF-o9PC zQmx&!(kdtBU9$@d(Z&z9Ow6AFV1^7Tm6{;j{_uF6Px8!ne0l&Gy{0X20<4Wl4_%xm zlkZ-Kkbi`YZ5xCuB_@F_P9mo`xwzW6%x>R)h~+0IA+liF!QumKP_4k-uyETJ=#>zk zF!Uo9%8>Cinq6I8)i&q!6!-AE1A^zleBeO**jHf6H0EUcI^l z#e+kN<~F}xV5PPG?(b;puL0W|93Fm*3NuOUg+KxHaD@n9Nk}MRn-Gw= zX$*%20q}Mra$_o$QQy?0&g2;iNG^z&oA|$ML7W>v%@b#+kw`|L)cU)LU4MAT-n|~M zv@<>UWpJDi=x#p)L#hx3oQmf!UKkX(>+2u6dGjU*V~im!E$txeM8wo2Ag&2yhXw}| z?C(u%EOG8TWp0U`gX0qlq)`lAsdJ8oIJt=P)LP5OpXG~?P|#l}CJ=XJJN7-g;Q^?LK2M)2 z^Wk*hGsyh8!#X-TgHuy_gAneJHMuG)dl^RDmz6jSp$haeoS4{1dTsnaCra*jjRHq^ z*xz!8(t?ArZy^Ik&=+-ziXd6h8hF6k=@*OHv9IAh$j89Kl!7*^x*j{XzKAR&k=esZ z070gw;vhvb5lT~B?b4E4A&FhY5iE2vflhq##nzAv1rcFk#OI1WO>2fG%y79igMl(Y ziM*pemAr?D->ixQsCV>e(@;}e57*NYC^tgPF1_7B&Y}Kkjj;)sEn-#e?jwVPZ^}Bq zIfa%wZk%fi1F8J`WA8d7$qR?(-yc$A>`j8@-Y(t6x zHOdv~wx0^zR9!Y|wtRi1a>cf~i68w5hY+%OKc`3k92g*-J6E`w`nQ0p?Do}RXB?B_ zL$c+(kdWGs%HX)TV?0N$T)|14(Z&Quz{srZY#1|4yDKlR0ftm`g!0H}-8R+wyfj!m ze*{7#{{QFi@+#fb7-i@^NA5fIr`2U=$#5Zv zvqFnszKF&MjoKCc$!U0X&Z>`U`Jzsd^-<8|F&n*RM8|K{hp;1CT7Fmf<>|BC9 z*MDzD@310G&{DQABzuGJ+unPot!yP+E>3DXBs6@9ymIpZkT6xbPbg_ z9W~`H{W3ddeB2auAhg(Hr*`P?5<>8Znm*Y={a%+&D1Bq8R{WM`2cJ{VE|I7IyBX|h zcD4QQw?6;(zSo+U`LCFM(NpuDUQl&8^qMiL{&xj((NHY6f8^bR^ROl4p;^sMO()?0 zx_AGa9+-AM56KUosbJx>iLe^ZEwZ`0A!A*1a~G98g)rja#8>U}&BZ}FUELRzL3#?t zJ`V&%JD8k$Dj+7Yl&#}L`Wdujy6y{`YqLa)pt#U6W^db}X*?~yIV=7e;#kJy=H!Q| z5mAh@GX||Es39fA8w~A+y;K;%c z)&k4f$AJD-VqK15L(#y4-9$io-*r$NmEjHr;ZMCnrOL6pP>lQK+ z1n36HVuW0@+}*#?F%ps}WH-WVHzGGaK28V@%fS1JIM|#dzRnJsp^qc)gBEZNuihqA0c$%-(2a>5wRCPy3eXF~^doTHq+R{6}T`bMd@iVLA?rUnjR=|IR zFLZb9!tIgo8*8LXQKrAztTMWr zI`4<)iS8$+RLdl)KHwxdF`F?b5DK8RgKSQmP9L_n;H(5 zcl-P$MZamL_lss(Wc>Ma{5A2mgWosIPZ#QIYztMAh<(}g!LX|>kA`ta#94=pf`;`= zyQe&lc|@3*ZB;s-{I%lhTL!)TX;HBf`RBe`lG5X9wF~48qbMuCKdj8oXzcvHpEt03 zVf9C1Mm$gb&wAg7OMO)W+u5qe<&kV)wcP_qPJ_fC*}%#~rZK`D(2`Vib(?VXvCSYh zu@2@fanmFs`n0rx29A-j{i9&#%}STiD8jQq_Z~x+jS2Sa%^0a;6j^Zl$^q-2h|~fc_~$Iy*bjy{~R=tRtK!p2e@DAr0^Zwsf3( z@?-ohK|w(ziNYY^JG>%WbDnFQv?Vn$ca3J9Fg*O4vDQMx+orqCAf$a|*?yK;tSirZ zSfu_{WO#GWm$k1?(mKbE+aEdlqsu6~d-w?V58v9}jan`95SqfY^|_nVeB7`9O8(qa z8LJ_;+WgZ|$j9Hdz>CZBq^0p{?KjS|yMFB4^PX{gv5-(Yk5g7za%ins!%(JQj?iuA zGyRbvHR(3wK6WDQ&P4YJpt2kJvU79w{lvS{`DH8@+SL`h3xNY8nyvzma`w8|+#`h;PDFI#=&URO$nT_PEbBqG=+5 zFDD^Ej>s9W_}U<4Wj9V`JwjO$$A@rUu?SAXQhTN+E%zqm&*HJfNWu|?lCw6<<28sK zXHw)2IlBkE4g$ZDmR3DJNTULuJHo*2g{xSqQ}iT`Dt$g*Ktez&#Bu^o@6FC zlzjN-iFf`@rIiNz`xUH@)En$S*z4$$*Lq1#glQpQ3q5n;?y%gyK9{tQJ^!S4fa#g! z_}xvIW^C;2z950A*^|bP&3K%?FkXtPtXykf{-Uk(afZo|t6!;xh0gp|$I?d61-7VX zd6H_OK0fy0Gs5DFU-~~%HZv<|DRa3D&;QkR^2oEHY;X#h5f2O4t9UTea>u`Jr?b{I z{|*F+PJX3i@b6Kem~P>gT2TCWkhlBwtsX6fiV@+6?NOzLp~hw3n;!NvHUBcG6VA51 z5uR9QmRY*}A`B1WfHe|q`$qk?^jAmx9Ec4sslZ|uCs-c6D;H(PLw7*~ z(B89^__QXMf%_EpXAUF_6$mlzn=Ie-(Hi=?kdL7 zo#LMXBG_t7P$%H>1`$Ev4VatibyHjO+or0r>ggBZHvHTtSwv)Wr{?kRe(?1$Q z*pE#+Ul2ZI`}N8E^gqAwxT1*%FV>8Wh3xoemz;kxj~I>Q>dTWw-#tjy%BufkZRD-e zT4&y1s=y2GZ%War9c``;UwJ>jnvrkvZm%be*X_x7S8Z=U7T!srP%9`OeA+G6R*n1X zbjZye!(FET3dIfEjSIvb4t_i=Fug5Zp?qOIKeRSo!~8(DlESTC1?smt&Tn<9ileHE zqK?-{QFY!h>bkBuWtDF?HSbziN2&ZT=CxxIpHrePt4A*h{r_Ut&%eSEPZ;;zvgeID zbXg_TtfWG{SwhA?Q1R?vvX8u%#eb2uZlP03E4%B!G5bJp+j$1BzdN|em2@b%5vC#YG4K3oQO&v=!>>jGj~HKk3O}Y?v^jHRk^B4Sp^R#hq!81q z&^q$ngs85azpd*Tt&->Z-25d>Gi&cOO8^c4?6o+j!SwQud55@?O{e2uVX%!TeJ5_y-D7V zkL^nj{%~jP7Edfq`NAZxq+otl+1P6@S#Wb9ol}wZz0VcSSl0VrRaZGPOQ^kX_xmhD z0U&~o_>R)bh()v@*?u>KJOl74w zu&17>E034}Y}MhC*{0T=O0kKf#D^w0I5^0?K2Jq_#FSJ3qi=WV)GaT_P>>-WHz0n| zJtTh~ANo^Z)z8e$BJSz#H~Kr-9U%>i(8Et!-j)c(G)u!nUs}Lv!JS*Tw6(W0Ex6`z z?d~oJ{9Gn7k!JBm&+){bXH!8x(W+KfTuWVkOgL}fvgBT7*`oViN4isB+O^c8qmex9 z^=JG;UsigeJfd4qyEZPyGVWbP15>LvQL4jv!?RrzleQokR_@?+zyOa5`X(lYm z`|YlcFK|`rZ^>M4XWGiKmiV~vpTUOTVR9dI5I4cb=)fTnLTRPu12A z;l}8@#ds#O1+Tpu9GRxqr{Rb$^4jE z&L6`^-alkCa0F#K-uf`bs}{e;c4&|Iow|-*M2@O2og0{bPpQ;QZ?U zMb&pebJ_lJlcHgjvZ)kO$lf#AI~3U@d(UKrjI2bIy~*BtW@YcavdP|?_j}j#f8X=| z&T-E3^yA+5b$!2|b)CUL?G5=eDBVv+2h7&sl2G;obKc!bXRTmNVu>>93www>OHL~(8WYZ*MQuU*n z8yLi8XP*p>1mlyDO&A4s>q^_&vLYTb#9RLsXcoFw@U0s|gcUJ%>FV9IBY2fZwi8Dg zN+DS09pZ7sd{CK=R|_~$eb>Mn&?rHUpXWi2K)@wdUJ(@)A{gNx)ByfoulWl^4WQmA z9L|PJF5pl~RI7ndvTKXH^f-8xpO^Q{Z5u4vXo39P+!ey>5#{Av(ABj zt@Y-yv8+{L-67ux@*U1M+$JKJ48F|Qx%x7n5o6ceKcAfneuGj)eDTx1;dysYU_N2= zAL`;CW)!?~8``JX0h@*y6}ebfJOqJxj?|rboi=Xl%{?Hx!d1Ndj<1vPdt}v9hHA@M zkv}^&XB?ng0vV7+drcsCWi4AM!%+&k=9j@;K8L}7hzf7|AxOE36@^AV1Or+Qt#ikjq2?8B?Ir;ft0Qs|8 zFI(KMm^!0(a((gQ7R>u}#aZd;i1qVk_~POs+(6gvWawLfZz`&|0M{z%CNUt01lYK! z?UC$3AZ3mQzr#iDP`I87M0IWYtX42fwJj`ax_oM0)7rm7K-!xVieU_7g4A~=VNRpb z_%}b-G%EoPcJJ@+m*HJ+rx_UE?fU8zHLEW$s&#JBGF;EM_f+fksjWO)m)`n4gAxBf z;Q~{hOP%&}iR#Z;tJ(2%)Na}@M7l1hgx!&8!0-&Q#*nTU&>VT+C^GG97kq1KtDoAV zJ7-j|kE%R?+_A8&XPPA`@Qax!CmC7x%)GDNv=yOZ|4DtK+USX`3jiKq5GBLyw&x7M zekISX0a#LScz82V4oXC(RYPc~V4f(TzmVAoOwY&wgU%exdP;hQvtU7*iL&lAWV}>W z@i^1OEr*X9s?D9QSgYr;#uqE|nUrJq->a6M7WT=^hTg9laC7v5palj!i80ew222f* zdg32U*z`${!_Un%*Pou(rf;&EA_Sfd#qG3Z70sW3xFQmB9@sw}MbPr{G9sz#W2vuk z0A0E_cNF{!4DLlf%5U}ct^fQ;^=ez}h#Z!xs z?oiQK3@E)qwEz!k0SIQ;Os;lhT|gxav`p{zctG`tI%@5Y<<}bWf&A1-O*qS$%3S!c$;_5zcpXC-Qm*CmrwikM=UNCn4TT?k|v@ED4QR-kPvI znmZhf9*`+8tn;VtK@iVR=zBhm0RfU4y8w;)Jf!gHKuE*5-I!^V-KZf77$A1$LS(IG z1K7U=2iL*Na|B%!QA`A|%k6;9ciwrWhQwb0Xa5_LR&^bocY>wpZxsf5#xAJLkkTDm zHH$-KkPO#txL&-6vAPu`0V;3>5d9YP!CdbI-pX*Iq^3~+V45pssm$?aH{v-6*u2o)iH45e);u{rQyvDr81L8X%wAnb2#DtvrWuij|ISIYjxS+omp^< zIJAquq4{_NeLTkaRHzX8@Ap$DKa+;W}=EQ)oe;qmnJs;8mLS*V!Wcc_jwF0?z^I@uH*K!yJK=l+|Dw1LMb=Hbd ziZnqWfV8Gpfv56pmPYL&=wfrS{Mn?cy}>n0Y8=Uh=?p3c7D}0~;O&8UglpIPNzp;O zgOOR@9|54Q9bQXvN=k9X<=4>q1L5ZyIS)<$o8Pq`Ov8Zk{|1tY6;IX>g_p9H0YTRc ztGK1QX?pvTDI2#3*Yr0Jy6m;s&z}!zJR2!p8VuJb`$pK7jE7EItj3P)jiMRviqS)z zVyvfG!yS{4pN5%wzry1@JR30?)7u0!w9u%mP;JBALhL_zclP2NAG5BBKYSygLyO~L zAN3f5Ws|Uu$(}tAWI;AkHorH1Fqu++ri76!tbSZh>Q(nHQNh+7L5$!x+i2t-++-%f z^A05H_PgNQ@Z3DB(n-Mn6Mki^7_69-`canXXXs4ASj$+O`C%jE;W1<20+kyTYD1t}GlBH4F&q zL38}^?78Dr?{tB`$1pd%>sK;p*%G2DKY=T!=d+7@ydNH>NW)VIe^ zCfFF#w$P!0*+W&J7ESVoo&Np%htJfH66eolO?AD4nqOXuob830Ql->F`*0todIzw1 zd}lW&2XLIO7=+OD0Li2m>bhNG{d=V;n$z4rp`*}B+FJ5B!sO%tV!R(&Syz68b`LsK zM3uAyfFJ@mdX9<jA4oqo`xRtSk)($h=Y18#ET=TiDDZ*!FHI7B)i%{B zApVDJJX$`>-g4;<^vFnN5fB=;$T|DeAUzJ8Xk*1k&=MiiJ3yuZzZL#E)B$($?f{`~|BUhg2SfkdApuuI;o=7peI zD(Q&+KgBpan*x4g07S`Ov9pf&w|fdJ|2`k$;25F*T^(Jys+yyI>H%*t*?74Aw3R$1 zDNKI>U7NBhrOC}2)&!BH#s57TbA(zY)~c5O-v$KUqm7vg8$EZ#L|%{MRRZL)FaKQG zvTF@Z{P(w99RlM2j=Bj=Ol#&S|Q7Z|OrhScpXfmpJJpB?ai6h6A>i#>1nNTBfxYz$>< z#bZd-uNr!>ZeKH--$su_5zcv$nEeKyx4HazLF}17XqzG+8R^r8^Tzwr?eH2Cc^7E# zEx~490Fr0=vbkHpGE~ddh|DqEG5enuAY-e3YFRGn?*Ng+ZE$SLuJu^%L9c?r^A2q$ zc=1Gn$U7jeVq}4BqjCqEI?CcF>OuI|uUj=2CpCyM464VEpdVs@X$4%KEil>2K*|dO zV1iAOj);f|h&3koa=mg#2H_4sLxz-62*3s#y95$#&B7llW(ji&+W9pN=jj34hoHCeC@}NK`Ukf6}mS4Y6w0?cQFBQo~r^W|g z$bk9W0@nZ}v!LwtMMU&&0F*F+trrxtn6+RmYX>3bJ207X?Ga1EBaeKRUq-kHDKTlo z+xNgIFab1JpS2i#p!CRbJu`N21c03!nlt0PHSp9xY76({`Hrt)VTjdfa}79p1$Fff zP+HK?V$xl_($s8*|HdifGZ||tUaQZ~(N({Pdk?5g5Uy6qkIE@1co;@tT;{q$OiYaA z9PaJy0pA9cidIAlpZRNfb@hO*#ro-Pb9U1o&@WV1YpkV`r5Ng{g#Agoqme^UGnG^B z8;3Q&urOm~d$BGquFl|O7PR|r;=-{Eyp;pxXla@0d}d=RAkyYO?NOUkhNeUw0YxCO z3au8#zF$yRNkCl$_qKm<5VVDTIwugk07W?JDDBh7kBv$yk7hxTJw|JuU0og1-rl~s z2KtrhgA?R{$_ELKnXX<=OHb#b)#Q;TsBPwkSj4QXMv!WC^m+;0a~oPwdT%JqK$w)B zjosG!d2ydhB24ecS>ex(gA6k+IZaL3-we(Ndx~Lt+GE9k#5VGTAWzX+>T*EFiqMCf zNT-dQ%*f(SQ5jqcemE5!orFuQy@`+HtV3C@7@{D!D!BVQAc5uAuV43AS-(OCQOR&M zP{>jIRPetr`4vNdwOg5x;0~e>@3w=W>Q{=2Fu8c51zN-7!UHtr^cZv~v!FoIhYmqe zKl~Eq?pg@^=PXW}usVpDGMX(bK>(l)IRCM`VKc~7#3dwjz`>pFf0hY_F-Ye_knBJM zByaqKfLn+RmpJgCpi2VQ5nDM+ZXYBZy$4Ddmp-5X7MPwm$h9{x09QO6E`qZ1FL-(_ zD<&o`nB!Y-Y~W9CIRySa?{|I6~&Iv2XBn#!X`gfMBfv$aY0y@7w6-u9_^6e^|s zcZ^nDCL_6SrUFAYGEgrtw6wNTFf)e$dkfS%WD5bvjOzwjZirn3o(P(!PZz;@sSEs{ z(<1j2Do~U3_n;d?ra$OZCLLGk26HsKq}eAf*Zze}VQ~YId$?L)k%i0=iwWByR0{w& z73AgX0J4_6hx;3<_ynLeH(U-XP{u9lv7q(xc%|{~qrkmdsQKN;*4_fh8SNNw^5xo! zlOHlW*kBln=fxY_&Tru@GxpE(Ng>F5cG&_kXhn~p5h;b1#bG6@#I2hp5h;|xA_>B< z2Bh}^2Di)S<^vQ9u4tsG1{De(0YP()%ia@ckvcY6&`ZWTKskk^QVT<(2+|)xP$f0^ z?ofi7>*1Sst9i|ks5=Ye#vCXt?z6G&#z%*SHo$&5?w)UA06c(XC4J*}b#Ztun^gEG z1UX`{11lp(D?ek~o1FVJq8-{`0m>;lA6;?-!GCH=!4F22bkfCXJdu_wFCMGB_9Isz73Z!`8jv^{5 z82UhclKg)|nFeXV&;qgigI=KJ4TSZv6U?)I!=AWX=jABcUw z{cj@$u#up&Fk~xHf48FN6>=?u{@-Yi7OwCBj++>%MPAN&*L=D5@9u_RN5>=6{`kpj zrF+3fpL(B&{w)~94u1Ei!;BjU$C|g08uT5LlehR^zRf+ z2^O@I1_qPQ9@>UIm`fpI&(ts>Ao$LxG}12g!2uwjiA{E+ZU|9-4m9v#V;9qy+l6wgb{#RC$mc|!cf7*YlXCS1vP zQ0?yY@9c3o*sW`z?*=TK_Jx$bT|nvlMJVu8*8g!0@V_ElTaCEKHE;jP6l`!T%?KFf zWm;EGE2|%~J6&)-II$YM=By~#jN6uc@KL521p>qHbIEb$r^E`8DCjRr*cs2(2-Zf_v@L z>)u2Gf}|eh=zdR`SCq?KxI;t;EZgQizF8jQ_P*eKUU7$wV^neWR$dCu`~Myo&g0iN zstD0BH&(PAG;vVwW(h^uH!KWP7WqjktNGUUmds8t9x4P+G6?U1d-S2Z# zOedGfJ0yRUW2-`zLnKR=XX2xHfAIehx14Smu$qI1BiNPEGnH={3xe~Zc_1S+9gjz4 zozN7v<&~_SXnV4;@usKM=nwI;Cu|=(wTJe~OvVyAjkkM-=1nH$yS^@dwccy5dlU8T znUqba%(y(XSjfl$RdU>q6*IzOrA?5joPz$&QM$zTfn&?l?D;TB*?W{d zi6QSU|2yAD|Gk+d35A!N1GT!}3^Y~o9KT?T%}Gxx=OcVw`C|;*)rd4&Pv{mby0;WQDYnneDXojk!`sf_$qfn44z(!K~w&Jkz|41oz-{B z@7Z(IG6@|BbE;}Yd!w-)e=j20OEh6Kc{&?I&Azo){=%e>=#MH<6C_9ZfY#*IR{IO@=Zesa%Tm1P>jeIHFHj-Cs6{ z)OVQ?*9p}Sb{*;OHZJFmI=g8LnD~2?Dx=jKxQ-2yB>DBWcC(}fRd+V8+*nGwO9jY9 zY7HSKw3qOiE3KhgcVEQL`DwINUKx4gvPZjSRQ-*e8(SDzP1ZwYJ9=8y$DR+@E44ZB zM5*k+S;&DgRiVXD(7DvCf_L)Us`%z@CX{-r-qfhGvT0H0PRTC-_{C&99ldIacQtkz&@Y!aWv#-rBvu{7jVaZ$b zSP>xd>R85mahS6aEnt`&WL`XY=`Be#Yub+@2g^x2FC2f}t)a%t?lPk>=Wjko{a)bi zL%&6`I2Rf4b(QAFn$z7LcY#{1#$)2i!$;wQXFB35Ly}m(agleS=Gucpoxgg|;0NzP zr_i5jmQ`BmyRhyDF-gaZh{?@$B=T~#wKALlp?(S@?wy2ZpFWZ02h8#{FSvHj3G2*zLUR96S za-eEqhqA!$he9f%Kr4ELgYJP8vjEvnDJdbI zS)@pVj6|2?zSvuMePx(i)AMDl|Mi@0gf~MC->qi@&lpSp^vzc5txCQE8mgalXXB>1 zG!=oX>U%KfPPbEbcuRh?#`8|$R2ulMIPe{VPk(1YYgkJ_I=YcyDpUuQPmvpoTG#{k z%w^q%_#bQae!DG#Bl$v?<8rIK^*3MyIC+kj5_!%KXTfy5$`_=mOZB z4{T%$49bWg=wlmgn+`NfkbfTq|IBE=u9jW3KMH;K3GP(I-$%}hkjt|C-bS(2r}S?% zNj~mh&`nnHor^n}b7|3Xx|*7rrgMD)SS>l}_J!Su0f2uoo|Te%&Guqe$CSr19#ga~ zPP(=bXC7Bb(r!%aHCr3g6Xi-KQQwa^!AKJfGUcD3Fa%J6T<)cuTrD64?I77hFfgzi zzJ-PaDP~E`N1-qr$(jZ_TJcSKDVwx&wz9c zIAry*?q9O`TfLFmo`{r|^qI50HhUrpszJE@Ixo%A?h~?LyWHzQrM^R2T$eRP_A?+n zxxl7FBgRpWx^ey-EjEtEtM89LwV#5_r}-Yu7zvNAnWq=y4dGK^URFzl-Y&VtrV2w^ zJG~emofcPZys%cYL@YnEBrGskdJRc(cf&#rkjTgd4m3=2SjMn@^H*e}9|1Fj96;rN z*$tWqxPQ~z;6G!8HQE5VLjjY39_T*lTEQkQ0cVT?ivO9+f=(lO7r;WjQn~g(J_S~& zudp5a?dYK`0*}A5l8cq|F zt<&|PDE@EEGHW-y3i2e)-LyaBlFWVDaom|*@n$wuyd)pEkZ59}R@@?+O4)+Z~+x<=;y15H6J=5Q1&=T<5 zMnyx1)fx8f+HTyqfg}0O4<~$FjT}Ide@!|NyFh9*HY%Y&|CQTG&pGFqx7ICJ3&0H7 zfoa3WK>>J#)#$4z$6jb?$N+=%_3Kw?Xn``!wO*8g9#E0U6ngug`Lvf%vFc6Ns+3ez zKmr)>I1vkw3g}+{oTQUM=N^&rCoN#Z0gPYe%Wj4dCZmPdyc16@0@WsQB^Kmgy|QoU zT0!V?%J>z_HcT?1K3hj?z2taPFa>p#*T&2yoPmZVN!X<;8<$0zsYG$7qsd6B?-K(@ zpHmtBdrgh+FUd!>Fh*X>iAy|Vx5-gmZrwh zhRnFYsJ@Q4Ew`ohM`GHzWEH!Bj67Y6p@%7p-}wj~PS#P~^X)H~n;6sc2L~1d{&-@4 z?xf;yTrn!&Am}>R~D<(vzsm~^L8?+?^OprK>&Ok$kwhH`S|U8 z{^qbGnawJl4524H>tkXdOb3S$tN^xx)#n8*w2qEWzs76%moL4VXCZi8@M{lxY?rrT zgb{yMT2>Y%*hFmc1W8_y6}e{V5_DUuwVbK@dOPdH)e~}P9J#3V#f+pDqlYdIJTkd-=m&nfOSP8= z->#k9Z^}<_n2sj>^yKU^TCRgI#(Qypfs^`u z0$FasyB62`ZDe84h)YVEzZwdr5YR9a(ogZDZk^AQu$%G3IRVfzy$yIs;iIR3qd+93 zDW9vj_%#sAasF05*1e2zKDZ$`8sKcrv+@%* zo8&u3PU(r*le!RXr9f7cP+lGZ27S2ZV8i_h*^lMI-W3n>1~l&ZEiElQaM3uZ(U2Kd z#tFSOZ-pwDpeBz&DwGqjCqM*yNJ>3;HOqC~WQ-~`E%44t`qj&-V{rjm(z>6iMsyi0 zov!P#3bL7HOqN_y*eQ0a8_g=poO+4(+HTB_Wu5u9Se}V;r4df5>>r{0=En4_d6TXT z!=rI8VLi&qZ5fwGahKe5Yjkf+EQKu1?JB%@l!KjXC&{|Um!V+$VgqY?F!f`5)R)O& zjb!G~#Dzy3v;NEwM61&k<1B;&26tlY&?o?ALDSF1lU!eH3IO?Ff)hAiSy)&+`E!wA zM*Mk50kU=Q^q<<$hvYlD)^IviFTIbGY9M<+3X3dsqtP=IqpiU^Nc|g}tyLJ3l)^pN z%{h69CRn_leeX$XJmcZ5cR?+=+Jl8IJWfNGU6|kBuE%IBcG)HlIeQ)U;C518WM@8B z;JsnCyDloNyg}x_C!zKpZX0>C*suG$!5~6LPnm^o+^l4D%=na9Dn)j={^I0_!^N`A z*Iy!t|C>nb7xO1H22t76(TM^b+W8Qc76TD4pwmPY0>IlY!nA({(PALFNr-Ow4qUc7 zr0fkr?jP7Hjj8>z~0=BAnzd21eKDS^WFeTl=J=wD5Skq&B{Ik)x4c}_sj>7 zPekR3LSMc_0nWc>(~^lMU1X>OWFOo4T{7zlbllV!|dXnmb z`tMD6LJ)Wl%0hbwR;K%8Jli`089U0wRiy~%V7A)QcCAQdEJ0mvySHG(PM zCjhHP@(47jh%6D7I=n`AQtZG85eor#`>~Pr3P2ePr6&#;m7PEYIbF$OR)S2!vwMzZ zP?s%%jvQcijZ{rtJ)uX}^fyW$MO*xRuo6u!Zmx?wx#{NIX8q)awPL5Qse=5P?q1lR zPK;(^4~`9&u&%DVuj}u|jx_8?9XDx-<6@F{_fhd-Cq&{CvC%S^h+ywoYy`>7_)ba9 zJ++tm{rx@l*2$Rc!~2~-^P{7f$_XG6MVx294HSjU9{TIi4Rqny6s>w66NK0}$f&Aj zq<9!l=5G~GhdlKwJF8Y2Kj;@>Gw1iYf??vJg&XEnz|K=3M3LH}CwFUZ0B0x9rQZXc(<3RALeW`12is;!3r`y=|T3UArqqrjIxc?j;uG})Ka^eV&X7*fqL`}$$ zYfkHY{=~zaO|jsuFu4<4XvxVZZ2w|m`(*F9HOsOT*osVNB z?cgz>TG`+KPF$+{k8CtTG-f8y?@ouT0!jC`vr99^tmg3FFK;?Ar4?1YXOPcZ6ZOd` z#DlEi4cfUt-*dLatr~2ZF0qBq4B6jFyO&MIl9I9mr+ACM_rGBeX{PQOo5hq3X2cZR zEd6F{#?SWVmBm5>-*Zt;hG!o77K98etbx{@OZ5?%(U_}?3#sfewY&b+%F6wNY`Il& zVZ4-l5njiaJDNPWzul9b-NDCyPcy8VEFev7_$LGo)$qA==6^05o3HyK{kSp5Nse5^ zmjpdBC#VFO#Wt1Mh~H)!nRSn^>EQZfGO8Dv>VsJh7-acpCAbcWAuRW)4fk>b{(twKG-c4#L= zdE?dUd}W-pwC!hh{^1XLoR4z0^X*iL7NbVJC&{V3)_yj{72p5Wdx!G&*OMbVAsTAz zKO7rSs^0uEb>I7(%U#e<{mP!v zqv#~Kmks%*=_Kx$OJs?yf1B0iXCmBW29$VwoMqD{NqUv*skcgth<`w-N-(VV{Bc7;ZQG|V-Yvd41P(ufO z^M}z@tr4~siH8s|0UGwA`cO1pNA34s9Rr zf~JE+*?hlGMt5)UO?41V-3}(aE2&|L);1I-r-3IyyR+u?_*mV}FZ#MSxb<}CNtyhE|SYa|?Hdoi@1{4S77#f9}+8lC$UN%kfLqz8bK$Tvo zOck<}{HK~wufwLsGt+-AU8=|GozX;Ji>F?rHc+WdyY8!XSavtR7e`~ zSMl|&4}UA4FH%!8I_vPy%#tY%pdX$_@}8~@keQ6Xl&USb9MZV&^xc z$IjO9%WGH}{A5L%Sjiy>s1`Z|)3~_h|E}iOQT4qR1M=ISA9NJQ*AW=zt3^(3(Kli5 z$i1G|6%}+g(4$17iVpp?HWZi?&-W|i?!tD5SL;?*$?b(%BKR6%hVR|gYm+uyGn3dZQ;x#`6eUa9%Kk+J5Xs_5fTahG^ePYri=R-)J5W1Os0!BcqjV{>b5!N{aw)=fIiMT#|42D$FN zg)<6Y!WLldD_~?J;sjL>kbtuuDHT#qyy}H-zkElZ!SOlI)|Q;o z*lh^UHpXJB@0}qWe~Mp!^ZE=VE^)2am6zcWkP$ zw|Q}y=Z>5oQ2yL<>@;iO8cZ%Xab{fO%pJ-@j}!7m zoZ1@x7QLn)XTH|nUogjpLluuy%Bmw;jWQYwRxhGf3XD`Dl{$O`p*12J_Ox1gGe4N8 zBYh=IPfH6G9}7#y#cAq#C(o1bl~E|hgF`IY9A>%9U0peMHG)h>Ji{n(u2^to#y$$b zS*8WDh~iGb1EVjj|ID9Bl0`$Tb_Va(6I+)VmgX&m!@L{Wa!pBU<_+v9gcQOa8<}Wa zXql7f4{6m=f5enk>B0)fqGqg<2KbMif3joo$qma2X;Lo_^qP;R8zjB6R?nmg1#4N} ztu{&%rgVHy-L0GUv>nTE*dE9-8P}L}Fc8tL_~6duS2FcvET`_{s6w|%Wj%w>L$O&4 z!re~_!|?@*y0?NQDDnP1w)mrw5(#QQMZ^0HKSz*CTYe0}*3}i+Vea>@n({TpH?br{ zmzT$7!6JXBhhgrbCN_1OebOz6uSLGSnkTyO+|0^~_;7RO*`O&7<)iiJ-!ttY@2w~2 zs&^%9@(fk)1l5#&^uDLuXK5#(=W#* z4z!#YLgyjRqGu|;ZKK;)u`ky)xcNRB1xk328Zwl+(vp-*DOkTdFT%xNW(P@ju6IrF zlXm>7yT(C^j>Sxe@82;y6lA7luEI;78Y1t3_l@QLHP`ML{*|_tC)?WJrJF5spQccz zqCxMeB((fc8b8EG(KF=QlYh#h-sI@gt6_^TqZXDdL^%ohep!DzAif@od6%>!$DI3v zs4HVIwTm-xcnmi#DOF3AdV2(yn*xkLc0`2erBT>2cc;Qv=*e5%ZWFxfB5nV5{~8Aw zI(B_6-rDa^%O0F3g>Ouv9#V@~pA6JH>=LAI{!&lfJoD=n+hrl4#2aZ(!7htJ7s7ZX z*S8F%vjBzs9dh}^|5#AbYk zzNAo(Hw=|}uWs!DH5Ap?^iLHd!#wk)O6}??di5fw{jw%+UU z)XU5tKa1lFDSlC3(9HWQ95Xr^@L<=YOWDo6duOVREj*99K-Z$b`%42;%D0YzR;ur} zpHi|@bpqj7aa!i}&2bX$Bcl$DOJOZuS*-p2Dujf?goN0B8e7=8apmjxQhG8Kcsntt;Om7 zEexxex$^b_P2#M>Z+$!(+Vt6{Pt6{1r^e$I<>#zP^HP2^DCyO=2?I?`yqh6v_ZX0> zf9IT6c_iGm!4L5UKSv~{zN+-=YubN%KEvenK_)xw4PiT%D=tAyjKuUL7oTdmxBJY@ zX5oWAt6P5pKkkndE=D-|eUy|uoXfbShJE!lE=yhQNA1f$GMV04Yv0`AANSBEyquX* zDcmVH%TQZrh}q4uK44(PB9juhQSa~TJfei#jPYPU_dzOoMo3@WjE-9Sw{cOdxqF}* zB)fiD=tIOq%3J@&NG6fXWO^VxAhl&QBOs#Qr_9ns<=&}^H+*&zEA86+*Q4DPiq2@3 z&*P_DgZi(EIc7W9ivrFFI@qp8;9`yv5@O2OZn>Iq1c%NM@||b;1YIaRt9n{$jv=CJ zr(HplD&5yfY~3^LS)6BzpF^v6zW8pA;Jr#gnS1Aa`ZeijEF-j70 zDWH){y!nrgR!@e>yH74KVjaYy3_2EYCf zSLGdR`vphrI4_?GY5aUyH2#m4ynsX-+5}y)2G-$h~~^y|VkG$Cq?yu3q{Shfi?3 zi2iHr@!CvIn)7i{mgI_VMpdC~R`Fg>`G-$4FK!G!Q=zm8dUBNJZ^oSnaMA^$c^LxoF{4ZZ`iatw-c6D|9rvCnqdnZ}F`@6oAEsqVX}(qTy5do!)y4~H6ao&V-=$bo5ckGqv&aBUF*k#>pbuD+0fN9 z5XOA{8EWmA`_4PjGJQRfP`Nakp_v8ZavlFfYdkjM>F;;Qy)=uWs;r!Ol<)cFI>t5M zp4AaeAFiToLNZ5ZE(WhK{V(e$h!oi3M!dsfuIz3hrxo;_`AA=&ydFAEt zcdExidOTQtHn-&d@18g9_47wYmBrL!Wr*RVxhpZkrT)|fo?$;uZ=dDCu1;%aNZcHi zVq4ABR&}8>Yx_Bmg-IpUcdxdi>%*qDqMC9Izdhv)RXgvFt9_3;nW zaqq#66xMH>Enh=%+gO>Ijy5xx_{J3dhK#jT78py?85w;2SPeH{f79^bDsg>n`rwT- zzDU96lzH?>7yB}2NnOH*2Db@xEE%*F4&;=?e3(n$zSZm`jtT5Y*^&fmE# z(z>G7el*MF0t>y!s9dnmZ~jbuuX5ao%0`}cbv&`eMR%jdFg!%BAueWotLi+7ax7>^ zOu79~=INx%@v{Zl4FbE~_1{xiy$YspaC|7)%2i%bJ!#LB@e&qsyv<+X{K)mRkwXMa z9HpLT;fyucx~G(7!KLKxkBR<>1qYh6WeR5f&Nl!=euxF&k}}N+d>CrB14vN3Ct@52 zg#GeY$k#+d_nF@O!hPEr)dJ|Rduz?vzAB=X1dtm^gMqkgeYqnb5+Rud9TxkThU5$_ zShl8lbK&oTZ@~-KTge=x{r))Gr)ExFw|{SDFqkU#mJt1w4B0Ow0()wzL8I$~f!m*9 z2wn?s{q5rG-x*l%HO5XApndq0;38n5vCVYtX1;yeUz__E{ua_Zsu$LD)%D5T(e=KRHTGmnfM|85TNqS9?`qY zaerqP(!^~pYFm2Q-znu8&x;r%a`|z)FIr-pJWlrJg1Nn!tSm^I810@seqvv+P#{Fg zr=xRo;+U>dTkgfe!cf=7@9%o=9S$^wOUF$yo{|R%Px4_zR%fLY*YUK{->$BqWUOmS zCP=ZKjiT-Sf@jk$i+hDteF^qHpIxy&gG5;^+n~pEb#%;wl#_~)k#TbzH1lGxxKtdH z>rM@8zsvC%_9n$`!;RB2KCLYtF@ZHX@25EdNB3c($sIz3>y|5j__1X>M^?Jz~An4b#O1()+NA;e`;yqSa7VV7v(}g_B z8n$uqo2Oh4+K&SQW54ytGaQYZoyM+UFc-R}WA9%}vJ+RI10g5)WH3NBiU^QFC|(Qt zc3=ZBAz__@mzUSzh@98)a#OrvRwb-dRzD9>SXp802f#&uYQ3^{7VgQ56YNV`Q|HbQ z0_dPq<3Q^NYTKx-MNwyGURYaugpt*kM*%#1ummeaEyX-e{0I*H1FX4upq9J4UUcJCGEFJHOvXwcsGVKQ{hy^~ybdR~%yw3y~;(u*@Kipqpjp zY>@%)6wu`rjJFOYPev4$Hn9Y?JtgI3YW15Nr;AV&poxz`q2v9(vU7%HW3A`|t6&`&Ku!YpBMfv&|0!VW?{$1S*LAtN^pO{+}n+_0dpCk+^^Au+|PjlKfY4PrV%VNyiaYZcJbqeQZ{;!eSG zqYL*jG%V~=^~DjgxCKO_8OEK~bV^2LM`%6=@YfE!yF$zVsgc37mX0~S5>DBc305=435g2ar@ zs)NZkFtQRTRzsSq#=H4DpoBmYk{ThiDiQ79jo>NCe|b2?EwuCAyZ9%eYcO&DbNY&j zPWz_4*<|lohtrR|m3tfYiv1j05!H1g9+x7sTg@_mYhsdUy#v_~755Y03+oA#?3iF- zi4q60T6$#O-CZ3Cff1@~0%!8S>9>4a~DIc znFk;5%+dPMMA__nw6c4t>^<_IaLPX!z5XkTpX0FI53rTM=4* z*2?68y=56yO4mi{w`4%#%$&ff|D1wAvpYlu?>%#_Ik8>GADb7C{__QLrktoh8L!&# zo$#`?Ianxex*~1s%r*b8shP>pgi6DQ&G-xTVF^N~~tsulf~iO2c?R4$P5Rfom06Zc(K>mc zHaCwh$J`Ry*yT7r2-LSosp~$se6zPE6H9(ADYdDG?*6CYZ;zx`$hB|(mw5P`L$#Zb zmF!!Ls25v){NOF zF#jf==>s-&odhhMgv8u2ll0-MDgU{HlRP5aKSTR!qt2jX6DP&kvFvzG^jvhrH!weh zFz1*+da`0IYL3LXtSw})5$!6>LQ zB_qpYn~QYCZ-4k}6J~j_e8^Ve;-N^Ty#gDRopn=evFu65MBb-hk-hblY}t3-#%Ods z+(M#@b9p~jLVnT63I-HPs{gYIK2sX1K&u(zDC@aq7{e1!Fs*8Zv1AJ?yadY=h#N zspoJPvzpnwdS+}XtXk#qt}TB)G~kxE6T0(Pc7G{Uk-fV>)uh`xdN?kn!y;w=OZQe( zO6T(QihFCVZkxNu*S*`X26oQGJ;bEDT?aqeu)-C(PaONt%qecEgvN>$6Sp|JVy37z zAYi3-+gFq^$;~yO7<>Pt0dZoAu6)OXi^H&?n#WgF!+PifSDq5adC>JX+8of4eSvdA z#+Zjb6_&kNHO2YCjwRs`g~o`7p1Sk%po>3yt(fy$uB#?jY4&w*^G{nNoK@nIKB!vF z@x;*A)(*^U;00&LSC}v3N_YJ2k#4n*QG7RKPP2SxSy~x7P#$PRQXR&K&X%aF9(!6K zWLroQ<{A}!SDnW{*!c^pgZ6|5Y-cHg(c$?+PjtL6Z1IMNY~Q7oHI1NWq$TDAxdO!4qi^O1pI$bQ{C%g4#SKqt##>+zlT#h*0nr{Pv1$3 zs@5OYr!3Ift$ujPDPiMG(BwiGh6b1?n zV08>`sAj+OBfG#8k5wNU)f!%^|QH;?E3n@hwJA~iOz`9i@rZh&0~JQmNu&k zt9kqKzwI>J(mHaMj93S-dv$yvP8`_?yCPHa*7er1#B~56Sw#Nfw+;^+?a}VpyUttY ztbE-&Ig*_WuALR|h*%JmAju1GEeqq6POg`h_VDIP^-f2h3LU>_d6Haexk~!Jpw4qP zVO21PchJ)I`P^{YcTXFi*Z)QvPy5MQmSTLRp5PDKqedzUE4!P12i4jpHH!4agUo4} z^>jv;#tcV44;3D+wtX-!Bu z+?uU-ouvj6{ykzmt~bJqpM5I^0aDGt&9F7;c?*xmz!d(6PydeZifWb|whMlNIbBngO?RJVbNDvL7+k&L`WD z@ku(c!n*R$S?53&%~ioFn?2HaQDcI4G3BOdGK zeXQ4_R(^xKt%lFb!`7?2P@qMdK|+3zIKZ;GFOm+hcqD`;+=t~+A*jaIfUM=uKP&#N zu&l3^+Fa?AYsTWU=JKd}v7WBa(!GWDHq=x^AZy4F_2<%o^lZ_6B7C+gHpQhids<~gV7 zqcl!5#0pT*$(6BOKgIx2%GyKO-8ekZT!BdIw&lA2Y z-&1_SU)(YZg?vEYa{tjSuc4LX_;^9vJ|F=sb*zvAEl zUUIiTj#JgI*h?K&+MKJ)aSHf5(O-4XF7l5^>zHgx$=BvSTM^OQmWY(S_sH*f zR_FQoUf1t?eQ(!w-fqr6ioD;i_v`h1K9A#gJnoNz{CD4-KL5(sulp4wa~0Cs0Sk3R zvFdD!=rAC+7$A5vlKcnzTd*=$jTW?9#T44< z##J(#RKEseIrx`YS^BC!wmH z&XVruNz10Ft>A@x(Xe=}6TR`+V871JC;g?8TTo!y>`x9C>HGH=NlDBu)xi6SI)A4< zXa*xW)o20%&I(IUosuBcK&J^<@z)g;dI?9K!xamQin3~uAG_7@#K|d6CvlEgX(DZ5-}9D|iM{oS@|qIw!|#kKdD223S45=y z#*4PgF6_H-g|xP^-=z$}>o+_O0ELB;2Or$6h)HdE*Gv^yq_Q_EC4TN(S|%_}q%LUm zRSh{8(q)d^+-jz&&hSmMS6~1})dV^1_j?&n^Uh=(vTR26g&nuykl=LFKZ`3b{XDxm z%?6+P*xV(fAY0ih!>JG7)1Knu3Iw|azEn8`P9Fd%JMNNr$|) zwhRm(tId?kfQip&N9P^*FW?mXp5t{po%bt%T02;5Mh0;I&tlBKf5GX~_+Vy$-g0%!gkqEF5g2Je zV7~Hl_?kCPd;_`q_i8(ps zs)21{8G$rDCdtUC{6f4N#DnYY)c_tGWIG7}o=c}{FY48+>!D0iVgTTfm}*Ra`&O-3 z85{a>AYz}nSwRZwYfu;alCpeqgvK^(MUbtR-)=tnp#N>&i%+^MzT42H0hzMqRW4B6 z!ZibCyQW_)CUl0q3T#^tE~t5x9%$F{*RQK;r)~A~19u=58bU;ttj!P;06N^IbLC@} zs+26E$ARng*UN=9?<@)Lz?;4o_%?{ngNPcCl@TLob_@a?Pn|jA^ZYpxQbf_nWXvkZSpG9r-q4)yy0sRD7;9*u{#7$F^i zCPPnRht~5LYSTb?kh5+_I;H`p5%M>0VuDNz^YrTyL4S-VEyJn-ybb7OtEOGe7SJ86 zbm9UG7lb2gNA^62-wGjvG%_C5oM|WFd!Qr!U?x2}M?g2LpnzReG#Zr7IRe9lmVKP( zA!+DX`L4A3w5!hd?F;MOvhYaBl1VwhGLGv82}Y1@%6q&3?du4?3)V1fBshoWS3PhB-rV?&*FjIp z?Cyz6@Wx;l_-$GPN)Sp&T^1go6Es!mehV!+7D)5$TZODWtq#(rQyd)6fUn^@4>#v) zXzwjoK*DeNh~k|)qlAZ#6*0|k^OSV9vRj1|8IC>sif3b2H43{+0B- zC)4%x#NDT@L!SB^KiQIWxNpX;6fX~rXl3agR1zUTMUtsRu)qP~eKEas z6`(r=?QoCKsCmF*^TPc@M~5+79`81|lYyI81SDgny*gQN&GrXSumGU@w^4m{@UHckk;`ECY}TW2r;h z0_tWjEW2Dl8H|-wpEp+dX{B6+fb64s<^4A%-k~M(cky%x7sS?V_yB|wxvj-f zTtU8k7+fuklnVkd3ag-UmJ@6;%tgTSyBM8?+z!yVB=oW>c;^SVT@o_#0VpyF{RQB+ zMEIn(P`~hd*tNkj8+ru4LLE-Ht+E2)1dS)DFttYpWf+ z5pi)h0q73R^!3HjIQz*~e#iuHU7yCThYmReIQ+t{2Ox$xEMhowC?^1<@C~NLE0FYn z1BZoA5F$vhZF=+!mO=3ltW=QD831S& z6tYu4*Mu0{JMbE?5o_aC-L9tmKr^IX6~QxB_$D+U;2RVHy@d8bCwk?)5Xu)~oT2?| zVQy{!b5CSYMa~W^;BfsOe4#lg0C~|oz+)v)pU=Ge6c)70I}ptWvACgwI#^+U27$uB z1&D+ALHh%q2h>1q{eckQ(8Ij2^Bue|89A$0z6WhCG+4T3Hup&Jmt-FZGB5m?+SsH0 zA{e$t;C#yj1a|UqFmeQ#6lyOUcIdm#u-R75uwy~YX#IW9Jv}VQU1B=Xi-7Imt|hG+ zWjbmJ(VNWLOlXY-oOBzOhL#q7J3I<_!Z6DtdwPjy1GRAc za90(%?>PewmhQ973?c*qFxmvH#4skrKpsN!Dgm?Gi#XfL07QX;qZ(YP+~?F`g1Q#@ z5gkc)+t|`vSLPcZ$hO~MqQo+rtV1`g&7;GEEnJpP&0BX`fU@Y_F?E`kHx!|2IUM!s zfI$5YIE0tkBP}|in;P0Si4sr|rw~rgcJNWF>*#z?h5=`?i3~V;Ul3kEK^QNRGv4JE zR6YlnSN&v}?wtG{eA@nbd3nwR*E>IcgkwQ*A2L7k8Pt1eb)b~NhSOK%==vn2U}mkG znb|qjJA(p3BqpG*OSu0Fm@dM6JH+)sr18VdTreDg5M9KOoB#@($+IX=AiLlkzB)v` zXqZgFI0L9`=pc1;H*_DN|CdXjzk;S3-P*4CC)Kz$;o7uU! zlK|htN3@(T9(xO3Zyxu-COa@IaVt-|0US6uGKV{#JRL7H5d3Ye0p(-{1wX(Yiv^a; z0AzXvgDb=je*iDt85k2PzjsEUFj#b9qGi)vA$-jkm`2`XYinz`b#-8B0t+XO9qerm z;BzE~+j9~)`)N{1g zBal6ZwH|tm{0x@?p$h{SyWTfj<*V3Z+-e@b_%h(ge;YnoIMN^!H5S|#o^bGIaCQe9 zu?c!vNpM_DINo;YZ`aNLJt2g4eMEd7MDPYUJK*z^X1hR3u=Q2ekZlW`z%V0|#)5(j z#xaCdwP2`?gj$8bW4~DQLy_v=?m3_h(R+nc6HX}PW_JN$sshle*!o;lhhB}~u_od! z2bWL%5{$TDiN`LHB>xEa*HyrGow_}96sCz0%F?tkT9S}s+imm%IvEW@?!|YQF1Wlv zPX&q|A*H+j13-tp-itV~e*>opbCH^n&gSZ+i*&S^A3bs0^Aw5 z9u{hp=fWr12WkqOi>|J&@_{iZG6iy2ivWy{)=DtS$0x5shK}HJ`A!XFrT~q4x>*eW zZ)*#(GeCSm5oNLhFr;PpdiwGJk`zfUgoK774;hlAXlQC82h`!&Mm`ns#- zy*#&DF>*asuFL1s*{_h7OclzmH!4#^hQqV8& zGLPz0dY1#Sh=2=zf&Nzfj1HrM0f}bhZxtalEB`K8r^zq>^VmVD_Rl=z^l1u~f5sI4 z{&GkEyKk4r10p>F6n=;8Y4VxOJ`0@0(Ngs?IvRHwR{S;Ct_uOEkC=gZtux%7TUBG& zc4O{a2#{C|o3}J^=0u}6zxELsT*4;Tv9l7GT} zp@}Lm$ct$7!p3!h7wyxY^mtJmWDY1q;&PqG!8R~ksWg58XBF6DE$9zTP1)Z-W;5q+ zD5o4N4vfNsfaeG+D}!zAjw!k@)*|#77C5y9s?Jr6Sa3aI&5&KhK7hsm9|;O*RlopP zS(*jvDQMVXp)(UeEDkPOF^ccNru^dbXEh)WT!MghhIKbh&69vx@YG&i1|kJE`X9ehvw!_6u-eQ({l$`RdE@5IN3evx z1JW8wwQ7YzT@I={0L!tV0XdeG>v}R|KS4yt&6CV z?pC>woE%QTbV@^R%XwfcqG3F`JaHm66Y$3w*Fu?qRT&IFA_ypYlBRLEHfFFkVIf>4 zZ2r+4Np9g#iTy+eh8?EB5bhL!%ceO^_W~T3JCEu4+<(>dDYHO2=Osw+3*Z zUKrbt#|5}S`rFm@gG$IU9kdpyDIZq3b?X^anvA6lAWV|130`ZP+sLsUHq@bl=Lmoa zxdM(gZoL!~6iQWd$)M3&L%Y-TJ4lUZg5>w)pD4YcJ4F^dM$G;+1m5BSc$ev@&QG`s zlqChQnPb-$C{yU5asVQaJXRx;3PCWB-NFCDc@9rLS{_kw5U6cXcMbF`u$e;eB9bjZ zzC77G?A*M(*Dx1BWzGY@P`+42?UHuAu5}F^9oo-C1a3SwP;(srw`&XGGD_QHHnHqn zT6xpxgGFsy(1)P5Eq&@qxFb&k#5OOF8=A9o1HTWIu;XV5=mMo5KD=YQUqtB1sQAdjV($wO*dw zEe0M5Dt@t0o9P;}`w6wh`;8({%fYHnt5yqO0L@x?4$0e--*erk5-Nl55ZsyHh>B2~ zuyK0DB1J*l!=XdKN-`(b@mm?F1k-79*hi#~y1D_CvIidaMAzAKf68lEquzof6mbC1 z>IWfcIslHw8o2wI2Ucxf$h$IuD%K6+i6LK`AD&38g1TR>XbD6`9!ge&N?aL^t3VW0 zp9*w*n^ad5DLits(%`1er->*U`j)b1rn6qZR|q+=Bf^B`^XzNBG#-inG z26~-XfMtc`Fi%tzK-=gg{@)rV6ef=KT*`iWDLuN_i7zPhCv@L+u@3YPCf>= zo|O7t7{h=#^E+UTLZbr$TaTCR_wjD3W5#Cl2!R-K%TsI4j*8Yjx!fc%TLVd#}=R{VMO z@_1@0l-MAx0$wfxPX9!^&i3**lp6NcCexd{w zD9TZ*1fB#Y_jOhD$bo`gyV#WU)@}PE2werOC<&~8UczM_%hMMjW%<-=Y#K;=gN`>u zE@3ajyGB%MP>^HO0Q>?bPd|f2zhW6c0Tu7muYMst3a+QC-dOtSLqGG8Z&< zv$C_rr@E-iM@TO|dL&+y7^5`g*D;j}V#OvtU8%bLC1E|Tj`&50?S$UwrJu@=uE>tl z_)5N^qjN*tHU@ErT|aRUy=zCMf0_`1Upxto)N3~I%@U!y-{6QV?aBJmBKBouB>Bno z5z0Hf%r&;7mE_dana^Kf{G4LyVGaF4x_F?=RwsLSA34!4}dRA8G>=kE?S%=w@QpLeeMdNd#2UBfp! zl{rbtP67t*erLGEqpOE%`=X`r%G6I1TK{XQ4<8fI=s^mUEouUfSbo! zM0k5(W3>nJKw_TIiRZ&=cq;V5s{CZO~mWz`UZfe3kmU8&}`pLh;4#f69r~m%H?zYfryo@7i@fL)okl6w2 zw?z7;$5wn{xCU_Wg1GXNF1GhA829&# zfn)|n%;oo1U%9^G5D=l(K!o4_wiCN|m!wuy#5{R|0|(SM?dHBfLjxfGt`b>VWo0Zx z(8F9Bwa4Kk@cTdqR#D+^&eope+bv|b^sZnDRqZ%cWTT*w+DLbr;bg29~ z>;0d%QqdFi*lWb8YE&wwR;7n0_5^M>+tP>txR0bpI_Mse<>{5y^K8CQY5=hZ8xi_p z4Cc@+0PLknAkZCsc?cnRB?lS5u?0&P8=mnjXwG2EeK}2o|f6+V+EY_muy4aFa=7jl-shYXTj{CWuH zdk=BxI)w(EJoAZWHTVPc~;4K{g_|UFzTk#DabxQgR?hw~Q^`-=(anRmsP6fte z?;l=I<$r*-zSBu<3#qj^PlkO6#P&P!<0WfZOC3d}5>7UYc=Ca;m;Y!12p!#Lzt%{?)_(!5rEuzl zQ!^T+9?AH+2$&wU`hy0U1pxlBv2=Gmc1$KFCpCBeejN34E8}_D&FY%Zi%Vj(%}8gW zs5^xC53orT4HdLIJMzYV>ehaITRpz=nUq7Le=>`^HS)7h#k%xhxLSbKJQLd;L)w;k~}w9JYex zo#ppXSX;CD_kQ;Nx~UFT$n1@*3qhIt$2WKB!WBfLOpLzGwf9b(Ov%0}md|6nLlkQ( z&clt7y}%TdBU_Q%VbAffZMKJ6&q2bTknUP>E}TzZ;*y`d0_W?I+w3ADWt|O7larGp z+ye$-f8Kc*1i`4@ay_bNfxTHtWK?}#-_t40q~jIUw~zM0ha-71ZXMmN>lBJo8jOv4 z`oSFHj1NCav$C2QgsQo9o9uY~O?{DjZPlj{g|V~awJ$XpOmOlV8Z6Hqx_q@*Wl9cD zSgT{e&rhUwtY$Ewl*VME!jdaX;Z3^U(y!)bi|wF-p2v>OZSXOg>o{;_dfPv2*&+tzZ{>iV=y(DrLv3I7Cze-1eE z)O=D*Imej1WUB(4B)l2Eoi%i=&xlQPzh^Ido>tRm&E?!*UkHwY;U`iLGDl5 z(Y%<$QCs~Yr&2d)zBphmD$i#q;NK>ZqrjUs**iVbsmB)C+tta0;_xL|lL@Im3WeN} z?f9u@hUa8ssr&VSx4+$Q@{QOt)>rP^Cdb)Rs}w)8 z>l4b!uW5=$naqqk%zkuy zLQlYrE2@ykXqe34awCH|n-_~%p+zlo$zYplLzu~{L8}gX=D~op%RE*}Lct(QbchwC ztGlzYaSq<&=hE5s(DCzw*GqCm_*fO><&Cw!;nE$Ii76y^C>r?MVzBI+Y*H%G9els% z#{(D9IKuYB5)2F%6n)*nnf>amQ8~E=#UT$t)y?RBU+`~J|E_~lw z(=13Vg6von-^cJfCi@0Eo#46uq%E62_CBR9t!P_$l5@vRYnyXj69-~*b56fKS%0ql zX^t0dGtJyzuPN9(!qA*&84b%z9+PcNVfj+Q-|1oP)hW^L{WQNYoqDZ8>z$t0{)I1K8i_yx?XC%t$QSK!G#COx|{a|bT80q*dB>prwP6k)K@(caY)To zh~;N!XqDBXuI9zX5-@mAam2m)-@7pW6lrOniWb)5ZpL7{q;PZIlsC4`)!Ur7Io zv9|G}qQ0mJPOes_BzX01$pwFeqNh)Dc+G4~@JRWq-hw-_r2NrbOmdMCH%Svw zBGbs#7+a;-b#F0H46Ffz9Yq(v4t@h0kO+x%FKzG^lFh)6f`7NR&?3eP_06ZH#U%ahunF zFA%%Q_0iHizLy9SSTk|FAkjVG=0eK<+ReXvjYrFgRs=(7sxm%q!0s|glN))ikcXSw za&C68<{>YEh^4(yqBav=-5u%n52OGIi_F{nnb{ zo#FlTfJH%t24j8P@UVT~c!&m%^fJR#L7UU|siF^~^aNs2X&?sw`^bp7NHN{kOs5Iq z1jj(ldPvBkl!DR_~Q_}|r~tzj~!P4CXtlD0nX*MykZj7Pb&by3Aq!hS*oM8rDp zOiAfQZ8Bv_KUogIibp^o;3)8K4Qy--}nJfYxtLC=14A@0tu1Uut7u1S4ncZJ1) zICJoK)P!&c`Xst)iQO19fAczyhc43u`Y_{#1HMCpQXeK)UX(v3!#{p&sQ#kbGF7B8 z;~<%~+=e2DbGvpb{<`ffnTfUmongk9?YqfeGZplKx_CkchBWw513opWc=vxCPc;(K z-0Jzq1rJGC)l6iIIX6--6m)z0um}ZN=rC=MZoTPPFlriz+ji}s(wE1$>0Nv`lls}w z824jqQU>Xc%Z0n`n^bB;X%tIexcJU1w7D)E`3MVvW}neJDc0MM$jeH;Q~4Y%r{NJ} z^|x3NE!}1Irm{az5c1q1{P9Nrb%&QiZ$tl7)9?8Oq-<|%nj9DM=QGsBl*9~6eB@T9 zqVgpm3$J=4*e5K^qh+vNNluH;4lG}Nu2?>Gw-GOCEg^RbSA%=&4e#g4YIe7&R$Nz3 zFg7III_@E1>!Stle(KRF^Q&Bkg0|G8Q_q~9FyQ_*HNXr9AC$b0ZVx61H>f-8`Ns8} zb0NKSC-Dmt+2hTH$%}@DBlNa5U#L5%V+Q%;%dU{u&FOvgeA6ti)UvzVQP0kkNuaEA zvL}O}iK)thDpq;#L_w)yhj=C7Mp#T}{sO`W9hZE4C^M?~6^ug@K z7?8ALeQ8YQ9vYUs1$rAio2ROmub!6`3J4n^~ryo@WOc0O}7y)tmeIid=WM z6znSw3XNW&N9F{pTg@$l=+}!scNWb6kMGoB%&CDBQE387j z#|FyklcV`9DyIz@DVK%H)tvGwZFc7{J+&S$n=b1t>u+9 z=zZ`*8cpqP-y5Wgg6R&*0z;|BY;0^uZw_rGG}l8DR~WLpKvQ(6&=42YJ^XrQ)~JX^ zQWSJa5Jep61Q5FSk`F!zm~^4AZ3bP26V$Qm_09C=S&&u6p=MRKnw`&-J9etY(V*!vR)j=T8%LRIh_3EYk)|>p zdUNwEv;C=v_gx>_EQ|ae=Fg^^_VVM~f8x!q*-$)nQQi2=hha6Pz&~#`wXi@54Uw^6 z6o`ccebDcO;WiHHanj0oTtM{{7H`cu1?Z=aIX$D}H^PJNSaM0p2`@CPGX?De37ybU zFGv860PPceygA?hxTNA)a3ka-V=fgB{dK;^KFARHm3%^y0X(8 zEj(P4a$Ei-13E`bPd};78z)_RGBsWbU_D_cv}AG^e);z3I*6B0pewW~u+MUF5uenN zfLG}c>b<*@(4)C9F!z40Np6(Aw!VjNW?E8QeD6u(UN-wT%ZeB+y%`hA6#d7F1LhpS|?pL9w)P z<|!Hy2N)GvizFmYyX>s9v9rVAY>oGkb)eyK#TQ<*^X_hAA*ol$MpG4<>ggr1Jl)P` zkYCYo6vlmvA8z?T<>jrXGBo3zWYOva+5;n3pS1AFKQH}O^6;hOaFV~_IETD%=_^hv zD|3mZi{_W_r(s6%#0wvi#fj!k&Ylx2(wVQhrLUBBE6OlNjOUWgYtG*IF=zy|fmE~^ zF!~CPjzS0>Fa>f4hiPjUne^!NKsKy$`$1$4?P)pHHgFg`v}>9IV1DNl=l|21(~2*?vD_2~i#%o*Js%*9(IEU%uoW^j%kai|g&> z(Ay#Xb#8-GG_#NtM%M;KSq*g|(aQQ78ohOSDzbe3->eqYqGqx+Fb}xRosRo2gjxio z*7gTyHr#8RwqV^EWo>0A=kw9%*H~T5c<9Pl&dxr3!@4BheRk0{k^&1S2Cp%PXY>dT zwzKt^Tu))xJ+q&f$=Qq#GwUk?ou{^IhUGQ0#64|5j0Bs)vgfZ)hlVgKNtr^a%42nP zdY9PFoH=u1X;>lyw91W*jVjdz=c{q_)YUPQ5Px~(NXmIltn~DBG|_~sxE??M0wC2J z4;;pRA|(jOg@Ry`Mvy{{HW$zgf&eW3)+b3@jD(o3qP;!;(Dkkdt29WY1$Q{BlvGWd z%X-z4*GLnzhzoUsrQe|hc!H!)*T52EK2&SWk!ehz*@E~#{HY(TsNZ`vgaKSMDgutxZog`j z69_1z6*CN*O2q1g;0|xWP$cWS;afEOwVrz=;mo;rY-y@K40qHvDeQA-uNbbNO6Vi1 zA8*$y=!Z5xXbBLIw9uJ*-r=Yi=~}zAQFgihr{=eBLfQ8iS;}lj+oNmO*{IIneh;#> zslq{6poS`)thA3f!R#&NTp8Rp?a;rf9M!*Xj}el^xCOrVUD`gTL8*$?qj=4X0NSID zGRvu%R}2-^v6fvAgoH~=-vim2RkBXT^!A0jO;oy4T*84>W@DW(jr?MaHZuD<+*Cww ziDDk@I0}SQU`G^hJoxgpCJA#~-T%`N8Ahxr{L;0pf@H#9US$!O*OwLXAjysZX(!<< zUX|7(aVUnH{G`3;4T5%L-XX)bO-Hxb9Jxgr@TFB~Y>t zB9xyn8USj^D(L;~AfR&v1SDi&RzMK)?%rWY1VA@4WX+){P)f+p08Ge5B#DJP10kc` zCh5oOm$oRGnLmNkKg-+d_kdEa7ORPmkDKbFfmdNGM}nV&cu&@ORS_Ag`#ebtL@#0! z<w6jZ!e52cRvz>WCE_44+O%qnF{TfcIl7fpvbYCHl+QrY1&_c;gZI%6Rvz%XXf)#cuch; zi~r5WX3EDiFC)8Wbnd~i(rWMSJe8H}0SXnS0tdTh-|R&nB=ZNKd6sKoYnkGyMAsE_ z_K7n!jTT>MsQ81oN{aaST2=UjGIu!tdcXbSu_^1BNO=_P@QsUTOh;Z0E9E8y#o(8D z+-(a>J3mVwD{o_d`rNsY=I>$?lg5SV8I(bRZMwGt(>H0T9-cY(f%dlaBLUS9S`!o# z7B&{?WtD!NRx+n)NH(`)>;nj!Ihrvlw?eB@$eZ}ypL>5skRz_5883D6^WXrD&zW|m z5<#LSBNiGw&-!ND#IN7A*zq$u3rMGgvj0^;;wz9`xP+T&7(bl+PD(frn~y}zi0Agp zsh&gY<013dBjg*CntWE zNiue}owHqfI|p_=kf0Z)Rb5V4CT#)I!@RS!Yf zALid?5#Wy(HWV^Q8m#stQnizn+T~@0<*m!T-DR7!wPu&@xpKo2hoAra6V;xWE-~He z^G~jdYsD67bKkpXBCryhZLL=qfcsdwZPMAebO?azUg5bM-Mo^-xqUaTI#WH8#rL+D zcplhd`S5t*Yl8qrIW0F%=V!qePsuwuNtIr+`jAX`rhQ&o%z^g`J!O5N%F;F8SXdzc zP~-gH(-vyy%W<$9GUb!<8aQT~GSm%YT!d``Nt#Zv6P^F?@I{=dafb&lz07^1ty)vg zkp8gfgsWH8PX})(!gY=ar&D&kj1zOddcBE+seVz$zw_9^n#1)E@=ciOjP&#)xt%v| zyIm-#80>BqaIQ*d{?3Z^qc$mU(<@dgolCqjNV^g`+VqH6SOWi#$G5OO?%a_j>Zk;(F$DC%x z!anard*?2*c51493x|cQxZ;1FBXZm-jBi^B#i(=Sq9$E&jen2o%b65S#k000hhxpA z!qZ9C#$9+^(Z^Pb9n^{Ua%Z!%A7JYA1;k&SbNSNA%HGn$%SC%DFCv*OGFgf|cZx5h z{%)Qr?r}T}Nu<^=U86RkcKGrT{zQ3(ajX`O_bDgg4F+k)m+Z=1Je`WKC1&3%lODIR zc^uNZB00Uj>)H|IO(6M!|GW)*=5=2>@-X6q<&?=bQuTL8i6CCprh`vag)1n*gc*2h z7%Nqt%s>6en!3qcQeLoljR7|Py}?)R?;?aH^XD&&XYAVvw^`DC_;7=z=3S(p zD_OQh!0QYSiiHx&k{=X>8Ae;t#>7mmmmiE6v}Jf(h^^?nT^VvXN}*WK+&sCd;`BNy zHWvR&B2@@uZi;@;v%m)na(X;AS7HlVXfa}UOZc1gODvwW6pja8Z#7@CWXn{7*GxVW zp;18rx{2b3!r?-W*B8}=@v3V+&G$OleNzfHucR`Q&?k$eFuc_87@C_Um;ejrCu8%WY=d5guKc6~m(GMGlgpku z{`%d!SFyABA&mK%1z%+qZKM|cbn(+)-If-LjX8UB`dlgF#h&ylia&k0La1#N_!w2i zA~2=i@z^K-`0PRG`l7|Oi7Cl{%lyQqS7=!VN5ZvHsao4>XJVIAgL|Cg3R>u}F!ghy zJ9g2bSv0)%=ZOnq%!wR6Usrsdo|+S4EFA2%JH(5HYKyQU%8#ynki2CpPQ7WiLX%X&iDY{vHZ=DGiQWhy)}jPwz(qJPHbMA z8^0krpuF!{8|JcmO&olmQ11Yx*?m*T0^f#%RR6#-1IR0!*{vSY>u^k*%3eFwxUj+A zWOODb#=FyY0N(-s-v@mMCr`Bp26cR+*BVCngPpG3e6`%*tN5{-=1ChheOoTGA9llJ zz9oZ*57)CRMfPu^-#+II=@SVLi@oRyOKH!s&*75+p}{OX=MtZz8=+5KZ;7R}%zM2T zk7B~e-;%c%59d<#T^XlfrVuUB@vS}D7nX(^wkKW@8Whu9Y;zI%b|hx`xq9@hdi07F zP6&-rxL~PaJj=hIaxGSFpP%q~T2F=3+c{naOI%w*xEj%4)ogs}u_)sV?d%JD^6t?; z=5(_j9{Bicl`5o9C_FMuPWQ%z9#s*6&lLWl57gADKc%$O9b?R%GFLpq-*q;=$Jv^S z_}9f3Kd5*wVOY;KT+zBpQPg8l)NjmR{jFHSX<_l+rXznNkvL<%1eo;b(PCelAh&bgwq)T!|ax#-CZC3csxicgdi0 zeOpG2t+U5hxv3&o{B`9s=NfzkhcB7>CA`L?j_liD$jkbnttgdYVZc9pKKq>giFM_c zlaz{SRRPM!v?9d#r}mG_ww5Nh z(=Y7hF{j9w;I`q6S--`}UPs&X&zRMM(d1#hepJs}hQBgr7X6(9V|1xyV{THYPI9kO};MuxpxRA%CHp#ey~@rFD2)FLw)?`QmF%V*^?rUl4W@AJQ3v3J-Y@+P;zD)0S`MNW1>{~;9?_-`=gX1{(LnjEL`an#Ckus z9sh1L`HvRhMt9_sGyHe*ExK^fr*z(Kq3Ww`l{1p#i4(Nl^}FO~DHBU)Rm3n#c1b^T zrXW-q<6cd#v1Mr8s5jhg;yRy5EQ0@n!RlN4X%5-lp&_kBV|`6Q+wy1TRSn4u^wHn1 zgvD7}CHch(p8ZTCb>am#UaC=1hKccm77JW4V)Vwtq&qc^+=LsXOCWbNghw#6zn?vo zj+14Y?7kG{YdG7wD1L^8GjD$hV?IM2omSzDHc$0Na3_vAp`xm44NG{$H zZ@&^FE^D{sKsjggIv?lr40UQ3Y!+$x+OpAi)xPoASL!d^u9&%CU4*gaJJs?zA6seA z{h=*C<>u1$=RylUECi(VUk5de1RaX5U71sqxhVwfi$5i*__%_6<13jU0|la#O@UEU zWr0dZ9)S>LAI0s}Wv9orG#u~Zr)w38G4o1>p^P3W)1awg<4KitxZ1>@oD zw(?+%kW^`z1=nXNf4>f%NL?ecZ*I1X$}Cp!jXer+e_bFm1%vi$5W^Rh6HUr zI{uFaE-SCCM6o@Kx?-qPA2?Dq&Tyy<4|&)74xq+G`C*Q-xfX z-wm0*eBxPSz$bt8Dn4{=zQd;{0WQdt)=u^~+?QYr(S6~s7~wD(c|Jk*}ZK!LDw zZeS)M4IG0ev)h7|zA;FfP@X079NhS;4w0E`9H5T`aa5qb@bk;=(CJGMysaMg z`?<6d@wf{gIO5Bk!5=O(1rmB>yC^La0Efp0vbAgpk6FO+!a_;TA_{K3NMHQ zH8W739IiX9w3`?T%YwF{H7Ng(AMNoL#636+RzDJi&iWVktx@E+BKXQTw3?1#a{!*ww{lqHH7gdSkyvSzz^XI{x#^Tkz52%3&1Xa_q4&!apT4f zq?W`2F0dhR5q{%lK_C1INu)r{g@sIoNU01qyoYSaHVmSJBqto(q8BfQnCH9#ms`!o z%f*KAmuw^j=cT)O9qG+0%CUVrna)d*iG680=w|CLdjB-pFr|-4)@m4w;@OE@`8DNa zD&e2rAPNPsSX0#0&!6gd?|NuWB!Epgi0x+7(*j^Igh6u|B(9$Q`q37#F&U5kHdT=z z?@fz@cWdiaXgz0zF7Bj44dewH(up_aNfB-4wze5>--c&oWZW(1MR3RQ!p%mv{TUaC(5hW!hg#({a(R6v-*WQ5vOmFY&gFg5H>?aM$ z&}j9j|J!JV4z6MG%(`4#M14{7w={wt^VMnx1!{bb57BYxbVuAOZXJLrUMMQ^~a z+72^!vBAtm;bG$?-zL3*7#cioXWaxsZOjAzn;ne>YmuHr?v#lH;3Dqn`S##*TXVCD zs;YgX3)uM2CdS8qv8pc4$Y4j>{>oa=Pl8ny`q4k{wPEbB@)8OIK&qe6d-GF^UsP1o zn(brg&0o|@`9NEeSlk9$Pdp!xH8FMoXm!sz1?og!13sFv10j)J*_}Cys#Vt!B5|hB zXS=S4N8;$;42ofx z`aeJSx+n7Kn=%|LV__{%K^xq5M{B&R(a+n51`DS>4jD)DC8+eQ@feEmC@=vxKCrU0 z5a#yf1TQ4X`Npn+1yKStvZQ)i>gtqq`s~oT)r+k8w?$k~pog6KqfA`jLE=bsIqg_e zQt}d>nvIIZA+anjUS1iFE%#b+*Y&f2Z~8fP>_?IbNK{cQB-n%3W*@bAR2<9EeK(yN z_+H5&F`^B5$j+WUd+JYzSa0dFPbgw%n?D}=(y;iS4zbP`RkCW2r_k4Ds(ThLGeVT~ z7=Fs2dAt|y4$u%bO*w!kBbhh$E<8`>o=z0fNui)1oXR4ATL;9N&>cR;Ed3E0#emhl z;TICU*H8GaQ_0BGQ+I;$X#XvY+x)4`3<+y&=8Pe&LW{VaVX+Thf|)5J+ky*QC-9mq zTdV>bnUk(6xQeLS2Z*JnZd^Tx$iC;!be5CT2z-r(by4VE;kdNua~vc6N5cB3A?r@DL^` zqI%Gc6V!v{Ed+9=4}XmvcEBjG0_1ww2p<{58|6bi5?tu`7I~c^D!0h|Kz{A^ zRjTakAG8;aant#*8)d0bvi!!UY$IF&bl4jV3fnIA7gVp_0B7*1)jsT+hM<$IY5mha zW_5s1z0b7CGLWsTsVCl8q#LC|47sfn8!xO0p3O8Zmr^Xd5|%07GZZEviDNv& zNi?r2gim7WjvrEir}i0h-M&&@-H_cVTjfEbxK7wq;6MNLs^baUaJA3!H?B+ON9=vy z42A%NZFs=SuK7pz9&%yKRUR{RTZ%`t|%K!A8 zzc(AOx)S{t=PGJe{QeJYum3dIkh|jl@ItKN#lH9LVR#*1){lZz1obFKQzdCsTZ!Qce`+97@Zkl?2J%Y zYJaYP-02|Q-gWZwFAl_H(%Yv%=lcb$N5~eg`oDY!HUN+L?^mzjBM$;ddhff>>7Aga zUSmF7VFpJb*QLKwKXPzN?0TU<(>~2xVTp;yKu5yTjwU^L`(S88dNT?T7!YbkSQ@ZCrX?RL7q(=I5@OY<*;!^r z{u~wT3Mrp_dt_y&tFw%W1RqWaHD3=4gU*Jyx*6ms23y1q=HmzSXmR^_cNiK@c*QV1 z5ak;RY2nMVI2Ip0az}sL`hZv|`X$Z+CX#+i{#`)x8snQyLkC0ElngSk=p@5-Z>rNz z`}Rj<)T>*Z<<7M+XxdlY@nNt9DLe&I1LjVFQH8r)d ze+-;2OLd3RFw6$mE@HP(!ICSuoi)@Y*!xM$1wGV7joS5&+r|P7UYCIrtsE%ezx=I8Gebn-S?7xZp6T=7MQxHGJwRxaG+AzT~l0g(x*(N$;-n^o0G5 zi4*Stj0jUdWe&xW-x8+Ulc|B{5d@U*q_M|NhyE(~xud{+RSduQeAT=F;yk$)kV&l{ zN45q1-Otf3fRsTHGALi~o&B|N-!l$5w1JDF1sD~8lDES_h3Wa2^N0=dPQ%p?`x{5# zv00pyAoK0gfcD@r=BXbH5@5uJy8%GflkP1};LWHw-p9=}0 z0C<8-Y%mG$Gfw#dfCy`DNv3mEm%?>QC$7yt@nc_k$+ z0asNmEx%7cje(za<0>_501Y#a;Ry-(_tqz-d(yzcF5tFp$2N(_5ssuFD8xaa5;$p~ zMw@)8C%B~v&OIimvPX6EwwxsHN^eBiGZI`^hMs{8ISlC0D}$OrpD%q#UEBpkH|RA3 zd8ja<9z-GN1gXE%ehUkPQAW^+fCvFZ43geomW8u$JD>*eApL+u^ot!XT#Wiy3uprHYLf>!|T2X9#YlmoIAVj(LR+_UJ@0b?ZG`oVB(vX2|xlrK6s81rSoC!8YA-Vz!9_Wmo0qG726A6(9B}EY7m6UE26oXKtLnNd_6r{rf5s{FP5|k1U z6cD9^J0IM8pYNXW-T(dWckh3XamLwO1m5@eKF?Zn%{A9tbw{T5hw6~^_Gd833|7XF ze;eLvu98d`E0&X94+#`vn%r@2_zl8aziEDGIlcz5%p2wP$Pgw_cUhb!`$?#oBy~EY z?20)qYg{lgiVE%QnpN9XNDL0}2*CtLWXm_q zL1Vpu!5CSNu%#BGc9#OfLPIOz$FT9%KwsZAJNehOwd)l^b@lX4ySux8u&Jr7O)4%f zZgJGp`*`-YgZ1*5C89#ZhP5U3&2qMjwAQ2Yt5e7**i?WWbs z^AFQ4xv!2I2hXDcMZeLHIFzD?cR%wzhVY&fIbsM4D144L@x_IC!4pNv5BC z0LlTAh8ee6m-O^)r7qC(V5A{cn1vi*7Rp4&nD3u;H zG&IcPV&Mv97nu59_1PkCHm3(Jmj+|+PCQWcV-mgG8)O+15z+R)E&Wl^pXib+`9AFm z-z*cCL1<yS=DU4dMJi1uRXy-Qc3Rkdka*!D(E~iZFrhlg58)b)oueKW}Wh<-q zF8$FOXlG+%V{%dn6QaO$ha5(!UmaRGv;yTb24`>Kv-r7#NyuymL7ecBYMH`#>zJc> zNkyJT=dG6E5y4VoIj3J;Bw6+@ zIw?#sa!*C(pNaX|!kE!igZ}C0n?;xbQ8!Pz_7Pg1jN-cW}C}DEnAk9&cfDczB*E3rglJnV=}wHaAg7*4cHu0EOx=YQI{EW6!~ZJXC1*MGvO+5vU3_tiN$Kvk^zYi_ z^@{hCgG3yK%s{JKHr*dtFf)|qG?bErvq{W?j4J&ZP}>kU2USIhS3s4=h8I8+e1?h8 zJ}CQ$_lHyO#?1WBx0L4a%GMEv0A3Z$_*x~CM-=QpuSpPBH#=A{!M?HQCd+;JFKheY4dq9NVbF*a&hlraHf+1AH$GviI%Ea`W zPV)CSeB&JsPrFU;wJ&Q__g6Aml9hImcem#ioSDr2@5UVY1B-4zTt$xzt z2oe^ijrLh%5J-<vgoO-v zl;W2B8#4N;t#xl8Y!nJ?%Jd^Ka-lGP>AoUl{wm?GH%q6@M`O zjkJBLpVBAV_8e)uIJLM6;_m9i|3V%8|8trCU;XiJ-ZxX27cb&*5mKN3`DgFT6{^W+ ziye8%BrHB5*NMn3>f7p?nl=#W!LiTl#iwopEfJKpKZv2Qz5p~4TMfgJ=by<PdMynprwK6^bum$BN8o$JRE+!7_UaorQ6o^_`*8)d9i7v1tS%I z$G#o!S1>i)NBDA3_;?w`u=O$+gE<2aAPUE}b*PHq&n(1I3da#3vTUskUpm2Ce!GMz z!+$%-@@zj9o0y*N3oimhM;U1H&ZK8%`opkB&*QfpB-&H;0q!q_OUGz{owjQs^FjcB8N<~w%67G{H`)gxrK{gy4 zgKc6sh7XYDoZyW}=sG-=h5QeYw1HCyDARp1O&>jrQ|Hg`1#!@W;^)E z;hP>AYWJ8Q@F4z|39|Rdk|afm$t_f5_8f9|BY04tK*3HyEn-|nv3r7n1WX^lfY+7{ zaE)-M?Pfqk@rk$mRM5!ZLTVcS{$f25a1uvSI1@O|PY*01cnX40rn&A_aN&h%@?i}P zjpVQ&a4+4weS0O^I6m(Odtq0!`{2P_2;nO{%YKho{^c8k!bu9|F331YtUCXC*9|O{ z$;8bR_OCj&^(PFfw*&2C1JVn~W6g=dhgK=<)L@{3S(`PSMeycpMHxyJ<2t?u1}+JS zi9hMMVatpm<`zeYqG^5hZDGQQ&KcLnWaTp)A$mPzE!ZpS7a|57C9n1ZGD$#iKdrZbRBpBH{nRd zs70(^!x6V&B|(@`Vtw%ew_({z>IeANgoK1gv$7~o?6OnS)0ugB$GpexTP&dDzt2OG z?B?MY8wM*K(hP%zv4ydPCPD;a~&K~7fWCU{tHJJFyXocz;| zCM;%CVbieSUtc+q13#o5(Mv*9>@d8TVT{YLOE&R%NY5s`Ag5btvc+5~iGsQrm5zSh zz#1~}0dOW!LG$7q&ca;3aM>X5{Of%I#K9R!o+JYq8JTm)$HdxFeD3>;^fTE@BWn_A z*slI4aOfeXDM(`ELhq;;Qw<@rJh5ePmZFb|g%58IXBN@kt{0XU zBxTaTgfcu~w4VM-_R+U+_xbT-fc1{~-&z1$l+^7P^J0ig;^hJ~I_i8<{!+dZce4HQ z+i}H%*)^~KUVXs)eX{6|WrA~>I4#5kN? zGZUoD!N#oZpYTlyhsaC=P7t(PDv7 z0^HoN%YcNtu9@aM8AZiPi#+y%jfu5eDDHDTOvL2;qw_E){?7CBs2>bYgrLwSyaNc) zC9E(mfU7uZWj5;<+Om-nkqR&1CG0q163!t@0*hg>a9BHFEk;i=tdE{Ju>w2@ zhxSunTvFn43I9w%d(-8G=L#hknHm(!WX1*U)@!_m6+8ydm%Ag;ejlr3nz*e$ceS`! z93A|^I`jT_(bc3RC8l1)mJsVmXnnkpn{UI&&kBJgcYl>}kJQ41)buZp1qZ@3;y}nM zDD=USm)1>&a6gxkxk)A=?+cA|`h(tpX%p{b=7U6S`mNx@PhvQ_+tqOLTmrMsK4L~m zw&lXek3eX=1ztEoKgc-omb?^rbUz2~bQ;dcBk7I!m=p77LZjXd}f^vW1jV}IL@fV=Aqm4QI$x3VjA_Jj zSU`nIx;cOFJVlI+aq5h~N)Z7+9*W(reFO6PGn)u=iEBrUQNNCPAj=ZZZ=zbTD4Ep2 zu-=w!3-Ej@2gUIc5FuRHk`X_L`nD99ev%?F;t7i3{#s zZ384g9tFq*7hlYTUDwL`5w<+i4}8@%tw%rL2+3-;`%UJp+qc=cxJCsfoqw3XtE%%- zF@7AHb$UI--VFb9${IsumD+M;2rMcvUfbtC4YPOq7&tHDyQ|*384`-O^uew7(N#pk z`rT>Q;5julWkJKheA$GS>u+e1#Po5N_O{BOrO<%cI|*DJ+;>Vv)KvAY>GriSjUw!ICLBV zv2tdPRF$y7!DmDqa6rdB|4im0$vh~|JMj-ATM@RvqMS%ta2pYD326!HGuDx3M=z}3 zdQi@sFZW!+apsZdOK2N9zWoBms(l1%1|-vKxYfh;ech`Pl%sHaCZsW1`s7aHFMEFm zY9wat#IO~dQq*6u`Id_~BXK7q_>B-<=*aqGcA4dH!g}7|rHfGVhQqU*#b zzX-5I(f}yVCJzIMoU8&RkO0pOW|spngWe4*XtLb{{_>A3I>iWoMph!D1kj&tYk`#( zSv5*RAf^Nf8x4J%e|6tyty^sPy!O+KSqTQm^`lBShxD>tzBzxag2&`6+~)>%k_u! zTl9ye2tG7dbx*2~$gym7Xd6DLUBSnYeK|*`j&J*hBeH5rTQ;gGDg7KtFOw=RPTF_7 z#&^E|gOg-tb=o_p&W`fgj)Br|6a6+GquXe^o}gWH>*merCMTpbhH*4!SUEW}+F(FD zP^PTcG?Ud=7IgP+EaGshqig@$ofuL50#2-SFbiG-p`8|G`-e+A!%iK5UF+)?FT6@r z1T%FC;!uYA$kSS@D=XiHZyLyI^^u?ZWl#FkD{~6Bq${IWX(kl;7AUG?`si(CDb0`T zbqg+^Kq&;0@Ngwv08k#d18o0-&Mn^k_y(juS zbkT2u?(F2A?~$K2#jLF{esb*Ii9)f|zBZrG1*YlARnfcp7#SI*Eway8n9j??s!bC$ znqMW>lZ&~{|5#|vQIUh#V}eZ(t$L|e)8iL#4n$8Ko|hjT z4P};|co%U(fW5c3Tlm%fpWZc*!kPvKv8N8)8$0l>s)|=S zR>1P3PAIp22%d0cq;{~Ag+=BIv4y6&@xE%oekt@Va-GmkvqjfY>B_bAC>6xbrPe z{RWG|Z*sYmj{IzQR9L$L;^X3i?%gxg39;jEH4|chiE>E#ti8=pBio9VE9pX$VjrV1 zg{RKO%6j9x4huWGJa&O$BVKw={4bGxRSB?k2{msFt1%9*G4}cG$C-3oB^9`JpkX!^ z6($}{Hh1YWm-M)}t8|^+684{8uA!)1Lns785w*_lrQk)eT?{OW)PdA0b^JL%vRwkC#R}3TA+o9$-uNAbQE~LB+&kp^!`< zosd~FcvlLFPF$6GIL#%4^wWU%;0@*!ps@&Xr^jGnvQV*nL9l3q;lcwn_v8YvdU@F) z?3&^y;|;zibI5n@JOJk+A1G%3H5tM363`o7U<(-}&B+l^h>_F{v_%&=6|H9Jd1P5u zDJgw4>I0c+cJdv^ly*IOh+CMLvIq^4s-6~N?UM>(wW_5>6WLD*`$WhK0W=uC?=pIq zSs{$)2)dr=FD77US`hQONX@3m<;BYupNQ4ovT$hy3MCcm!~1%A;_K>^PMkc+&dXc> zn(RUG!;*EF$~J%%PsG*P6}nQk0l!oLk(u+%qr)&AZb66VQpZEtK;jKP`KA6Cc_>&G z9fz*F9$qf*IDYqW0|SEyVaxkyHfG@RCL`oX&m%ddpvf6=$RQKPaXFV-xO~=R5pVPO zSljVlSuo+%`P3CPH3|?p-+gty9%sSs)AJR0dAp7slMolr!BP~`MaQ!}$i>x+^3~VL zbf`Ha4K61K_w73dU%%j@q9Qm%zinPm-mK@n>6*97{E!A1ddsPcOGxCJweoRlKTUsA z4BJet?DN+UwUsZ7j*Mg=C8-y=>+9>6fGTL}_oEYRz=wXu{9}%(svCezQ~#AKSG1lO zE95%Cg*gY`PMP@r-3nx3_N&DQ+x;?sfno6pd)|AX{O4JBGn+In{t=bP8aly0|91MS zG!0e$+C%%ve?~F*(f#>jmLu^i7yqqXwk~{`UVwsN3=nnj zH+(VqLZC2D57MtS;u}N>{N~-e%<)+g!w~eF(Uwnri{40ZSXgpLN5{bl#Cafljp>s8 zkcG%#Ns7jiBS(Ur5Z|GqOov;*!3jhd1~_peI+{%{z5$Uvp(c&UlW)4Z${NaGIosrhFW6RYJv9T`BLyFo zGg^^b>FLu^kcswoxlZ_LJvB>VQ%y`JUqg+_Fb>PzRD3l;%r2FD4b#$HdlmqanjT*u z5l;c9pb=-h{|;$;tfi(FD4hpPI;1VN_<~PY?57jN`+D3yQc(u6sGw=Xaa_sEXW$24 z${&*5YDU`%dL$0eb{LhSnEPCQ>C6y2PO5w$yi$qJBbXB#Ie0IhKOZMK)xGT1%a<~N z9NTWA%8dubcmk~8-|-T2vr6)nga6E9!rcH6LU(4)+*d_@-QveZLRs{@$0Qhn@VD`WSJl|v}CPo)HDU|Eq;~YC{vS@mE7Iu6+9Sl zwZ7WV5`0l%{3HeIMvb3@10*5Yqh0{UI42aTuA{@Ds8~zM;~pFmSI@mv&`(eD*5flA zwC#!p2I0)=&L5jVQM~T#jQ*uTgGX#66q%w-gI+}C54xvJm~J+=D7VN#U#RtMeHObmIHbCu=KpE1E|SPei;7fE@`|*>0$@9!YQO&Yda9cPZdu3orNG zq7yLbe&mXlXz;yz`(zh1Mu&%8Ud*9(5$p$}J=HX937zu8;#65bdcm8 z*zf?#;OUT(19_J_^WLw5@L`LOF#~;YNJuIwuePc}-QUN`kwSyQ!_Qu@;0tHgWnlR7 zX|(_GH{PPoQJ$Nw_Ek;t&v4;&ou$x;Kj7f-i^5V^*<_qtW%Sw6&TD-u`ZILu(Dcv= zM;)HT)xncTogjSc8L$*qc^t(xez;+(#;QQN28?(diUhKe#>&wh`kjgeLU^=2xfnq1 zC3bVE&d<-&vixdjDmx{N%N#hL0srl8{Wn<)RQ7J}5E{!lq3Li*XJInlEFo>5IdZ38 zB@9Sr&?FEvse2GPuZ1-U=BV5EP`IvPj>;fi(l z?gwv*N3Sp?G^o(fi_~2>*$B%2jhWCPU&&L|YCmEu_$@L@eg>-dQNn14`@j1jYw+jjfaYZxcdkAoKy={PYo zlcgvtYdtk>A?^N~br+S@3H$ytNJXBTty83H2(H1(j$yR;Z5<_rg*Bn};b&wLvz6&` zpUvPEwL}vh%}9feO>oZE#`B60GLK+R(Jyny>Vj~>0m^D}Gm-dY&Qb z+Q}~4R^pO(`o6fS@1_X83%5{a*?p@tXee;My>&T;loFdc(BlB2%5bNfh?NEj5YP`GCnS{V5HA33#eZS1q8_+z{Uu} z0E;aK;q$*|*(D@&Nlk`3?N@y(b0>VY$=gTnNWs;nbQb0wV?=zw3YUYTqGu7r1tK_F z#9?(tC+V6`UL@is2?Dg!;3~J^Zz8U;iHPJk7@ZA}oJl@#D(nHObXI2OmnIE56WDNL z3g2J=`g8tj=Eic3nGkg4g)N^zSCq4c|?b5fxoAVATtASOoN z&k6Vd?XY;tgr}z`g=KSaf^kg*3FBsyi$6$&ZFB|o;=lbbkrw&tzX3AO+mG5)2*m3C z`oDl@|H(IzKl*nR_kZ-)tiq*hkcpC3F*=ri(nmFU7gk(fuu|_HVsa9a>Bz^D_D{ng z3VwjWPk}av?vQ;t4mQI$3NKE*;xpKuS&v0Gwm9fM;;0dze++Z}HJTxN@4z7PA}y&I zNhtu<<`VWzh&hcX9J>!$9_o@*T&)OR!|P~AGK@pg0%k-WOmH&{vl4K<5A`M~#x#Rw zCrK8ysbVOX?)hQd9V)8ws{B*&U}3aCEutt28_Pn^R(k%&d&OaND}bh|NojzcH7Za# z@QmXg4Fbs2A!(P@;TvXI%x$2-s{w)m{YLes3~fzht$VM|-(@E71zp_0 z1T1M%H-2&!#&37PA~1)aIzYo9hTXJ!&bZKaeQ&nn%D9>RCU1;NWaoO#nF~HNyArY5 zR>QdzF&0;wR22l3kne-ys0B^>EG*tQCMqfl^zR29OXEZN?KD_zi^97ZykP=RCL7Qe z#Flkf#@zJOTpP0u{?A6`Mxv zC6whZ{(e>R_YYOWoLlJJ?XNnYos2_MVS@@+4bYEy5Y0MdDHR+IUsq75(u~+TfT@R6 zKyupY-0FB7)vX&h#^Q#mhhrWt8RhKwN`3G&y`d86eC59 zf6s4oNYgEPiq|uIXFa`$(&(tBz6I72AYngg=+mZhZ~qEMbSf$h03S2k?%jt0TkhD}xrZT(97@l^TVAlf9~7i!=nf#f z&k{+*Z@5hPBgzcQgg&IW)U1VvKqGFbk}7(@tD1rU z8*0lPR`5^&IK^3Vk#BY@x_SGy>ha^=yp*-U(c(@jG;{(=pn=&c?6BY>uDty4^z5u= zxJw1I%)`E+j)Escj`_w(2T*1MKp6DKf9vBLOgc6`Hm2N?$)ICDD{40s1l&T7#JF*) zG-45#UU4$;i;6Gh497MG$a4*yw*tw6)j4HuaSjs_Lu6)QASu&#qE9~wdgCoi%Gd~) z2N$l5d9G0CZJA{ej#>~2jQGNP2M5_?QJo4&Za|5zgM%1YW6Uott`nnQ>{$QFMJMLs z#fu9kov=vcVKL5%v^&H?bQAigU0peu_>5$N4;A1J>LX_)%`7`QATGO~RU zs&{tk60UE2{Ct)g4fg*vW$DL^JdALQ$jfuc%`JG^MZw%&wGaiz09c%X(rY+>@kx7d zUK-+AGO|~39z_X)QrvshcslTB{)2+_J(rqNPXMTc(tv!RqCiAMf2GdMhfk@8igB%y zaaRKqAo)24ZG9u2l4$9eL~vxE5_wNnD~-D{%9#r(HeEj&)8PX(yOU)eQTS?6eG7LHg z6pYC3pgqME!ud{+Hb)Msu&vv+QPA)UOu#XspjM_sE&xHDfZ8w>i6?@`;3{(pPBi%! z^ugmvVT?Sae)zB_mTv5q#VJq2dcREMQdlDa*-BbzFhQyzyQfHh}m}k#p!IezPXVw0(63uAW{aSrF#j`1o6W@ z?A$Mq6ZKdwk%`ig@N!r>OKW3*hWsfo?CU&AuLf=l)r|H zo0|kS>LLZG16$`}Shy)gweq#p!kkS_l&C7CC_|k^dG2##Cw*=n5jOkq02~0jkdOwZ zzUz@i7!GK+7dt(X_T3Q+mh*(PGzHLZ!S!xuXD<5BC_R+$6G#Y%6Yiyvl`ue@)v zX&nspxpW*~^r2}CVaNgXa71d;wj5JD39aA)AYQ^%plPqBYqEZQO^lR2kU|#UuOB}e z4Fe=JMrt*%8x*)CzpbOXS{{QPr*eVZ0`7avJ3V#$^LuvuTKb37`tMO$RvRJY+e?1b zX@8Jr=+7pIj=5$#2e0L!bXq;v-z@vq;1?;k|} zhcM76Y6Pvi%jKqVF6?llK+lqTDcHRdzXv^)WE>&_h1vVicdG}$CV;~ZE8R1g7T+#Q z&!&Bp)6rrNk!c5scrFJ7j>KOO9VA2cTY~BcGu3)ArU|Te8irEd8884VsG@0*13PF= z2w{PpY^G)iRZDOs!6TR;B8Ofht?6|a^fBb@9J2pfN4kf8Oj3IGMV+s)k^WyGn$nQl~53}>)e zljs_f0CA9wO&OsICFDaWD3lP%oH4F432DnE^hyr0vMS+c{s8P_1K$Kj(ix4<9QWVO zgcU-7RYw9OcZ&czv|pO>nsOl91DT=-7@>sMLdz(gjHQc1Xv}IATz*LY6HLQF#67gE zwQ@}Ew&hu7Vf*WFw8O{xY{SYebabiULhJ?`P7rz?pFc^ba3?+rfUJZMf%F4#$S9D5 zP}*|XzAat+RHM9cC&$1}MF_Mo=24pD@M`SOWCCn}4*-gzt;Isb!z3s#TQ9zw@C6Md zP+{OlwgFiUZMRrRdt?I9YY~Ln?>BZ@hcW}2Dd4TN;O%qDk{%IpyV%!-c!3vgp!2RV z?!-Ezo42paz{wGc2$40IRGOF2FR1v1F?kgbYj{ToW#KNb1Qd}A1%PY=Ql~A{Kq;W^ z0u%7t*#!jDKstQ>J64Waz0wy$j6xa+mac{PR24;DNE4-rIBbx^7YJ4;h17CLoE49s zJ$p8T56rI^?Cm&6_17R3D7|PUSclnQ07D43MqnHOT5OJkv_nu!(3h4LitpPOUy)U& zJTH&9CGB-G5WVIEbfPdD*Y%_2nMGc;MP65+U6%!#jwE6uPe0O3l}7nM`gr@j@}Y4& zwVhi}4dh*L0VyLM#z>?UGG>bUxa#TMCx2$mn z3((q6ziY8D*2ju;^`l3xj!j{D58zUr%Y~m-yydi~=<0T`jOlRd_$fz#=Rt1ni!Lcw zU!7M*WO`Wd*`nh@N}jE`9Xda~0Hka_xZk?PA^ls&sR%zNnBw)OQu;~jDL=}&C#~S) zQ#O9fUXVeTT*2^N=mIC;i~!|(ui z3``(=|0d%6sxvKq%IX>$_Y=r=fQ98GzJvZ8(k) z6M-jyfHX?la&;a{L*l|0SwlNJvr=?_Lx>s4zz=_s6^Xb{sRHAh7SOkTH8!*oM+P-T zvhr5)xkJTlZqbeV54;+WCK4MyT+eEo@DRJxr~Ppi5;1t7iOTj3O{j4Asfv+2%u{D! z2y+lDtn_`{c0fD1!nR(hBnXmCts(52#e$m~suJcWtvcrS!@|CNZy6t+K;##ln?u0j zle~j6|CvFMZxclw9fTLFwk<}9?_C3%{34*&BTpP;HEEV$wk;8TL_@w$U~MP>9B-0=W z9-t|NI)Mp*<=}VDK;jq*IufWX2MC2gt3bl_PtVLyUftRX-n#9PffoYZ`Hs5~U#bX2 zDv-&+3)yFWKn^pEYA!;+G?3X8%xt0S^kdS2g<~%i4M}*TNW~CoR^?9wGD~1m!x`-cbpUU)i?k47??gsMf@_*Y z?i+-tkuOBI#5ot*XD%itCVbq7p}9$Q{_(}27pmQQRDONv{oyklLiFa9PS3|>L<@mX zB|w`G0VxNu*^rm(6Bv_{w2JIAtW8xxXp-FH!{{sn2;_^n8jFbf z8Xz#R7e5+`HwlQ6NjpU=&w~7?q24YJ504D}GCjbl7Cf1)K!afT==NP-{D?hmtZqsx zAk^A{IipP6xUrguAHaQVyZ!LZN@**rNBCU!-K$dT%YA|-s~EKQxlTZicBEr6fnYq0 z_Dxei8D|Brky_&eK!Q0l$^@tsHiXQ!KY)Jj7jMQ4H{NfjRZx{*1bvwt{qB#^M4E3<=;TYN&$O0{^JK}65$~j+*(Yip7&yvdzK=A3PAqD zsgth+kxxA$Cizd$MYopaZz%e@<DIFHKBKKcms22G!mqNJ znryw+H^ax(lenOR4^Vb{r<(0L6yCCumg@bn>xgt@mzf|Rzr0#msjeSaA%a&zbVl9X z26{mgCA|xJD%N(rue*9`tKPp5oSg{e+WMN3JM^S?Uz_8X;^TaaStxNkgQTg6Y6OFp7#xN%6%UJk-Efp?)D}3)#Ot`Z&`kl z9JLQTvBsizLwglXbb@q~n6$Y?@o#3;Wh9+&`>j>~?#Z+EN*6VqAXj;NyLMlnzlSM| znu0+FKfD00jcbI4jvp~d@a+5dpFlItVY=lDT2_R=Mu2WTmu3y! z0HkP&U@f=wh+#i38h@dAdBU%f2{Qo_mAA7W!vw*L-|beTUJR`vP&o+BIupcL<80SBO)Sgukau zjW=_lhh?PJjec>Y);{Di8D}YM5*%6uIyMo^ILCet826SA9SK z(EFs^;^G~T&q6-A_g~ta1O0(Vk4CWX_PiG~l#tj4)}blZtLMveUWXj3={Dy5!3gy7 z2-FvjrATXNgz)hz3^m>EQMW_#vJZ-BkntZ3#{R^Edp^ajRE?x_(|1-mHoI!#B>TJ2 zB{!w&}n92YqO(P@bSBrqW{3cIa#R}VRA9azx%OLqo1{FX7JLHT# zj7@PXJY*N^y7KV4Wb2J2l7gQ=yfP)FJeZ9HR&t}ku;Rmf4i*D(p8MM3AC=$aJ^6Pf zXZ2)ZT<>kU9pS8jQqxcb3A=UTq9Bhik}WL-4;~!GE$j$Xw-{2=C?NA2`)0FzK&~6y46C`RNfSp`iRH^+37x>T#fyQilYD&qM z4kGUA%F2_YqjLP|%oueAE>nc`#`)*hD%#$G7(lmos~*x}%t{(;bH~-fwZOiB_yxe@Y|?_DvvHS$L-$oq`QqYvJaH1 zP$30-zu(l>VgU6B?gUy{$}%#m71HVcG$zvkKJdqXu#gH6DPA@(5CQ4{>B@MVHO62j zPNFN7u%iWv5jCJrl(wgMDN055<^+)3zkh#=*TjYHL@c&{WG2*Z{>z5{>4ra_`~RV; zg~jfv4Bf9$_lHK?m8rKkq-k=4jjTq62*5`W+%!6E7kY6|XYh`F7rViDKPZoU2Qa)~ z!m(?|4#-t>_T7dO>@gW&_m@2-X4F?WGgtBm+|lD-Of06RzndqnS58F9GG&pG|ZBo!~t)OjO(>z1xHe~ETj zE+j3#LoZ!OMx#+zEY#0nj-Me~)aLqgF}A-XKq9eN^esqfl)-?gD$Y=XhmbkW(X?d- z{2Q(Ve8ajUl=acQxIbyrb2IrAjcf07F19Bj0kj5{r;2+~F|m+}K^2HFkHxtC42N*! z^xfzUe@TU+8lmA%MNl#Y^qav7SB|32On7os!>FfAOCOajK}9%lXgsK#V>$kp{D`F~ zQDK)Kql6aZ2wG+CB?)OYkqa-0Rt@!&3G^l4IUc?~ef=*!=CoOT+=+^X^{@`W)tm`v zZo#Gtk~{-*F!Q|jtI=O*qWdKwMo>rN0M^EuPQ(ypH<6!#G{kQ|_V;h^@9$3}{3N8I zckkX6z7P#$)Qn-IBi8@CF`YV?DL{X4cu$n?szntQRk;(y?j2U)AQw!)9tPj9efRE< zk}`6v19K*V5cSGcJG;2p4Bp#({&GWu5M>((b1tRa~no+zM-^@*Yr^bg|Tz@t;7 z@IyahZn*VH(y{sXic$nZ$HvB@CaH@MyhJ~al$C~JcP%6*hyppN;(c7f+fnM{t|7i` z#o>h#=-nO|&_v_!Cn7iyffR_@VyngLL?Wv&#*M-WMF(u2hEf*)*8)I`%G8+SUTA0% z#=WB@-Lyz&59P1lTQi|ccs3+}2ji;N191)pJ=?V5hP&@?gD|oXS03{w3U4T4vI!{_tqlE{g44$+y z4-%<}QTCbSSFm`3WXCV{TwZ&tyG1!A_v2%?o!>ZHKDc8QwqWWrEcJ>{4ftjzRpx$Z zrf-^S-l^2pT>quLS}rbdRcAEs>Zy?j$~O0Oi>OVZRgvB&9Y~h5kcJ=HokTT;7Frs* zqp9c$k{*f%Nj<%1yG{U40yezxD8t}QeH&IVzPVFJn79#^IXXy}4penhg{K};o z86Ji8miX%isLqehQ%^8M5`;k;kA?_fG=pR_$PQkvryxx}%IOlmroq3VC z9^~;MTNmF=z58zj0t5`lQR8iir~AVg4AsRhdzF7022GfKrJv8II(hc&**9-5f*@MsPR`CMzQgZA^W3c_59~pZYlRC5 z&&{#QHk*NR6rx0wp?D`1DG2>V2Wo9Nu})$jWXzM6{5w|p@@#Y?>a6E z*dvRcCmg1JLbYl+j67f%OidBsg^o)$r7Un8_)Ic1R5ry&Y856rACiYW?{* z)beAW7r|}!ag)Z5<-jp&(4agV$kNpkfI^rknF-uNwrnb}xfp-*at+3u1fc$b+;s@b z8x@3KhVi8P&T^Q{R>n~viXH$)v0lQPkR}QU?_ppv!YLn{$LyggqBCjU1NL1Ia9Vsu zo)fsnn*}J1$rbf%B!JY#zMx^rre3oKU`!_02~r@CgQ+!yLINkR8R;qp(H?y?q^|1 zgKB^2?+V_yKTR1HXx!p)hg(PNGjxkIh?s);0tz5ApNuNdzzV<~4T3XG`4En~ z79MK>@yY984_=T+TM+LBH*G(}D1+2d0|Q{pI6pEEwmMhsFAIRp_l zs;H`xbb-i~3UErqBLwV#93tIu(w=mm`n2jdU^)?TVMr;F81H>v2(=#E{O70|qd7p1 zAkp(4oItPKkTDX#pJYC~(|YnPkgp7vUs(q3ouq9%bRu*y@5ooii0@s*H?*>uWr^GZ zgwfV5TM`J8ab!9CHF0x<=UsB<4kh2-qgX#>DTZJ>_E-U%S=b7A`3?M?#r#-cJ#d0!;`yVT*Ju z=OPF>ki<|TXp>V^)O2zxAY)jv4RCbO1Y1WQS6iW78l>C;7!4pm61w(9$t@d5eBDA@ zWgJ1I?Gt0sRDXX`IGHzI^pM1{@RZW@<~1O}K3VCWz6gD;&D*vekOeqS6F8cLj}7^7 z-3f=iIy1}3OZB`x1O8EFV-)p&LB3vI)jHBhWwrJHM)Tf(fY2S7K>@ponIx4yK5))$ zMGje*^-g^Ym!VuVyK$Q`-^?L#lP{n!Q@U^;u4GU{4S~K^?;og*X~9PiHSCLZi}%EH za~!TL2FZw?oQRPyikO5eJ&36=>oaIWm6Gaf8@FCMmu}&gl%)HYUzg2|%jcT6Yv4#| zkoFVC1zYzle2EVrx@jPGR;X7OU`#XoM|}>_JAt%l9HCII-J_4W1dIvh7*%Z|!gs*O zzH5uOzr2cIDg><@bYXzcCneo6<+7Vf_V^JsqTDARte)ei6{wYE13~1H!kTCdp^{NT zu|^I8EOTW7adLCuhK41v7eOC-!ZwhAb+jIl1~v_#6Vrl(L?jY0O*}$$LC}*!-AS@C zx-f}kng*I8XUrn{-bEWQs-CoUNMb;%nbZuVVULLO58^_Ke}L+8s68)h%n_X>gnV@< zD4E&N2qXr?%A^kJ3GAz|0~;oM~t%SA%sx+fsNUQC!tM|(RZJI$V?RONG*-% zrS6yQEOO8U=?7_-zHA^6B=VA2ieijpU; zfxf?Ss-x1?NN&861NUTMx|Rg-uNLwn%lZJ1kVE3)I&hERCrv5zX+(SRW_ia!$@+Y= z<{!W0m!MB3$8>a}#-{N253;it?Vo7-Fk&xz{vEw;bk7&$HMD2?Xw>}BhQT|{E>U}C z#FqlmwgJ=9llYiGoSR9hWAW#`PKzWVL5z>8VdhgX^~$UB28U9mO-?Gp6L8FsR4F#RC&VTm(Y7Jdfmn0*!QJ@f(TOKvt7mcl_s1`$e3M$dbvc zsGyL+E9Egim!;=nWo1PK-2vPX=%67%7r?Ds$ZTn8*>ka=broY|jXhBZ@kZ8=00Io+ zxLG|OqaF|{={(k~@$m$2Z8L7~MTfkB*bT`u;uSbkdEhbtCcI5j1wg@VSV+eB)Um4z zH7&HuAianp)xlxqoc_Yc!nSnl_b3~JBD4-Gr*5F5lZ#(j#wI|qh(o?0Yyt|S24n%> zfn9S7R|5MfWj@jjE*uix0NozW~SW3RbP{QGF6O!%W z4ng`ZevI>hiU3iAzX8(B>QvTIjPZe+Nv@%Bg^32yl8_Ejh222)u)iydESX`L`T0qF zwbhE8bW38(PlHk!!j}dpB){Gv_Zz$=tw)svgw56}^oS2I0c$mTdT;_)Z3r!1+Tl6H z3H%xU>i6gW0l(lp(uvt7gMmp$U8eqq@caN&U%=>vFcato2+@KDJ+mx+TVNAK@W6I| zOr4?gEP#%>ENQx6P%s`-pIU+NWMKr#p}V(P5dvq_LuzDj=G^`#NCj&G4d5a&Y7F@ZDGPvbh@_fX7VvZixtq*m5WNGGg@jrMcwv^s zDb6J7Y#4gzkns-zlRGeCF!jAr(A*0H#*pVPUI@Zhi_B#aLM1+l_#_49~*WDPGGbY+!ESW zPz@LI7AD>_Ky8%<0S#?6J%$B{t^*(fQ}SGkaTEfn88iloGz%JfwV8P$wS{;BcYy^} zS&|Bw$brN?6@x)Ttxs6EcPze))7Li&3d@2JB@7Aax_ZpEcOC4*q*VZ^4J0t~v8et4 z_7?3WT4R`KF^+sJ(S~7bJ!CTU3sj{vjV?~+QN1t!gxM{ZO#ELB|mMf>`%4NiBm?xFB zX(Sn%U#he*i+(4fQ6$2j`*bqu+534b|6MaSOYTy3X&$yWT4x}C^yf7W;ezU4?qy|C zM>D-S9_+vJvZW`AzQn0(wx;j*^v7@Gr7_pQy%5nJSxydl+eSGVRR~DLEsU6ip!%hVgNlXy7x#|p z$2X+ST_v*ut^Jh)BKVC!3&EKG$8;1HeZXzC?ZWCUT*kNiaVc2eALNWe*g|C@OOwP{DFwaFzw5T!OmZ8ijTeAoc&1wNQ!KTskT9?mp^;9)$^sI8jRL3^?E)a&#d(*Bw&zDO7=`!(XXns zp1IKOrNF!?7Cb;qZPI!?b9#u*3U>L0`YX&sa1kIxITxN4b`Cq?DH4f8k@t z(0C7GtC(^5iD!wQ&TM81*06cRc7uj_s@SM;#g5J2`inbGZCdd6usor_`7N|rrNP+C zK&6TPoWJ?KNZ#~z@s!KM-`dKa59b-;VGFV`J~wx3SyxxzG^%m9 zT`l`TmzAFWmEZmIFodJcm*$R}o125i7X3K|sjwb%ojZa&INyz6F8yF{Olw$A&QUF3_q%|aBfJ{>-> z$ef^{+TxLh@ypG@f=b3tH=I9Hec0}Qj!ERr-T)upMxzm>0jjN(l`PIM}4h>SmB+Dkt}JS zr5B`ZA)?gt0c0xrKCm)ljLQM8A?UdmPXKJ6FwMhwxodfpEb`<&_)Z~Au ze0LVqtf*?Kc2%P#D}UZ64U2ub-EE?~to>5Nny@jJx#r!J9RlH!PlZo@4{G9T8Q3?P z!2NYrMaw;?g|<;?M6O&k_LtVnzF20;9p~=Vno&|~%KFnqht(cS%3po9n=xROCnQWT z+{Gv4|4P{MBYjaUS(yTHnryjkvbhd9J=`aZBO)Wg3K*fEh)yciBm9c_nU*rR_#St2lOX1(qK@BpLqW5Gn7&c( z67nV9*;XQNq1mfMf+4=4RK(Pes%)ky3nvhsZ&zVNWKlY8Ol1LYH@F@W7gKy?4o2ns zH?01SZp}_lZHt0ypbW;7q<-I&VSbLV@82@)_uFkTkYV4aS$pE#8U91V%gz|rdg;?s zz32`bjL+dbI%|3=+A2LIJWuXmioj={Q<_=rG*g?V9lMUuBuOp4uQ_7gX21J zK7mR4JMekyZK?TZ@B$Q^oyAay!a_2Hm_{RIEWy?iV!-DJH+)$cneLNPGaI3p(lOiN z)iQhmV4A!mxU0?+Q(t&h-VEw7+yvS1nOw@&vgsPj8vy|W@EPR9eDq$fOvASZT3U1{ z)RvNG2fvYvn5ZGwGl@{jqG z(Yf5Gxf9l=GP|lw?J zJSOX|H}1X|{?GD~i?WwsC0wWx_p$NgZ2^C;_NTAyubz4ACpNcAJy(=v|kUF-BN8CHk zVlVB7>-D^Et9U+5@pbqw+w)!_S~%Qz*_&O@jT(dbxmqg|>YY$18MF)a@TuhQ-eO>R zc0~aILKv3V{k8Mu?vr!cWA1dt5*(EyA}@Wk3dGs3tgBEc7odLnwY9Q&#-7f#%G-~b z-haFCDd*_+3Tt|^7rV{!SMOSx@$?vt#jQwl$qtpHN3K@A9gE)?5+N7WRlEP@W~+i6 zs+!h+OmJUQt-E>7&ewu5ruW+dX&8a&7vO1r7{QU#z|IqSztl%FL8)RQ}Zcb$lgr zG?mtmj_xUcW?R6k!)3OclAk zY+m(56dFZHs}cCYG@ z3SI^J7bkQ(c$%A6h}J!dblQB9cdrCZsQl)&G5=bPOuf7MF{_2yzAuiV){|zThW9RL zdAA=+zeKBiiP>O*;lyU_9n!TZcHgdC{rr}9_VYb)cX_x1R=ppo5YRm??*3Cuk7Ccz zf2-d76#ZS_L|nAW*l3n%_o^R<<&9A>_)#za58PQ;Df` z9lzJ-*_yjyvt5Cy?f#9=M0z~l^HB3BbVy2FAAG*n$0|;lqrD<0F26+YR9>Wz{$8d= zpEV}}_*}lO4&0op^Xu??+joD`%HCxO870xFtCur8kr%C?GNVp+XKi>nUoplghBuj4 z>+|XIm7a6j8~4WUu9sQ9%EZR0wZSO+Va~0e=al?}%K!Bt?Y8r)Os3KLz@-_w=4$k2 z_ZijnGd(}Mf6nl{^?&TS+3)-cujBg+c&L{qhSty=u@BbV>?hr1wM^!pgStdz^XkhS zMxTBd)`qK2=bY)O=o@PgjnD4+aQy0#beSEMJSKc@W+|qLIfWWD!k+*7%~W?~yrrvm ze;JQ@#i!u&PWHM&-mdp{M@`GEZ!Guv5OU2hOznf7eY4Q}hPKp|^t=Bto}=~*IqW&} zlSer;smn(s+TXvgNH=WsUh~e!=IKhSE3I!7c6>E88?ZXY|I|Ef<0{X8qSCSlDH+k% z56_-u_}RGmMc=I&_4jOaxnc*l+tV-oxLh(NIX*@3fiykUfA~#8)v}FUkdU zvz;!Tr}r^o_U-wARQ~C~F?H26=O${Ps6Pf=@2vfMSl4((nYZVhV0fMwX_FyH^Wuv} zvNioGcKXDTGyCqIJ9NZS+=czp>JuD-#!@!yD=tdQ%daCFDzWZj9ap|5CUx!FtH2?> zB5s|4G9cJkFc+jNnS*Rck34b_UazDZ_!6bZ=X#SM&{2<3`hkC}B(~u&V&h=Ed3*}? zkuL$I!o{8VAm(5n-CVKr`q4f2o%(lzr+JA=vbLr4r=Sj3DBbDB7jHy$Cgj9!GlTG$ z-$LRV$^5?j&AXE1$7*@Hy<-*qdp7tTPnO7u844`vRfv`y4H~4k$Tgu^d2_<4>A`Oq z3Cg=W$kBLJ4w$M~{~flU4!1F<^Ly^Qj}%;S?}Be3Yiu^(Jbr!*P3rc!uNS3=9XfgO z@SKKunnlrw5TnaiUYJ8TVOZfT9797#Jo?!6KUJ~BFk^QD<|D3V!6RAsGhogINNAZ^ zSSH|WOH4LX>i>KID`j3jHo1M~L;uBv=i*yT9`=_@?j0F%`~C3 zb}u604an}ISJiF4_TuL9#Ui#4uSH0uE>jMv%2=z)zu@ z_FP?4_Z3)jvYSN2_R9vK8?xi`0(z{jU~|#$@PQ!W37`=0GPWailSNiLMK3d>zkCku z14w4Gfdk=(0lVRT;wCR86$1_J4LtXhu|IErCVZ=31LIn+u~wIOwQpV3%`N)7zs-f8 zipcZyHrSc{NiKTDPWBxMaZ{W9M?2*Dr(}EeMYhKWC~q05Q3!HXJ3cutki5^K-S%p-i;}Y6l9=Mo^!voM2!3Wk!Q7S) z;D;4ZDBzh;m4Ey7#BuCA7^)f?;SQn?C8rR!C{(+#2MLYw#~2pNEF3zBviBIQ{_rTM zphmGGdyX*Six%Juu!7<6E(KpiMf?!uxeB~NBl3fWBR>`EL5^I$%twWWmjxkp4uL&Uq1sWQKHB22_# z$rxMZHwPN8Tg^NXd?_9&O8iu)+3DCH+d}+u0Zdj&1?)aVp!JR!Lm%qr&aC8Q>;`kF zP;}2)fJ}a>h-l=k>N-^Tske}4YrbUMywz2C3#d_5m{l$6Q>R9s`ev)_zLZy1vGc`1a~ z!>L1GzLl-7HdJKgYZ+*3d1d@pHpkUVU)e;)m!g(X5Y>03k0&mxwFLAAq>_|?8Id~U ze$P^14`#+r#ixR8;Z?PEjq>XC;zr(0g`a)OlcG%f>*WjqqP*$c=UIv)?B7k@IHFFO zv1u3qe`H{I?Y54mCauatTz|P4HPh(Xd)i3Oe4=bTv$Az`Xny8X$(6b7`f6!6b*G0t z%;|S$L=Rfk^lshgbT_X$_E1TM8te1t4Qbnx{!D2R5fUkG%op9v)i*5C>2zlcwIGvw zNFgq+iGIF;YujmS|IFQu+ztMtdHn6`hfB86{Zxb{6r$ZKLbeoB_K5CIw>o<->Mie@MoJ~(0J9Z30!B6yx`AMpJ ztZFdHc+hmRtuAaCJKKm-G)bkKQC?+-iU4!)(~rjZfwW0Zu0yLdDf)c-ix;3m=sP&v zOnerOA_C2DHS$-yk(lsuRAk?|Q&YHQL_KRVkbAq|9;FYY)&i2{(`GR^U;;zdW53T- z$#OWLoP9X;?U5&Er1hTi4$4?eR#+|%syuNCG$=J!cGDDdYX8&$78kd=$TRfG*~#;X zDkm$)@|%2`PunO|%~r|nqexwQBE>Fk&*^o1MqQApk%79X`q9TYmG9E(oI=-x%4U)r z7C0F2xZ9wFXb~xdzL~YlpI^`eL(S}^{cdUl9R*{+(93I7R>BJP<>21;<^GGr+ay z3+n=gTx{}~&uU^2KZp7|#7g()Dc)E`jp=<&3E0S8(PS~%$EMrBvtznmTF}vf786C4 zYIfVn$$&JrLq?7nNC-o8!om>n{W70!8oK)gMjk8lk@wW3mE zKMxYT2;>Y5#5$YUGNUAV4|Z&%PDoaJShKErgbE$RUZ#^RzLnK@wx?c&%pD*cgEy}# z86RO91~ZSna80&H2aXtoX!EnUmGm+)U_=Derys`a_(>P4g~$y&(4$-8-e$P%*~^!8 zn7yrsfPk>F*i}!;q2PX?`rMCfk%=LWYPdIj_2}f+%+ugESL?)P9C`MDsrAcnhhtfH zbh<)R!{`Q@v7YeJgWA0Oq)qA}k>hOLV3fR4#<*h2WB4zYuVz2!nI@ zQ=c~EQ?BcM$?wjZIe(FtrB3=oag8Iw$3GafSotwa`Rq!-({#D?r>?tfCayg)~_rz~mcgu&nW z)~iX(^7`R)2gDe|8yow~?TnxMY0KhjsokEJ1k0~-FKancdh_0+KE+kf+#`x^+WO4~ zZ=auRag0~yUtRo4Kk$PT>)y74zjr)wT@>rmxm(|2E6u@5XK&qCo*v(!eDNi_3kCeT zKbW-Pg#WCauy8OSPqh=C%wbl3?dWtSl%}XyWGKg47pa{s*3tOc_hr1AIEB8!O2^!D z1v{fpd!2i!yZ=K)QUz@@KaB9j@Q+ye_ybazLz*$pJ_jeoMBYX*I*v*H4o0;ZH9Yt7 zE75tzi;GysYUa8-y||XCa0;<9^=WT)=@jmIYZUa}`a|QYwl{kkvz_nl-mRZ?BbQNy z74eZ%ZVvxHX4Ky1k2#Vm1})bY&o+p4F92srIa87;Lo0k@z?CNNl&rn!&VLR^A|=X1 z*W>!8V_8o;+~?o+8pnpNTh*(rlH91k6xwS`HO+Sy3y6B@Vb`eh)Yq-nyHr`2IrdDB zl37z{)q;>u&6A?7VKH2dXY_~8$1C6I9AvJp@h-Iq*(~3sX*4Y=Z>RomGH!RO-yX2v zqH`CMrC5r;cpcBWw=ITk&U>9$wba+TTG`&jDY7kf#2U)+Y_0a-+FR&Z&l$pFm^2{w z*M;8kX?8USE&<^cN=~W_4D1<13qUb8|BxT6?moDsg~X_|E6TXqZs^yS7=m zIm7=YJJ@zolU1V&_oxmOh0gho-70V`@6ZWCFXw{b62_&=fXSVnh3eR`AL$#T;{Ffrl6)w+%UNmufnr!rF$ zociu*Qp$~EtW(qJOH-Ma6u$~@9C<%{+4gWWT_-J3maD2YE9}3&CIyUJElo#8mLT<_b@Kc&Mvbmou}kX1}XvIXzX_r(LT( z^2p(rVj^M+25fC3j|Y5gOPXS*+<+$bs9(WDyY**jbVEZ8ZGXp+t75cY`L*N0^d9Q{ z0)YmDXxiM8U1CLn+Fjh|C&u@Ap1b?T?qUh~Bu47vf|1xw)X&)+?9_054hQOrJ%jXf z5h_utr?zoA*p!^E-C!glB%Lv|F5`O7|0-9j+|z_h52ghwZB62ySzS<&drsMS@p_4L z)3t^+6~~k*zK6~;sj#o|001gon3l9wf)W0*d^f| zZJgJijkP%_X&Y`}7pAH-yh|Tcux&lTWk9V9HT?U}>KIHQv9+kSuCvmzjZ4}usa?D)>Y5idGOaAp-H98fz)_|SrUu1ZUSHrMc`sm?Krvm0T z9xV6yxE`D{Gf9tK!R(f@GS0bA)?$+Ryq|#HEvn#V`LotJV!+r6jdT64X2N^qqHu$* zt{_e#k2@D48ck6yVXjrS2Tfe4j7(i_8!Y2sEb+1S#38Sj^`-wit{wGKOgKfBc?TNU zH4ScP{9Mqr`@634$S>))njf&rcU2iz6c7l7Vn%(uZaX?3(515@M447-6skOz@!lD_nx~K3dNK-x` zu0H)Nj`;gl|Ge8H(zOr(^AZRrqK8e8f|9hy`yAXjo6+BKM}nUZT_CYjM>BNY$0t6p zMe6;2a*EKn3GRph84(_VK%y914gFI+;)(Ij<3sS$u!cz*36U7IGffCRn?N84n1NUk zRDMg!BQ8V?IQugbRnM(Cpk?VI=F4 zLGSgEWqgCys}KSPGSLAY1Yz9jgBF7XUI@nLkZ#4njf6l#Okvdz--ReD%^Lj+%#__j zhlBXVh5dNMrxw5Vzke>=s(WXBq4UaJO3&9kic}7r?|I;)w?5ir=IK8fn#~L+!AH~8 z)3bp*7}>PQ_zZ;27}zj!a1ds|-n2Ql#2iB<_=z<9R(|}Xl@dp#E#d|01W_uPr~0{*bDZH^Z2k7FsbP)UoMvOs z_>02#yKPf?nmH(sQ^)>DoD=EhfMa+;L&V))h6 z7H%OVDB>XxHIg(_D4JS8i58#P{^klt9t*QG6~d6A%PmMSzzc{-bRmHuHjyzR-6C+4h?$+t$!BlgM7nnV zv3QYCX80U+#0OU(&u+{k+U?k}wy9gIi(G9FCVRGrB88Y6k~2%LMVKdTc=v?ox9^2C z#Zx<=v)UAJ)V4s~&Dpv;W4!f2_i>B(nj8MXecg;bb@DEQ_%9}=M}SKhjQ-aPe} zy}i(nEcuFrTc!!%ooK)7=}4l><-xUUrd4b_6DUKZks~7`I0XZPYrzl0)ATH5zpA<9 z#!D)d*Wp0Mw{4>BBk#m#0S%Q`6v_^xr|%`3Klxq?%;Dvx1PrgORo=Bef@z&H>y@R5 zBnj{rtA|&!{Hyr$=iBb4Xbd&1;1v-GhsNVXR2;X8!ulQz`V{*O1uDWB4Q))HsMgCW z#hn2vrF(2I3PbPlp>XkwwC4xSKg8h>F~2E{rwpR5YA+v*4dXamFeM$k0l2G!pUDjVU z|2+?IMalI&0Azz#Q4mG!&A}=zG@Hd(x&S-VV(y+UQp>XFBO;UhEh~qI2j_8fQu}^v zsWK(r<0)TqQXWPyxrw(kor3dR5;0>L1e#11%-AECi^9Ae7OTLer_c+eK#)r(7&nq| z5O{4#x7&|=iBMuOigK_rPzj2a6$@WiF91)r9-k-A0HMT@t`d*Hkz-ag{9^BIt2uEf zp)K#M@fvyXDiL`yP|2K!b&30D6K3*bexSBxmY*HG~C#X^*$ z2mlfzJ~WAHbGU^ok?IumrG~gcksex<1;ZuiHX7iUF-?CF^K(SXWsdmqafcLHEFr$6 z!EDq4lRmJ26y9&)Ti#9h%(#6=p@)%$$&)2$w!fF=7fMTSD7Qy2sa}NYB6^eh6#}0Q zQWHU`lmbdeMIC%V2_-!s#?j91E6R$6ul!J504(DSrxw}c?3PC`2koVBCh+Z2{g)Zo z!ty(CV3OgS7L@V+9f`;cJIPolWa#}02NHJFO?fABvT)e^j_)|^@-HaUHvspOG{-4iB)~EEF z?vk)WkNdy>4G^jZ&_-7(c((U-gCl_8(0aTkAKhVd3Dn zxxM{deSDT?^9GH802P4Z*Fnhj5Mq7g>;*_;xwuDmYKL8aRdnF%+)xpHmO7vp7E!cMPaM9-I@4ONp%83?n5T`f=%H&YXa0RLoVQCh-paw*bwhfWri z@_m9eT3coQ*v=PH{QRLBC%oc&HgI1p5{jlNn(vEmJ|U|!Hg&Nj&zzY#_{VW!qXdv0 zz9})P;|RjmuP=mBs7<_`>sZv+VBKzx5UW^PeqICb-AzTRuRpov_wEr04k`C}ZON6r zDuLbU#+GYyH#+A{H)gzTGdOxrS$BNUY3Igd`)BV)s^*5y8rnZBsz|hwq!{TW9@kh1 zI=zRwChHF?s`H$N>t=@VkBhUm2XDHyqHjX$?A_M=$2mC zRnkQ~N>xHUcVHn`T~mtn#+{bNjY{v@}aq4|ZSB#+G&#VV)S9wI_D&F1KxH{b1jE)7j~K?3e%)l?}sZT{{8b zk|t9&G(mLG!onEI&~daHgdy>lFUL5q8hM)$W*z?ybjnmS;f61RvHChP0SB0^8wamR z=Lq^4zMY0x$2;c9qCQm#hFKoAyiPkE%N6%@-ggaRN99Z8+v$7OiUFGxN`l@h91#5H zQ$x)$d-%x%%PiZy^ZqVVO~Iwv0koLREr%6GSN7---4M`-V&a-(*3kA`zi~qv{#N*c z1n!14P~gAX9s|ryHYhe=i-M42*z;3LE_C2xn>W+0MR`|#xXoVRW+AViBEC!Hn425H z?3jC;^=wO#KXwbqC2`BViItS7OOi&PJS);U9KXic%icd95qR23GgR^v#L?d%WkOke zu?CPN5RuwLU7;8IE8A4kRy|tDK;mnJmJMj>s) z(=e#ti!v5;m0<*iM6j89KcZ-|Qt9B+cxQvvd!|}^FD(C+i+ZJ=Z2uMRN2Ck1XX>nVOl^BjESLUu_GVDExd-3c zJy6*cQB$)#H(U)}E4p9TS)YZ1Ef-H5)L4QD#2C#sJ?+&PUwUkOB0e$Zmu&`F<-FlW(|O(&8lavT^Xs_BE*gF0ro)|23< z6cA~&EjbX-$JCN#y%0mnL}ZKMjv8D%MM$!@1fyF?;Cb^OtLCFfTMI#s$-aH-qIx_V zw_Zp}qIdSpCqriIIgoS!OF)6yY}iJ(tH?s(*PaNwvV5d~kV`Tk7skZZda8#Xnvt+c zazxsMdh*rG-4D=ugS?)+jztClXTk9i57D(Ycmsh$_n$XrU2zstuny;eAVf;$h*+o3 zME$sB>#xdS;uv%Ds>5%Kdlh44+20Oho*)$hCc?5SujNlz4rzQJCU*Lev@Lr#)3cz4 z>4ohJv!nNmi;*!eC&M&2@yj0pHD>72IYmwxFMJc^uQ893J)bB)iXG725x;DNnA#PBPH25B+sTMCOb%S zM-;D^Fn)*d%o=Nrxle~ZRAuq$=}eDM*_~`K4t@gC+hedNtAUrqSO;@()h4P3JDRJ6 zG6t_=0m2pQ0V@?6yP5${>x1i&FWfON#Z^nr2f>(RXK0(Bv1Z3%o=-@4OnRCszHStM zIK#<8aZLom(-(f6#d;-2409S_gtNNl`Xw{~6DgFCH{9!5)`i+Ux?_LvA<|yb7Vn1# z@ePz60MBs;{)=JHp+kDO5x{Rf_(>kC9u20OOO9YB6P)1_QR=8+GkoeQW$P>|9ljP- zRzsL=OSmii(a5ved9h^@({>DG??<9kz;EbC@B|DlT_p)HSf)nGiPU|8uAAiOm~X)Q z*=PjtjbR^x8h!F*VhOkHWfdAZ6R;-j2Z}MKSsL8VQo`UxxQ_upa%M^RuMoZy`SAuz zKR*1nqJ4$~X8C46`&LD*%Wn~+dSvJ2i3kYLR)#Qu^vesYnNjVr49I4SJGp1L>a&oG z0)uCoLRZ6ywCz+tV@gkFHDzl`BR(}f@mbU!QEkz+A3U!le;yjc&IWtdq-dU8+rx(r zp(|)AGrh;k${2%CgPCpLzYW2kr-e*sI8wZk&-0K)LDMiK=qqkhvT|X(Uj*6%I3FjF z2RLLpDk&vB>)`ng<51_~7^=gWZ_chA}Wm2u~3 zBAd2D%WQ=BERPOPVrW30ITFLytOs-G)SK3!Nf{7Bc_`2WBz*xDYFG%a6G6z!>T#2T zZCT~lW$}Z94yWv{ALuOd z#|Y01(RVxMZ4S!pE=NHO-x8@bLKsqofL9AUJkW7vA=ci~$gi$$)j&Y=e`21Yd&Ts( z9OU7*Zr?5+yR)v$V#4Kh{*IwB9;D>sm^MJ#ilf>MPEJ_v@X0B!XVB$V&1{8N z@ZK|;8D{jpem^V)&z$5G z?Q^sh2&$?){TPQ?7l1;bH7LJ5H;SL9ZKtz`J(KL@aY2U^7=WIa