Merge #334: Basic coin control.

bfdf0b2 expand non-empty tree sections in Coins tab (AdamISZ)
9295673 Basic coin control. (AdamISZ)

jmclient/jmclient/__init__.py

@@ -45,7 +45,7 @@
 from .wallet_utils import (
     wallet_tool_main, wallet_generate_recover_bip39, open_wallet,
     open_test_wallet_maybe, create_wallet, get_wallet_cls, get_wallet_path,
-    wallet_display)
+    wallet_display, get_utxos_enabled_disabled)
 from .maker import Maker, P2EPMaker
 from .yieldgenerator import YieldGenerator, YieldGeneratorBasic, ygmain
 # Set default logging handler to avoid "No handler found" warnings.

jmclient/jmclient/wallet.py

@@ -117,13 +117,19 @@ def wrapped(*args, **kwargs):
 
 class UTXOManager(object):
     STORAGE_KEY = b'utxo'
+    METADATA_KEY = b'meta'
     TXID_LEN = 32
 
     def __init__(self, storage, merge_func):
         self.storage = storage
         self.selector = merge_func
         # {mixdexpth: {(txid, index): (path, value)}}
         self._utxo = None
+        # metadata kept as a separate key in the database
+        # for backwards compat; value as dict for forward-compat.
+        # format is {(txid, index): value-dict} with "disabled"
+        # as the only currently used key in the dict.
+        self._utxo_meta = None
         self._load_storage()
         assert self._utxo is not None
 
@@ -135,6 +141,7 @@ def _load_storage(self):
         assert isinstance(self.storage.data[self.STORAGE_KEY], dict)
 
         self._utxo = collections.defaultdict(dict)
+        self._utxo_meta = collections.defaultdict(dict)
         for md, data in self.storage.data[self.STORAGE_KEY].items():
             md = int(md)
             md_data = self._utxo[md]
@@ -143,28 +150,46 @@ def _load_storage(self):
                 index = int(utxo[self.TXID_LEN:])
                 md_data[(txid, index)] = value
 
+        # Wallets may not have any metadata
+        if self.METADATA_KEY in self.storage.data:
+            for utxo, value in self.storage.data[self.METADATA_KEY].items():
+                txid = utxo[:self.TXID_LEN]
+                index = int(utxo[self.TXID_LEN:])
+                self._utxo_meta[(txid, index)] = value
+
     def save(self, write=True):
         new_data = {}
         self.storage.data[self.STORAGE_KEY] = new_data
+
         for md, data in self._utxo.items():
             md = _int_to_bytestr(md)
             new_data[md] = {}
             # storage keys must be bytes()
             for (txid, index), value in data.items():
                 new_data[md][txid + _int_to_bytestr(index)] = value
+
+        new_meta_data = {}
+        self.storage.data[self.METADATA_KEY] = new_meta_data
+        for (txid, index), value in self._utxo_meta.items():
+            new_meta_data[txid + _int_to_bytestr(index)] = value
+
         if write:
             self.storage.save()
 
     def reset(self):
         self._utxo = collections.defaultdict(dict)
 
-    def have_utxo(self, txid, index):
+    def have_utxo(self, txid, index, include_disabled=True):
+        if not include_disabled and self.is_disabled(txid, index):
+            return False
         for md in self._utxo:
             if (txid, index) in self._utxo[md]:
                 return md
         return False
 
     def remove_utxo(self, txid, index, mixdepth):
+        # currently does not remove metadata associated
+        # with this utxo
         assert isinstance(txid, bytes)
         assert len(txid) == self.TXID_LEN
         assert isinstance(index, numbers.Integral)
@@ -173,6 +198,8 @@ def remove_utxo(self, txid, index, mixdepth):
         return self._utxo[mixdepth].pop((txid, index))
 
     def add_utxo(self, txid, index, path, value, mixdepth):
+        # Assumed: that we add a utxo only if we want it enabled,
+        # so metadata is not currently added.
         assert isinstance(txid, bytes)
         assert len(txid) == self.TXID_LEN
         assert isinstance(index, numbers.Integral)
@@ -181,22 +208,59 @@ def add_utxo(self, txid, index, path, value, mixdepth):
 
         self._utxo[mixdepth][(txid, index)] = (path, value)
 
+    def is_disabled(self, txid, index):
+        if not self._utxo_meta:
+            return False
+        if (txid, index) not in self._utxo_meta:
+            return False
+        if b'disabled' not in self._utxo_meta[(txid, index)]:
+            return False
+        if not self._utxo_meta[(txid, index)][b'disabled']:
+            return False
+        return True
+
+    def disable_utxo(self, txid, index, disable=True):
+        assert isinstance(txid, bytes)
+        assert len(txid) == self.TXID_LEN
+        assert isinstance(index, numbers.Integral)
+
+        if b'disabled' not in self._utxo_meta[(txid, index)]:
+            self._utxo_meta[(txid, index)] = {}
+        self._utxo_meta[(txid, index)][b'disabled'] = disable
+
+    def enable_utxo(self, txid, index):
+        self.disable_utxo(txid, index, disable=False)
+
     def select_utxos(self, mixdepth, amount, utxo_filter=(), select_fn=None):
         assert isinstance(mixdepth, numbers.Integral)
         utxos = self._utxo[mixdepth]
+        # do not select anything in the filter
         available = [{'utxo': utxo, 'value': val}
             for utxo, (addr, val) in utxos.items() if utxo not in utxo_filter]
+        # do not select anything disabled
+        available = [u for u in available if not self.is_disabled(*u['utxo'])]
         selector = select_fn or self.selector
         selected = selector(available, amount)
         return {s['utxo']: {'path': utxos[s['utxo']][0],
                             'value': utxos[s['utxo']][1]}
                 for s in selected}
 
-    def get_balance_by_mixdepth(self, max_mixdepth=float('Inf')):
+    def get_balance_by_mixdepth(self, max_mixdepth=float('Inf'),
+                                include_disabled=True):
+        """ By default this returns a dict of aggregated bitcoin
+        balance per mixdepth: {0: N sats, 1: M sats, ...} for all
+        currently available mixdepths.
+        If max_mixdepth is set it will return balances only up
+        to that mixdepth.
+        To get only enabled balance, set include_disabled=False.
+        """
         balance_dict = collections.defaultdict(int)
         for mixdepth, utxomap in self._utxo.items():
             if mixdepth > max_mixdepth:
                 continue
+            if not include_disabled:
+                utxomap = {k: v for k, v in utxomap.items(
+                    ) if not self.is_disabled(*k)}
             value = sum(x[1] for x in utxomap.values())
             balance_dict[mixdepth] = value
         return balance_dict
@@ -285,7 +349,7 @@ def save(self):
         """
         Write data to associated storage object and trigger persistent update.
         """
-        self._storage.save()
+        self._utxos.save()
 
     @classmethod
     def initialize(cls, storage, network, max_mixdepth=2, timestamp=None,
@@ -610,17 +674,28 @@ def select_utxos_(self, mixdepth, amount, utxo_filter=None,
 
         return ret
 
+    def disable_utxo(self, txid, index, disable=True):
+        self._utxos.disable_utxo(txid, index, disable)
+        # make sure the utxo database is persisted
+        self.save()
+
+    def toggle_disable_utxo(self, txid, index):
+        is_disabled = self._utxos.is_disabled(txid, index)
+        self.disable_utxo(txid, index, disable= not is_disabled)
+
     def reset_utxos(self):
         self._utxos.reset()
 
-    def get_balance_by_mixdepth(self, verbose=True):
+    def get_balance_by_mixdepth(self, verbose=True,
+                                include_disabled=False):
         """
         Get available funds in each active mixdepth.
-
+        By default ignores disabled utxos in calculation.
         returns: {mixdepth: value}
         """
         # TODO: verbose
-        return self._utxos.get_balance_by_mixdepth(max_mixdepth=self.mixdepth)
+        return self._utxos.get_balance_by_mixdepth(max_mixdepth=self.mixdepth,
+                                            include_disabled=include_disabled)
 
     @deprecated
     def get_utxos_by_mixdepth(self, verbose=True):
@@ -636,7 +711,7 @@ def get_utxos_by_mixdepth(self, verbose=True):
                 utxos_conv[md][utxo_str] = data
         return utxos_conv
 
-    def get_utxos_by_mixdepth_(self):
+    def get_utxos_by_mixdepth_(self, include_disabled=False):
         """
         Get all UTXOs for active mixdepths.
 
@@ -651,6 +726,8 @@ def get_utxos_by_mixdepth_(self):
             if md > self.mixdepth:
                 continue
             for utxo, (path, value) in data.items():
+                if not include_disabled and self._utxos.is_disabled(*utxo):
+                    continue
                 script = self.get_script_path(path)
                 script_utxos[md][utxo] = {'script': script,
                                           'path': path,

jmclient/jmclient/wallet_utils.py

@@ -18,6 +18,7 @@
     LegacyWallet, SegwitWallet, is_native_segwit_mode)
 from jmbase.support import get_password, jmprint
 from .cryptoengine import TYPE_P2PKH, TYPE_P2SH_P2WPKH, TYPE_P2WPKH
+from .output import fmt_utxo
 import jmbitcoin as btc
 
 
@@ -40,7 +41,8 @@ def get_wallettool_parser():
         '(importprivkey) Adds privkeys to this wallet, privkeys are spaces or commas separated.\n'
         '(dumpprivkey) Export a single private key, specify an hd wallet path\n'
         '(signmessage) Sign a message with the private key from an address in \n'
-        'the wallet. Use with -H and specify an HD wallet path for the address.')
+        'the wallet. Use with -H and specify an HD wallet path for the address.\n'
+        '(freeze) Freeze or un-freeze a specific utxo. Specify mixdepth with -m.')
     parser = OptionParser(usage='usage: %prog [options] [wallet file] [method]',
                           description=description)
     parser.add_option('-p',
@@ -332,7 +334,8 @@ def get_imported_privkey_branch(wallet, m, showprivkey):
         addr = wallet.get_addr_path(path)
         script = wallet.get_script_path(path)
         balance = 0.0
-        for data in wallet.get_utxos_by_mixdepth_()[m].values():
+        for data in wallet.get_utxos_by_mixdepth_(
+                include_disabled=True)[m].values():
             if script == data['script']:
                 balance += data['value']
         used = ('used' if balance > 0.0 else 'empty')
@@ -407,7 +410,9 @@ def get_addr_status(addr_path, utxos, is_new, is_internal):
         return addr_balance, out_status
 
     acctlist = []
-    utxos = wallet.get_utxos_by_mixdepth_()
+    # TODO - either optionally not show disabled utxos, or
+    # mark them differently in display (labels; colors)
+    utxos = wallet.get_utxos_by_mixdepth_(include_disabled=True)
     for m in range(wallet.mixdepth + 1):
         branchlist = []
         for forchange in [0, 1]:
@@ -816,12 +821,15 @@ def f(r, deposits, deposit_times, now, final_balance):
             jmprint('scipy not installed, unable to predict accumulation rate')
             jmprint('to add it to this virtualenv, use `pip install scipy`')
 
-    total_wallet_balance = sum(wallet.get_balance_by_mixdepth().values())
+    # includes disabled utxos in accounting:
+    total_wallet_balance = sum(wallet.get_balance_by_mixdepth(
+        include_disabled=True).values())
     if balance != total_wallet_balance:
         jmprint(('BUG ERROR: wallet balance (%s) does not match balance from ' +
             'history (%s)') % (sat_to_str(total_wallet_balance),
                 sat_to_str(balance)))
-    wallet_utxo_count = sum(map(len, wallet.get_utxos_by_mixdepth_().values()))
+    wallet_utxo_count = sum(map(len, wallet.get_utxos_by_mixdepth_(
+        include_disabled=True).values()))
     if utxo_count != wallet_utxo_count:
         jmprint(('BUG ERROR: wallet utxo count (%d) does not match utxo count from ' +
             'history (%s)') % (wallet_utxo_count, utxo_count))
@@ -894,6 +902,104 @@ def wallet_signmessage(wallet, hdpath, message):
     retval = "Signature: {}\nTo verify this in Bitcoin Core".format(sig)
     return retval + " use the RPC command 'verifymessage'"
 
+def display_utxos_for_disable_choice_default(utxos_enabled, utxos_disabled):
+    """ CLI implementation of the callback required as described in
+    wallet_disableutxo
+    """
+
+    def default_user_choice(umax):
+        jmprint("Choose an index 0 .. {} to freeze/unfreeze or "
+                "-1 to just quit.".format(umax))
+        while True:
+            try:
+                ret = int(input())
+            except ValueError:
+                jmprint("Invalid choice, must be an integer.", "error")
+                continue
+            if not isinstance(ret, int) or ret < -1 or ret > umax:
+                jmprint("Invalid choice, must be between: -1 and {}, "
+                        "try again.".format(umax), "error")
+                continue
+            break
+        return ret
+
+    def output_utxos(utxos, status, start=0):
+        for (txid, idx), v in utxos.items():
+            value = v['value']
+            jmprint("{:4}: {:68}: {} sats, -- {}".format(
+                start, fmt_utxo((txid, idx)), value, status))
+            start += 1
+            yield txid, idx
+
+    ulist = list(output_utxos(utxos_disabled, 'FROZEN'))
+    disabled_max = len(ulist) - 1
+    ulist.extend(output_utxos(utxos_enabled, 'NOT FROZEN', start=len(ulist)))
+    max_id = len(ulist) - 1
+    chosen_idx = default_user_choice(max_id)
+    if chosen_idx == -1:
+        return None
+    # the return value 'disable' is the action we are going to take;
+    # so it should be true if the utxos is currently unfrozen/enabled.
+    disable = False if chosen_idx <= disabled_max else True
+    return ulist[chosen_idx], disable
+
+def get_utxos_enabled_disabled(wallet, md):
+    """ Returns dicts for enabled and disabled separately
+    """
+    utxos_enabled = wallet.get_utxos_by_mixdepth_()[md]
+    utxos_all = wallet.get_utxos_by_mixdepth_(include_disabled=True)[md]
+    utxos_disabled_keyset = set(utxos_all).difference(set(utxos_enabled))
+    utxos_disabled = {}
+    for u in utxos_disabled_keyset:
+        utxos_disabled[u] = utxos_all[u]
+    return utxos_enabled, utxos_disabled
+
+def wallet_freezeutxo(wallet, md, display_callback=None, info_callback=None):
+    """ Given a wallet and a mixdepth, display to the user
+    the set of available utxos, indexed by integer, and accept a choice
+    of index to "freeze", then commit this disabling to the wallet storage,
+    so that this disable-ment is persisted. Also allow unfreezing of a
+    chosen utxo which is currently frozen.
+    Callbacks for display and reporting can be specified in the keyword
+    arguments as explained below, otherwise default CLI is used.
+
+    ** display_callback signature:
+    args:
+    1. utxos_enabled ; dict of utxos as format in wallet.py.
+    2. utxos_disabled ; as above, for disabled
+    returns:
+    1.((txid(str), index(int)), disabled(bool)) of chosen utxo
+    for freezing/unfreezing, or None for no action/cancel.
+    ** info_callback signature:
+    args:
+    1. message (str)
+    2. type (str) ("info", "error" etc as per jmprint)
+    returns: None
+    """
+    if display_callback is None:
+        display_callback = display_utxos_for_disable_choice_default
+    if info_callback is None:
+        info_callback = jmprint
+    if md is None:
+        info_callback("Specify the mixdepth with the -m flag", "error")
+        return "Failed"
+    utxos_enabled, utxos_disabled = get_utxos_enabled_disabled(wallet, md)
+    if utxos_disabled == {} and utxos_enabled == {}:
+        info_callback("The mixdepth: " + str(md) + \
+            " contains no utxos to freeze/unfreeze.", "error")
+        return "Failed"
+    display_ret = display_callback(utxos_enabled, utxos_disabled)
+    if display_ret is None:
+        return "OK"
+    (txid, index), disable = display_ret
+    wallet.disable_utxo(txid, index, disable)
+    if disable:
+        info_callback("Utxo: {} is now frozen and unavailable for spending."
+                      .format(fmt_utxo((txid, index))))
+    else:
+        info_callback("Utxo: {} is now unfrozen and available for spending."
+                      .format(fmt_utxo((txid, index))))
+    return "Done"
 
 def get_wallet_type():
     if is_segwit_mode():
@@ -1047,7 +1153,7 @@ def wallet_tool_main(wallet_root_path):
 
     noseed_methods = ['generate', 'recover']
     methods = ['display', 'displayall', 'summary', 'showseed', 'importprivkey',
-               'history', 'showutxos']
+               'history', 'showutxos', 'freeze']
     methods.extend(noseed_methods)
     noscan_methods = ['showseed', 'importprivkey', 'dumpprivkey', 'signmessage']
     readonly_methods = ['display', 'displayall', 'summary', 'showseed',
@@ -1120,6 +1226,8 @@ def wallet_tool_main(wallet_root_path):
         return "Key import completed."
     elif method == "signmessage":
         return wallet_signmessage(wallet, options.hd_path, args[2])
+    elif method == "freeze":
+        return wallet_freezeutxo(wallet, options.mixdepth)
 
 
 #Testing (can port to test modules, TODO)