Merge remote-tracking branch 'upstream/master' into dev-via

README.rst

@@ -44,6 +44,7 @@ For elliptic curve operations, `libsecp256k1`_ is a required dependency::
 Alternatively, when running from a cloned repository, a script is provided to build
 libsecp256k1 yourself::
 
+    sudo apt-get install automake libtool
     ./contrib/make_libsecp256k1.sh
 
 Due to the need for fast symmetric ciphers, either one of `pycryptodomex`_

contrib/build-wine/deterministic.spec

@@ -35,6 +35,7 @@ binaries += [('C:/tmp/libusb-1.0.dll', '.')]
 
 datas = [
     (home+'electrum_ltc/*.json', 'electrum_ltc'),
+    (home+'electrum_ltc/lnwire/*.csv', 'electrum_ltc/lnwire'),
     (home+'electrum_ltc/wordlist/english.txt', 'electrum_ltc/wordlist'),
     (home+'electrum_ltc/locale', 'electrum_ltc/locale'),
     (home+'electrum_ltc/plugins', 'electrum_ltc/plugins'),

contrib/make_libsecp256k1.sh

@@ -1,5 +1,15 @@
 #!/bin/bash
 
+# This script was tested on Linux and MacOS hosts, where it can be used
+# to build native libsecp256k1 binaries.
+#
+# It can also be used to cross-compile to Windows:
+# $ sudo apt-get install mingw-w64
+# For a Windows x86 (32-bit) target, run:
+# $ GCC_TRIPLET_HOST="i686-w64-mingw32" ./contrib/make_libsecp256k1.sh
+# Or for a Windows x86_64 (64-bit) target, run:
+# $ GCC_TRIPLET_HOST="x86_64-w64-mingw32" ./contrib/make_libsecp256k1.sh
+
 LIBSECP_VERSION="b408c6a8b287003d1ade5709e6f7bc3c7f1d5be7"
 
 set -e

contrib/osx/osx.spec

@@ -69,6 +69,7 @@ hiddenimports += ['_scrypt', 'PyQt5.QtPrintSupport']  # needed by Revealer
 
 datas = [
     (electrum + PYPKG + '/*.json', PYPKG),
+    (electrum + PYPKG + '/lnwire/*.csv', PYPKG + '/lnwire'),
     (electrum + PYPKG + '/wordlist/english.txt', PYPKG + '/wordlist'),
     (electrum + PYPKG + '/locale', PYPKG + '/locale'),
     (electrum + PYPKG + '/plugins', PYPKG + '/plugins'),

electrum_ltc/address_synchronizer.py

@@ -449,7 +449,7 @@ def get_history(self, *, domain=None) -> Sequence[HistoryItem]:
         domain = set(domain)
         # 1. Get the history of each address in the domain, maintain the
         #    delta of a tx as the sum of its deltas on domain addresses
-        tx_deltas = defaultdict(int)
+        tx_deltas = defaultdict(int)  # type: Dict[str, Optional[int]]
         for addr in domain:
             h = self.get_address_history(addr)
             for tx_hash, height in h:

electrum_ltc/base_wizard.py

@@ -40,7 +40,7 @@
 from .storage import WalletStorage, StorageEncryptionVersion
 from .wallet_db import WalletDB
 from .i18n import _
-from .util import UserCancelled, InvalidPassword, WalletFileException
+from .util import UserCancelled, InvalidPassword, WalletFileException, UserFacingException
 from .simple_config import SimpleConfig
 from .plugin import Plugins, HardwarePluginLibraryUnavailable
 from .logging import Logger
@@ -113,18 +113,21 @@ def run(self, *args, **kwargs):
     def can_go_back(self):
         return len(self._stack) > 1
 
-    def go_back(self):
+    def go_back(self, *, rerun_previous: bool = True) -> None:
         if not self.can_go_back():
             return
         # pop 'current' frame
         self._stack.pop()
-        # pop 'previous' frame
-        stack_item = self._stack.pop()
+        prev_frame = self._stack[-1]
         # try to undo side effects since we last entered 'previous' frame
-        # FIXME only self.storage is properly restored
-        self.data = copy.deepcopy(stack_item.db_data)
-        # rerun 'previous' frame
-        self.run(stack_item.action, *stack_item.args, **stack_item.kwargs)
+        # FIXME only self.data is properly restored
+        self.data = copy.deepcopy(prev_frame.db_data)
+
+        if rerun_previous:
+            # pop 'previous' frame
+            self._stack.pop()
+            # rerun 'previous' frame
+            self.run(prev_frame.action, *prev_frame.args, **prev_frame.kwargs)
 
     def reset_stack(self):
         self._stack = []
@@ -144,7 +147,7 @@ def new(self):
         self.choice_dialog(title=title, message=message, choices=choices, run_next=self.on_wallet_type)
 
     def upgrade_db(self, storage, db):
-        exc = None
+        exc = None  # type: Optional[Exception]
         def on_finished():
             if exc is None:
                 self.terminate(storage=storage, db=db)
@@ -158,6 +161,13 @@ def do_upgrade():
                 exc = e
         self.waiting_dialog(do_upgrade, _('Upgrading wallet format...'), on_finished=on_finished)
 
+    def run_task_without_blocking_gui(self, task, *, msg: str = None) -> Any:
+        """Perform a task in a thread without blocking the GUI.
+        Returns the result of 'task', or raises the same exception.
+        This method blocks until 'task' is finished.
+        """
+        raise NotImplementedError()
+
     def load_2fa(self):
         self.data['wallet_type'] = '2fa'
         self.data['use_trustedcoin'] = True
@@ -332,7 +342,7 @@ def failed_getting_device_infos(name, e):
         self.choice_dialog(title=title, message=msg, choices=choices,
                            run_next=lambda *args: self.on_device(*args, purpose=purpose, storage=storage))
 
-    def on_device(self, name, device_info, *, purpose, storage=None):
+    def on_device(self, name, device_info: 'DeviceInfo', *, purpose, storage=None):
         self.plugin = self.plugins.get_plugin(name)
         assert isinstance(self.plugin, HW_PluginBase)
         devmgr = self.plugins.device_manager
@@ -356,6 +366,10 @@ def on_device(self, name, device_info, *, purpose, storage=None):
         except (UserCancelled, GoBack):
             self.choose_hw_device(purpose, storage=storage)
             return
+        except UserFacingException as e:
+            self.show_error(str(e))
+            self.choose_hw_device(purpose, storage=storage)
+            return
         except BaseException as e:
             self.logger.exception('')
             self.show_error(str(e))
@@ -414,14 +428,16 @@ def derivation_and_script_type_dialog(self, f):
                 self.show_error(e)
                 # let the user choose again
 
-    def on_hw_derivation(self, name, device_info, derivation, xtype):
+    def on_hw_derivation(self, name, device_info: 'DeviceInfo', derivation, xtype):
         from .keystore import hardware_keystore
         devmgr = self.plugins.device_manager
+        assert isinstance(self.plugin, HW_PluginBase)
         try:
             xpub = self.plugin.get_xpub(device_info.device.id_, derivation, xtype, self)
             client = devmgr.client_by_id(device_info.device.id_)
             if not client: raise Exception("failed to find client for device id")
             root_fingerprint = client.request_root_fingerprint_from_device()
+            label = client.label()  # use this as device_info.label might be outdated!
         except ScriptTypeNotSupported:
             raise  # this is handled in derivation_dialog
         except BaseException as e:
@@ -434,7 +450,7 @@ def on_hw_derivation(self, name, device_info, derivation, xtype):
             'derivation': derivation,
             'root_fingerprint': root_fingerprint,
             'xpub': xpub,
-            'label': device_info.label,
+            'label': label,
         }
         k = hardware_keystore(d)
         self.on_keystore(k)
@@ -535,6 +551,7 @@ def create_wallet(self):
         if self.wallet_type == 'standard' and isinstance(self.keystores[0], Hardware_KeyStore):
             # offer encrypting with a pw derived from the hw device
             k = self.keystores[0]  # type: Hardware_KeyStore
+            assert isinstance(self.plugin, HW_PluginBase)
             try:
                 k.handler = self.plugin.create_handler(self)
                 password = k.get_password_for_storage_encryption()

electrum_ltc/bitcoin.py

@@ -565,8 +565,8 @@ def is_segwit_script_type(txin_type: str) -> bool:
     return txin_type in ('p2wpkh', 'p2wpkh-p2sh', 'p2wsh', 'p2wsh-p2sh')
 
 
-def serialize_privkey(secret: bytes, compressed: bool, txin_type: str,
-                      internal_use: bool=False) -> str:
+def serialize_privkey(secret: bytes, compressed: bool, txin_type: str, *,
+                      internal_use: bool = False) -> str:
     # we only export secrets inside curve range
     secret = ecc.ECPrivkey.normalize_secret_bytes(secret)
     if internal_use:

electrum_ltc/channel_db.py

@@ -38,7 +38,8 @@
 from . import constants
 from .util import bh2u, profiler, get_headers_dir, bfh, is_ip_address, list_enabled_bits
 from .logging import Logger
-from .lnutil import LN_GLOBAL_FEATURES_KNOWN_SET, LNPeerAddr, format_short_channel_id, ShortChannelID
+from .lnutil import (LNPeerAddr, format_short_channel_id, ShortChannelID,
+                     validate_features, IncompatibleOrInsaneFeatures)
 from .lnverifier import LNChannelVerifier, verify_sig_for_channel_update
 from .lnmsg import decode_msg
 
@@ -47,15 +48,6 @@
     from .lnchannel import Channel
 
 
-class UnknownEvenFeatureBits(Exception): pass
-
-def validate_features(features : int):
-    enabled_features = list_enabled_bits(features)
-    for fbit in enabled_features:
-        if (1 << fbit) not in LN_GLOBAL_FEATURES_KNOWN_SET and fbit % 2 == 0:
-            raise UnknownEvenFeatureBits()
-
-
 FLAG_DISABLE   = 1 << 1
 FLAG_DIRECTION = 1 << 0
 
@@ -102,14 +94,14 @@ class Policy(NamedTuple):
     def from_msg(payload: dict) -> 'Policy':
         return Policy(
             key                         = payload['short_channel_id'] + payload['start_node'],
-            cltv_expiry_delta           = int.from_bytes(payload['cltv_expiry_delta'], "big"),
-            htlc_minimum_msat           = int.from_bytes(payload['htlc_minimum_msat'], "big"),
-            htlc_maximum_msat           = int.from_bytes(payload['htlc_maximum_msat'], "big") if 'htlc_maximum_msat' in payload else None,
-            fee_base_msat               = int.from_bytes(payload['fee_base_msat'], "big"),
-            fee_proportional_millionths = int.from_bytes(payload['fee_proportional_millionths'], "big"),
+            cltv_expiry_delta           = payload['cltv_expiry_delta'],
+            htlc_minimum_msat           = payload['htlc_minimum_msat'],
+            htlc_maximum_msat           = payload.get('htlc_maximum_msat', None),
+            fee_base_msat               = payload['fee_base_msat'],
+            fee_proportional_millionths = payload['fee_proportional_millionths'],
             message_flags               = int.from_bytes(payload['message_flags'], "big"),
             channel_flags               = int.from_bytes(payload['channel_flags'], "big"),
-            timestamp                   = int.from_bytes(payload['timestamp'], "big")
+            timestamp                   = payload['timestamp'],
         )
 
     @staticmethod
@@ -154,7 +146,7 @@ def from_msg(payload) -> Tuple['NodeInfo', Sequence['LNPeerAddr']]:
             alias = alias.decode('utf8')
         except:
             alias = ''
-        timestamp = int.from_bytes(payload['timestamp'], "big")
+        timestamp = payload['timestamp']
         node_info = NodeInfo(node_id=node_id, features=features, timestamp=timestamp, alias=alias)
         return node_info, peer_addrs
 
@@ -321,11 +313,12 @@ def get_recent_peers(self):
             return ret
 
     # note: currently channel announcements are trusted by default (trusted=True);
-    #       they are not verified. Verifying them would make the gossip sync
+    #       they are not SPV-verified. Verifying them would make the gossip sync
     #       even slower; especially as servers will start throttling us.
     #       It would probably put significant strain on servers if all clients
     #       verified the complete gossip.
     def add_channel_announcement(self, msg_payloads, *, trusted=True):
+        # note: signatures have already been verified.
         if type(msg_payloads) is dict:
             msg_payloads = [msg_payloads]
         added = 0
@@ -338,8 +331,8 @@ def add_channel_announcement(self, msg_payloads, *, trusted=True):
                 continue
             try:
                 channel_info = ChannelInfo.from_msg(msg)
-            except UnknownEvenFeatureBits:
-                self.logger.info("unknown feature bits")
+            except IncompatibleOrInsaneFeatures as e:
+                self.logger.info(f"unknown or insane feature bits: {e!r}")
                 continue
             if trusted:
                 added += 1
@@ -353,7 +346,7 @@ def add_channel_announcement(self, msg_payloads, *, trusted=True):
     def add_verified_channel_info(self, msg: dict, *, capacity_sat: int = None) -> None:
         try:
             channel_info = ChannelInfo.from_msg(msg)
-        except UnknownEvenFeatureBits:
+        except IncompatibleOrInsaneFeatures:
             return
         channel_info = channel_info._replace(capacity_sat=capacity_sat)
         with self.lock:
@@ -392,7 +385,7 @@ def add_channel_updates(self, payloads, max_age=None, verify=True) -> Categorize
         now = int(time.time())
         for payload in payloads:
             short_channel_id = ShortChannelID(payload['short_channel_id'])
-            timestamp = int.from_bytes(payload['timestamp'], "big")
+            timestamp = payload['timestamp']
             if max_age and now - timestamp > max_age:
                 expired.append(payload)
                 continue
@@ -407,7 +400,7 @@ def add_channel_updates(self, payloads, max_age=None, verify=True) -> Categorize
             known.append(payload)
         # compare updates to existing database entries
         for payload in known:
-            timestamp = int.from_bytes(payload['timestamp'], "big")
+            timestamp = payload['timestamp']
             start_node = payload['start_node']
             short_channel_id = ShortChannelID(payload['short_channel_id'])
             key = (start_node, short_channel_id)
@@ -499,13 +492,14 @@ def verify_channel_update(self, payload):
             raise Exception(f'failed verifying channel update for {short_channel_id}')
 
     def add_node_announcement(self, msg_payloads):
+        # note: signatures have already been verified.
         if type(msg_payloads) is dict:
             msg_payloads = [msg_payloads]
         new_nodes = {}
         for msg_payload in msg_payloads:
             try:
                 node_info, node_addresses = NodeInfo.from_msg(msg_payload)
-            except UnknownEvenFeatureBits:
+            except IncompatibleOrInsaneFeatures:
                 continue
             node_id = node_info.node_id
             # Ignore node if it has no associated channel (DoS protection)
@@ -599,11 +593,17 @@ def newest_ts_for_node_id(node_id):
         self._recent_peers = sorted_node_ids[:self.NUM_MAX_RECENT_PEERS]
         c.execute("""SELECT * FROM channel_info""")
         for short_channel_id, msg in c:
-            ci = ChannelInfo.from_raw_msg(msg)
+            try:
+                ci = ChannelInfo.from_raw_msg(msg)
+            except IncompatibleOrInsaneFeatures:
+                continue
             self._channels[ShortChannelID.normalize(short_channel_id)] = ci
         c.execute("""SELECT * FROM node_info""")
         for node_id, msg in c:
-            node_info, node_addresses = NodeInfo.from_raw_msg(msg)
+            try:
+                node_info, node_addresses = NodeInfo.from_raw_msg(msg)
+            except IncompatibleOrInsaneFeatures:
+                continue
             # don't load node_addresses because they dont have timestamps
             self._nodes[node_id] = node_info
         c.execute("""SELECT * FROM policy""")
@@ -671,7 +671,7 @@ def get_policy_for_node(self, short_channel_id: bytes, node_id: bytes, *,
                 return
             now = int(time.time())
             remote_update_decoded = decode_msg(remote_update_raw)[1]
-            remote_update_decoded['timestamp'] = now.to_bytes(4, byteorder="big")
+            remote_update_decoded['timestamp'] = now
             remote_update_decoded['start_node'] = node_id
             return Policy.from_msg(remote_update_decoded)
         elif node_id == chan.get_local_pubkey():  # outgoing direction (from us)

electrum_ltc/commands.py

@@ -414,6 +414,13 @@ async def getprivatekeys(self, address, password=None, wallet: Abstract_Wallet =
         domain = address
         return [wallet.export_private_key(address, password) for address in domain]
 
+    @command('wp')
+    async def getprivatekeyforpath(self, path, password=None, wallet: Abstract_Wallet = None):
+        """Get private key corresponding to derivation path (address index).
+        'path' can be either a str such as "m/0/50", or a list of ints such as [0, 50].
+        """
+        return wallet.export_private_key_for_path(path, password)
+
     @command('w')
     async def ismine(self, address, wallet: Abstract_Wallet = None):
         """Check if address is in wallet. Return true if and only address is in wallet"""
@@ -467,7 +474,7 @@ async def getmerkle(self, txid, height):
 
     @command('n')
     async def getservers(self):
-        """Return the list of available servers"""
+        """Return the list of known servers (candidates for connecting)."""
         return self.network.get_servers()
 
     @command('')
@@ -1004,8 +1011,8 @@ async def list_channels(self, wallet: Abstract_Wallet = None):
                 'remote_balance': chan.balance(REMOTE)//1000,
                 'local_reserve': chan.config[LOCAL].reserve_sat,
                 'remote_reserve': chan.config[REMOTE].reserve_sat,
-                'local_unsettled_sent': chan.unsettled_sent_balance(LOCAL)//1000,
-                'remote_unsettled_sent': chan.unsettled_sent_balance(REMOTE)//1000,
+                'local_unsettled_sent': chan.balance_tied_up_in_htlcs_by_direction(LOCAL, direction=SENT) // 1000,
+                'remote_unsettled_sent': chan.balance_tied_up_in_htlcs_by_direction(REMOTE, direction=SENT) // 1000,
             } for channel_id, chan in l
         ]