From 4cd50dd75a2613a3d631cefaf0bf88831786b6fd Mon Sep 17 00:00:00 2001 From: matejcik Date: Tue, 31 Mar 2020 12:22:59 +0200 Subject: [PATCH 001/117] trezor: bump lib version, implement new passphrase-on-device UI --- contrib/requirements/requirements-hw.txt | 2 +- electrum/plugins/trezor/__init__.py | 2 +- electrum/plugins/trezor/clientbase.py | 14 +++-- electrum/plugins/trezor/cmdline.py | 17 +++++- electrum/plugins/trezor/qt.py | 69 +++++++++++++++++++++++- electrum/plugins/trezor/trezor.py | 10 ++-- 6 files changed, 103 insertions(+), 11 deletions(-) diff --git a/contrib/requirements/requirements-hw.txt b/contrib/requirements/requirements-hw.txt index e4a9024c06..a442d71475 100644 --- a/contrib/requirements/requirements-hw.txt +++ b/contrib/requirements/requirements-hw.txt @@ -8,7 +8,7 @@ # see https://github.com/spesmilo/electrum/issues/5859 Cython>=0.27 -trezor[hidapi]>=0.11.5 +trezor[hidapi]>=0.12.0 safet>=0.1.5 keepkey>=6.3.1 btchip-python>=0.1.26 diff --git a/electrum/plugins/trezor/__init__.py b/electrum/plugins/trezor/__init__.py index 2d4267d785..6aff1e0f48 100644 --- a/electrum/plugins/trezor/__init__.py +++ b/electrum/plugins/trezor/__init__.py @@ -2,7 +2,7 @@ fullname = 'Trezor Wallet' description = _('Provides support for Trezor hardware wallet') -requires = [('trezorlib','github.com/trezor/python-trezor')] +requires = [('trezorlib','pypi.org/project/trezor/')] registers_keystore = ('hardware', 'trezor', _("Trezor wallet")) available_for = ['qt', 'cmdline'] diff --git a/electrum/plugins/trezor/clientbase.py b/electrum/plugins/trezor/clientbase.py index 99059f4427..66e32ad15a 100644 --- a/electrum/plugins/trezor/clientbase.py +++ b/electrum/plugins/trezor/clientbase.py @@ -9,7 +9,7 @@ from electrum.logging import Logger from electrum.plugins.hw_wallet.plugin import OutdatedHwFirmwareException, HardwareClientBase -from trezorlib.client import TrezorClient +from trezorlib.client import TrezorClient, PASSPHRASE_ON_DEVICE from trezorlib.exceptions import TrezorFailure, Cancelled, OutdatedFirmwareError from trezorlib.messages import WordRequestType, FailureType, RecoveryDeviceType, ButtonRequestType import trezorlib.btc @@ -30,8 +30,10 @@ _("Confirm the total amount spent and the transaction fee on your {} device"), ButtonRequestType.Address: _("Confirm wallet address on your {} device"), - ButtonRequestType.PassphraseType: + ButtonRequestType._Deprecated_ButtonRequest_PassphraseType: _("Choose on your {} device where to enter your passphrase"), + ButtonRequestType.PassphraseEntry: + _("Please enter your passphrase on the {} device"), 'default': _("Check your {} device to continue"), } @@ -259,7 +261,7 @@ def get_pin(self, code=None): raise Cancelled return pin - def get_passphrase(self): + def get_passphrase(self, available_on_device): if self.creating_wallet: msg = _("Enter a passphrase to generate this wallet. Each time " "you use this wallet your {} will prompt you for the " @@ -267,7 +269,11 @@ def get_passphrase(self): "access the bitcoins in the wallet.").format(self.device) else: msg = _("Enter the passphrase to unlock this wallet:") - passphrase = self.handler.get_passphrase(msg, self.creating_wallet) + + self.handler.passphrase_on_device = available_on_device + passphrase = self.handler.trezor_get_passphrase(msg, self.creating_wallet) + if passphrase is PASSPHRASE_ON_DEVICE: + return passphrase if passphrase is None: raise Cancelled passphrase = bip39_normalize_passphrase(passphrase) diff --git a/electrum/plugins/trezor/cmdline.py b/electrum/plugins/trezor/cmdline.py index e435aad138..5a7ee15963 100644 --- a/electrum/plugins/trezor/cmdline.py +++ b/electrum/plugins/trezor/cmdline.py @@ -1,7 +1,22 @@ from electrum.plugin import hook -from .trezor import TrezorPlugin +from electrum.i18n import _ +from electrum.util import print_stderr +from .trezor import TrezorPlugin, PASSPHRASE_ON_DEVICE from ..hw_wallet import CmdLineHandler +class TrezorCmdLineHandler(CmdLineHandler): + def __init__(self): + self.passphrase_on_device = False + super().__init__() + + def get_passphrase(self, msg, confirm): + import getpass + print_stderr(msg) + if self.passphrase_on_device and self.yes_no_question(_('Enter passphrase on device?')): + return PASSPHRASE_ON_DEVICE + else: + return getpass.getpass('') + class Plugin(TrezorPlugin): handler = CmdLineHandler() @hook diff --git a/electrum/plugins/trezor/qt.py b/electrum/plugins/trezor/qt.py index 3a8b2cba31..2438f27fe0 100644 --- a/electrum/plugins/trezor/qt.py +++ b/electrum/plugins/trezor/qt.py @@ -16,7 +16,7 @@ from ..hw_wallet.qt import QtHandlerBase, QtPluginBase from ..hw_wallet.plugin import only_hook_if_libraries_available from .trezor import (TrezorPlugin, TIM_NEW, TIM_RECOVER, TrezorInitSettings, - Capability, BackupType, RecoveryDeviceType) + PASSPHRASE_ON_DEVICE, Capability, BackupType, RecoveryDeviceType) PASSPHRASE_HELP_SHORT =_( @@ -119,6 +119,7 @@ def __init__(self, win, pin_matrix_widget_class, device): self.close_matrix_dialog_signal.connect(self._close_matrix_dialog) self.pin_matrix_widget_class = pin_matrix_widget_class self.matrix_dialog = None + self.passphrase_on_device = False def get_pin(self, msg): self.done.clear() @@ -163,6 +164,72 @@ def matrix_recovery_dialog(self, msg): self.matrix_dialog.get_matrix(msg) self.done.set() + def passphrase_dialog(self, msg, confirm): + # If confirm is true, require the user to enter the passphrase twice + parent = self.top_level_window() + d = WindowModalDialog(parent, _('Enter Passphrase')) + + OK_button = OkButton(d, _('Enter Passphrase')) + OnDevice_button = QPushButton(_('Enter Passphrase on Device')) + + new_pw = QLineEdit() + new_pw.setEchoMode(2) + conf_pw = QLineEdit() + conf_pw.setEchoMode(2) + + vbox = QVBoxLayout() + label = QLabel(msg + "\n") + label.setWordWrap(True) + + grid = QGridLayout() + grid.setSpacing(8) + grid.setColumnMinimumWidth(0, 150) + grid.setColumnMinimumWidth(1, 100) + grid.setColumnStretch(1,1) + + vbox.addWidget(label) + + grid.addWidget(QLabel(_('Passphrase:')), 0, 0) + grid.addWidget(new_pw, 0, 1) + + if confirm: + grid.addWidget(QLabel(_('Confirm Passphrase:')), 1, 0) + grid.addWidget(conf_pw, 1, 1) + + vbox.addLayout(grid) + + def enable_OK(): + if not confirm: + ok = True + else: + ok = new_pw.text() == conf_pw.text() + OK_button.setEnabled(ok) + + new_pw.textChanged.connect(enable_OK) + conf_pw.textChanged.connect(enable_OK) + + vbox.addWidget(OK_button) + + if self.passphrase_on_device: + vbox.addWidget(OnDevice_button) + + d.setLayout(vbox) + + self.passphrase = None + + def ok_clicked(): + self.passphrase = new_pw.text() + + def on_device_clicked(): + self.passphrase = PASSPHRASE_ON_DEVICE + + OK_button.clicked.connect(ok_clicked) + OnDevice_button.clicked.connect(on_device_clicked) + OnDevice_button.clicked.connect(d.accept) + + d.exec_() + self.done.set() + class QtPlugin(QtPluginBase): # Derived classes must provide the following class-static variables: diff --git a/electrum/plugins/trezor/trezor.py b/electrum/plugins/trezor/trezor.py index 71341d91c0..4a0bf9037f 100644 --- a/electrum/plugins/trezor/trezor.py +++ b/electrum/plugins/trezor/trezor.py @@ -32,6 +32,8 @@ InputScriptType, OutputScriptType, MultisigRedeemScriptType, TxInputType, TxOutputType, TxOutputBinType, TransactionType, SignTx) + from trezorlib.client import PASSPHRASE_ON_DEVICE + TREZORLIB = True except Exception as e: _logger.exception('error importing trezorlib') @@ -52,6 +54,8 @@ def __getattr__(self, key): BackupType = _EnumMissing() RecoveryDeviceType = _EnumMissing() + PASSPHRASE_ON_DEVICE = object() + # Trezor initialization methods TIM_NEW, TIM_RECOVER = range(2) @@ -109,11 +113,11 @@ class TrezorPlugin(HW_PluginBase): # wallet_class, types firmware_URL = 'https://wallet.trezor.io' - libraries_URL = 'https://github.com/trezor/python-trezor' + libraries_URL = 'https://pypi.org/project/trezor/' minimum_firmware = (1, 5, 2) keystore_class = TrezorKeyStore - minimum_library = (0, 11, 5) - maximum_library = (0, 12) + minimum_library = (0, 12, 0) + maximum_library = (0, 13) SUPPORTED_XTYPES = ('standard', 'p2wpkh-p2sh', 'p2wpkh', 'p2wsh-p2sh', 'p2wsh') DEVICE_IDS = (TREZOR_PRODUCT_KEY,) From 0ea21c59d256aaafe743dc6718c654477a7cbdec Mon Sep 17 00:00:00 2001 From: ThomasV Date: Tue, 31 Mar 2020 12:43:43 +0200 Subject: [PATCH 002/117] Save channel seed in localconfig --- electrum/lnpeer.py | 25 +++++++++---------------- electrum/lnutil.py | 18 +++++++++++++++++- electrum/tests/test_lnchannel.py | 1 + electrum/wallet_db.py | 11 ++++++++++- 4 files changed, 37 insertions(+), 18 deletions(-) diff --git a/electrum/lnpeer.py b/electrum/lnpeer.py index b93cc20e3e..950e941eec 100644 --- a/electrum/lnpeer.py +++ b/electrum/lnpeer.py @@ -20,7 +20,6 @@ from .crypto import sha256, sha256d from . import bitcoin -from .bip32 import BIP32Node from . import ecc from .ecc import sig_string_from_r_and_s, get_r_and_s_from_sig_string, der_sig_from_sig_string from . import constants @@ -484,38 +483,32 @@ def is_static_remotekey(self): return bool(self.features & LnFeatures.OPTION_STATIC_REMOTEKEY_OPT) def make_local_config(self, funding_sat: int, push_msat: int, initiator: HTLCOwner) -> LocalConfig: - # key derivation - seed = os.urandom(32) - node = BIP32Node.from_rootseed(seed, xtype='standard') - keypair_generator = lambda family: generate_keypair(node, family) + random_seed = os.urandom(32) if initiator == LOCAL: initial_msat = funding_sat * 1000 - push_msat else: initial_msat = push_msat if self.is_static_remotekey(): + # Note: in the future, if a CSV delay is added, + # we will want to derive that key wallet = self.lnworker.wallet assert wallet.txin_type == 'p2wpkh' addr = wallet.get_unused_address() - static_key = wallet.get_public_key(addr) # just a pubkey - payment_basepoint = OnlyPubkeyKeypair(bfh(static_key)) + static_remotekey = bfh(wallet.get_public_key(addr)) else: - payment_basepoint = keypair_generator(LnKeyFamily.PAYMENT_BASE) - - local_config=LocalConfig( - payment_basepoint=payment_basepoint, - multisig_key=keypair_generator(LnKeyFamily.MULTISIG), - htlc_basepoint=keypair_generator(LnKeyFamily.HTLC_BASE), - delayed_basepoint=keypair_generator(LnKeyFamily.DELAY_BASE), - revocation_basepoint=keypair_generator(LnKeyFamily.REVOCATION_BASE), + static_remotekey = None + + local_config = LocalConfig.from_seed( + seed=random_seed, + static_remotekey=static_remotekey, to_self_delay=DEFAULT_TO_SELF_DELAY, dust_limit_sat=546, max_htlc_value_in_flight_msat=funding_sat * 1000, max_accepted_htlcs=5, initial_msat=initial_msat, reserve_sat=546, - per_commitment_secret_seed=keypair_generator(LnKeyFamily.REVOCATION_ROOT).privkey, funding_locked_received=False, was_announced=False, current_commitment_signature=None, diff --git a/electrum/lnutil.py b/electrum/lnutil.py index af50a32db9..8da446da25 100644 --- a/electrum/lnutil.py +++ b/electrum/lnutil.py @@ -77,11 +77,27 @@ class Config(StoredObject): @attr.s class LocalConfig(Config): - per_commitment_secret_seed = attr.ib(type=bytes, converter=hex_to_bytes) + seed = attr.ib(type=bytes, converter=hex_to_bytes) funding_locked_received = attr.ib(type=bool) was_announced = attr.ib(type=bool) current_commitment_signature = attr.ib(type=bytes, converter=hex_to_bytes) current_htlc_signatures = attr.ib(type=bytes, converter=hex_to_bytes) + per_commitment_secret_seed = attr.ib(type=bytes, converter=hex_to_bytes) + + @classmethod + def from_seed(self, **kwargs): + seed = kwargs['seed'] + static_remotekey = kwargs.pop('static_remotekey') + node = BIP32Node.from_rootseed(seed, xtype='standard') + keypair_generator = lambda family: generate_keypair(node, family) + kwargs['per_commitment_secret_seed'] = keypair_generator(LnKeyFamily.REVOCATION_ROOT).privkey + kwargs['multisig_key'] = keypair_generator(LnKeyFamily.MULTISIG) + kwargs['htlc_basepoint'] = keypair_generator(LnKeyFamily.HTLC_BASE) + kwargs['delayed_basepoint'] = keypair_generator(LnKeyFamily.DELAY_BASE) + kwargs['revocation_basepoint'] = keypair_generator(LnKeyFamily.REVOCATION_BASE) + kwargs['payment_basepoint'] = OnlyPubkeyKeypair(static_remotekey) if static_remotekey else keypair_generator(LnKeyFamily.PAYMENT_BASE) + return LocalConfig(**kwargs) + @attr.s class RemoteConfig(Config): diff --git a/electrum/tests/test_lnchannel.py b/electrum/tests/test_lnchannel.py index 90f0704cc8..6e28e28e02 100644 --- a/electrum/tests/test_lnchannel.py +++ b/electrum/tests/test_lnchannel.py @@ -70,6 +70,7 @@ def create_channel_state(funding_txid, funding_index, funding_sat, is_initiator, current_per_commitment_point=cur, ), "local_config":lnpeer.LocalConfig( + seed = None, payment_basepoint=privkeys[0], multisig_key=privkeys[1], htlc_basepoint=privkeys[2], diff --git a/electrum/wallet_db.py b/electrum/wallet_db.py index 44065400fa..77c76d43e9 100644 --- a/electrum/wallet_db.py +++ b/electrum/wallet_db.py @@ -50,7 +50,7 @@ OLD_SEED_VERSION = 4 # electrum versions < 2.0 NEW_SEED_VERSION = 11 # electrum versions >= 2.0 -FINAL_SEED_VERSION = 27 # electrum >= 2.7 will set this to prevent +FINAL_SEED_VERSION = 28 # electrum >= 2.7 will set this to prevent # old versions from overwriting new format @@ -173,6 +173,7 @@ def upgrade(self): self._convert_version_25() self._convert_version_26() self._convert_version_27() + self._convert_version_28() self.put('seed_version', FINAL_SEED_VERSION) # just to be sure self._after_upgrade_tasks() @@ -596,6 +597,14 @@ def _convert_version_27(self): c['local_config']['htlc_minimum_msat'] = 1 self.data['seed_version'] = 27 + def _convert_version_28(self): + if not self._is_upgrade_method_needed(27, 27): + return + channels = self.data.get('channels', {}) + for channel_id, c in channels.items(): + c['local_config']['seed'] = None + self.data['seed_version'] = 28 + def _convert_imported(self): if not self._is_upgrade_method_needed(0, 13): return From 08bc8617ad1a07bf3a32fb5c055b741ecdb769d5 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Mon, 6 Apr 2020 12:53:57 +0200 Subject: [PATCH 003/117] change derivation of ln channel keys: use hardened paths --- electrum/lnutil.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/electrum/lnutil.py b/electrum/lnutil.py index 8da446da25..ac02eb223c 100644 --- a/electrum/lnutil.py +++ b/electrum/lnutil.py @@ -23,7 +23,7 @@ from . import segwit_addr from .i18n import _ from .lnaddr import lndecode -from .bip32 import BIP32Node +from .bip32 import BIP32Node, BIP32_PRIME if TYPE_CHECKING: from .lnchannel import Channel @@ -77,7 +77,7 @@ class Config(StoredObject): @attr.s class LocalConfig(Config): - seed = attr.ib(type=bytes, converter=hex_to_bytes) + seed = attr.ib(type=bytes, converter=hex_to_bytes) # type: Optional[bytes] funding_locked_received = attr.ib(type=bool) was_announced = attr.ib(type=bool) current_commitment_signature = attr.ib(type=bytes, converter=hex_to_bytes) @@ -1040,12 +1040,12 @@ def extract_nodeid(connect_contents: str) -> Tuple[bytes, str]: # key derivation # see lnd/keychain/derivation.go class LnKeyFamily(IntEnum): - MULTISIG = 0 - REVOCATION_BASE = 1 - HTLC_BASE = 2 - PAYMENT_BASE = 3 - DELAY_BASE = 4 - REVOCATION_ROOT = 5 + MULTISIG = 0 | BIP32_PRIME + REVOCATION_BASE = 1 | BIP32_PRIME + HTLC_BASE = 2 | BIP32_PRIME + PAYMENT_BASE = 3 | BIP32_PRIME + DELAY_BASE = 4 | BIP32_PRIME + REVOCATION_ROOT = 5 | BIP32_PRIME NODE_KEY = 6 From f3995350e8ad588f0ac788dc9cd59d7aac6ca5c3 Mon Sep 17 00:00:00 2001 From: ThomasV Date: Mon, 6 Apr 2020 16:53:48 +0200 Subject: [PATCH 004/117] localconfig: rename seed to channel_seed --- electrum/lnpeer.py | 12 +++--------- electrum/lnutil.py | 6 +++--- electrum/tests/test_lnchannel.py | 2 +- electrum/wallet_db.py | 2 +- 4 files changed, 8 insertions(+), 14 deletions(-) diff --git a/electrum/lnpeer.py b/electrum/lnpeer.py index 950e941eec..70e182c663 100644 --- a/electrum/lnpeer.py +++ b/electrum/lnpeer.py @@ -483,13 +483,8 @@ def is_static_remotekey(self): return bool(self.features & LnFeatures.OPTION_STATIC_REMOTEKEY_OPT) def make_local_config(self, funding_sat: int, push_msat: int, initiator: HTLCOwner) -> LocalConfig: - - random_seed = os.urandom(32) - if initiator == LOCAL: - initial_msat = funding_sat * 1000 - push_msat - else: - initial_msat = push_msat - + channel_seed = os.urandom(32) + initial_msat = funding_sat * 1000 - push_msat if initiator == LOCAL else push_msat if self.is_static_remotekey(): # Note: in the future, if a CSV delay is added, # we will want to derive that key @@ -499,9 +494,8 @@ def make_local_config(self, funding_sat: int, push_msat: int, initiator: HTLCOwn static_remotekey = bfh(wallet.get_public_key(addr)) else: static_remotekey = None - local_config = LocalConfig.from_seed( - seed=random_seed, + channel_seed=channel_seed, static_remotekey=static_remotekey, to_self_delay=DEFAULT_TO_SELF_DELAY, dust_limit_sat=546, diff --git a/electrum/lnutil.py b/electrum/lnutil.py index ac02eb223c..c99c80a074 100644 --- a/electrum/lnutil.py +++ b/electrum/lnutil.py @@ -77,7 +77,7 @@ class Config(StoredObject): @attr.s class LocalConfig(Config): - seed = attr.ib(type=bytes, converter=hex_to_bytes) # type: Optional[bytes] + channel_seed = attr.ib(type=bytes, converter=hex_to_bytes) # type: Optional[bytes] funding_locked_received = attr.ib(type=bool) was_announced = attr.ib(type=bool) current_commitment_signature = attr.ib(type=bytes, converter=hex_to_bytes) @@ -86,9 +86,9 @@ class LocalConfig(Config): @classmethod def from_seed(self, **kwargs): - seed = kwargs['seed'] + channel_seed = kwargs['channel_seed'] static_remotekey = kwargs.pop('static_remotekey') - node = BIP32Node.from_rootseed(seed, xtype='standard') + node = BIP32Node.from_rootseed(channel_seed, xtype='standard') keypair_generator = lambda family: generate_keypair(node, family) kwargs['per_commitment_secret_seed'] = keypair_generator(LnKeyFamily.REVOCATION_ROOT).privkey kwargs['multisig_key'] = keypair_generator(LnKeyFamily.MULTISIG) diff --git a/electrum/tests/test_lnchannel.py b/electrum/tests/test_lnchannel.py index 6e28e28e02..fe1163c29e 100644 --- a/electrum/tests/test_lnchannel.py +++ b/electrum/tests/test_lnchannel.py @@ -70,7 +70,7 @@ def create_channel_state(funding_txid, funding_index, funding_sat, is_initiator, current_per_commitment_point=cur, ), "local_config":lnpeer.LocalConfig( - seed = None, + channel_seed = None, payment_basepoint=privkeys[0], multisig_key=privkeys[1], htlc_basepoint=privkeys[2], diff --git a/electrum/wallet_db.py b/electrum/wallet_db.py index 77c76d43e9..e439b0dcc3 100644 --- a/electrum/wallet_db.py +++ b/electrum/wallet_db.py @@ -602,7 +602,7 @@ def _convert_version_28(self): return channels = self.data.get('channels', {}) for channel_id, c in channels.items(): - c['local_config']['seed'] = None + c['local_config']['channel_seed'] = None self.data['seed_version'] = 28 def _convert_imported(self): From 55d0a9587ec1dbadbecd6b97662bda0a84cee565 Mon Sep 17 00:00:00 2001 From: ThomasV Date: Mon, 6 Apr 2020 18:35:12 +0200 Subject: [PATCH 005/117] move maybe_save_short_chan_id to lnchannel --- electrum/lnchannel.py | 34 +++++++++++++++++++++++++++++++--- electrum/lnworker.py | 31 ------------------------------- 2 files changed, 31 insertions(+), 34 deletions(-) diff --git a/electrum/lnchannel.py b/electrum/lnchannel.py index 85ad99a469..ab28dad292 100644 --- a/electrum/lnchannel.py +++ b/electrum/lnchannel.py @@ -1105,6 +1105,33 @@ def update_onchain_state(self, funding_txid, funding_height, closing_txid, closi else: self.update_closed_state(funding_txid, funding_height, closing_txid, closing_height, keep_watching) + def is_funding_tx_mined(self, funding_height): + """ + Checks if Funding TX has been mined. If it has, save the short channel ID in chan; + if it's also deep enough, also save to disk. + Returns tuple (mined_deep_enough, num_confirmations). + """ + funding_txid = self.funding_outpoint.txid + funding_idx = self.funding_outpoint.output_index + conf = funding_height.conf + if conf < self.constraints.funding_txn_minimum_depth: + self.logger.info(f"funding tx is still not at sufficient depth. actual depth: {conf}") + return False + assert conf > 0 + # check funding_tx amount and script + funding_tx = self.lnworker.lnwatcher.db.get_transaction(funding_txid) + if not funding_tx: + self.logger.info(f"no funding_tx {funding_txid}") + return False + outp = funding_tx.outputs()[funding_idx] + redeem_script = funding_output_script(self.config[REMOTE], self.config[LOCAL]) + funding_address = redeem_script_to_address('p2wsh', redeem_script) + funding_sat = self.constraints.capacity + if not (outp.address == funding_address and outp.value == funding_sat): + self.logger.info('funding outpoint mismatch') + return False + return True + def update_unfunded_state(self): self.delete_funding_height() self.delete_closing_height() @@ -1136,10 +1163,11 @@ def update_funded_state(self, funding_txid, funding_height): self.save_funding_height(funding_txid, funding_height.height, funding_height.timestamp) self.delete_closing_height() if self.get_state() == channel_states.OPENING: - if self.short_channel_id is None: - self.lnworker.maybe_save_short_chan_id(self, funding_height) - if self.short_channel_id: + if self.is_funding_tx_mined(funding_height): self.set_state(channel_states.FUNDED) + self.set_short_channel_id(ShortChannelID.from_components( + funding_height.height, funding_height.txpos, self.funding_outpoint.output_index)) + self.logger.info(f"save_short_channel_id: {self.short_channel_id}") def update_closed_state(self, funding_txid, funding_height, closing_txid, closing_height, keep_watching): self.save_funding_height(funding_txid, funding_height.height, funding_height.timestamp) diff --git a/electrum/lnworker.py b/electrum/lnworker.py index 6da31e5c18..78a24a31d6 100644 --- a/electrum/lnworker.py +++ b/electrum/lnworker.py @@ -659,36 +659,6 @@ def save_channel(self, chan): self.wallet.save_db() self.network.trigger_callback('channel', chan) - def maybe_save_short_chan_id(self, chan, funding_height): - """ - Checks if Funding TX has been mined. If it has, save the short channel ID in chan; - if it's also deep enough, also save to disk. - Returns tuple (mined_deep_enough, num_confirmations). - """ - funding_txid = chan.funding_outpoint.txid - funding_idx = chan.funding_outpoint.output_index - conf = funding_height.conf - if conf < chan.constraints.funding_txn_minimum_depth: - self.logger.info(f"funding tx is still not at sufficient depth. actual depth: {conf}") - return - assert conf > 0 - # check funding_tx amount and script - funding_tx = self.lnwatcher.db.get_transaction(funding_txid) - if not funding_tx: - self.logger.info(f"no funding_tx {funding_txid}") - return - outp = funding_tx.outputs()[funding_idx] - redeem_script = funding_output_script(chan.config[REMOTE], chan.config[LOCAL]) - funding_address = redeem_script_to_address('p2wsh', redeem_script) - funding_sat = chan.constraints.capacity - if not (outp.address == funding_address and outp.value == funding_sat): - self.logger.info('funding outpoint mismatch') - return - chan.set_short_channel_id(ShortChannelID.from_components( - funding_height.height, funding_height.txpos, chan.funding_outpoint.output_index)) - self.logger.info(f"save_short_channel_id: {chan.short_channel_id}") - self.save_channel(chan) - def channel_by_txo(self, txo): with self.lock: channels = list(self.channels.values()) @@ -696,7 +666,6 @@ def channel_by_txo(self, txo): if chan.funding_outpoint.to_str() == txo: return chan - async def on_channel_update(self, chan): if chan.get_state() == channel_states.OPEN and chan.should_be_closed_due_to_expiring_htlcs(self.network.get_local_height()): From d2a58a2ec3682daea6a1f248eb7a543e2bad47fa Mon Sep 17 00:00:00 2001 From: ThomasV Date: Mon, 6 Apr 2020 19:06:27 +0200 Subject: [PATCH 006/117] lnpeer: do not assume our privkey is the same as lnworker's privkey. --- electrum/lnpeer.py | 2 +- electrum/tests/test_lnpeer.py | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/electrum/lnpeer.py b/electrum/lnpeer.py index 70e182c663..2bbb21eccd 100644 --- a/electrum/lnpeer.py +++ b/electrum/lnpeer.py @@ -75,7 +75,7 @@ def __init__(self, lnworker: Union['LNGossip', 'LNWallet'], pubkey:bytes, transp self.transport = transport self.pubkey = pubkey # remote pubkey self.lnworker = lnworker - self.privkey = lnworker.node_keypair.privkey # local privkey + self.privkey = self.transport.privkey # local privkey self.features = self.lnworker.features self.node_ids = [self.pubkey, privkey_to_pubkey(self.privkey)] self.network = lnworker.network diff --git a/electrum/tests/test_lnpeer.py b/electrum/tests/test_lnpeer.py index c351da3981..4089ef25b8 100644 --- a/electrum/tests/test_lnpeer.py +++ b/electrum/tests/test_lnpeer.py @@ -173,16 +173,17 @@ def send_bytes(self, data): self.queue.put_nowait(encode_msg('init', lflen=1, gflen=1, localfeatures=b"\x00", globalfeatures=b"\x00")) class PutIntoOthersQueueTransport(MockTransport): - def __init__(self, name): + def __init__(self, keypair, name): super().__init__(name) self.other_mock_transport = None + self.privkey = keypair.privkey def send_bytes(self, data): self.other_mock_transport.queue.put_nowait(data) -def transport_pair(name1, name2): - t1 = PutIntoOthersQueueTransport(name1) - t2 = PutIntoOthersQueueTransport(name2) +def transport_pair(k1, k2, name1, name2): + t1 = PutIntoOthersQueueTransport(k1, name1) + t2 = PutIntoOthersQueueTransport(k2, name2) t1.other_mock_transport = t2 t2.other_mock_transport = t1 return t1, t2 @@ -205,7 +206,7 @@ def tearDown(self): def prepare_peers(self, alice_channel, bob_channel): k1, k2 = keypair(), keypair() - t1, t2 = transport_pair(alice_channel.name, bob_channel.name) + t1, t2 = transport_pair(k2, k1, alice_channel.name, bob_channel.name) q1, q2 = asyncio.Queue(), asyncio.Queue() w1 = MockLNWallet(k1, k2, alice_channel, tx_queue=q1) w2 = MockLNWallet(k2, k1, bob_channel, tx_queue=q2) From fb5382f75f2168a9cbadbddd2d53f8820de59bf0 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Mon, 6 Apr 2020 19:49:56 +0200 Subject: [PATCH 007/117] follow-up prev (typo) --- electrum/plugins/trezor/clientbase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/electrum/plugins/trezor/clientbase.py b/electrum/plugins/trezor/clientbase.py index 66e32ad15a..430905b0a0 100644 --- a/electrum/plugins/trezor/clientbase.py +++ b/electrum/plugins/trezor/clientbase.py @@ -271,7 +271,7 @@ def get_passphrase(self, available_on_device): msg = _("Enter the passphrase to unlock this wallet:") self.handler.passphrase_on_device = available_on_device - passphrase = self.handler.trezor_get_passphrase(msg, self.creating_wallet) + passphrase = self.handler.get_passphrase(msg, self.creating_wallet) if passphrase is PASSPHRASE_ON_DEVICE: return passphrase if passphrase is None: From f11bf1dd4a98a5eed571a38f9456a118d2a768a0 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Mon, 6 Apr 2020 20:12:14 +0200 Subject: [PATCH 008/117] rerun freeze_packages --- .../requirements-binaries.txt | 82 ++++--- .../deterministic-build/requirements-hw.txt | 206 +++++++++--------- .../requirements-wine-build.txt | 24 +- contrib/deterministic-build/requirements.txt | 199 +++++++++-------- 4 files changed, 256 insertions(+), 255 deletions(-) diff --git a/contrib/deterministic-build/requirements-binaries.txt b/contrib/deterministic-build/requirements-binaries.txt index 4747cef69b..504e53cab0 100644 --- a/contrib/deterministic-build/requirements-binaries.txt +++ b/contrib/deterministic-build/requirements-binaries.txt @@ -1,39 +1,37 @@ -pip==19.3.1 \ - --hash=sha256:21207d76c1031e517668898a6b46a9fb1501c7a4710ef5dfd6a40ad9e6757ea7 \ - --hash=sha256:6917c65fc3769ecdc61405d3dfd97afdedd75808d200b2838d7d961cebc0c2c7 -pycryptodomex==3.9.4 \ - --hash=sha256:0943b65fb41b7403a9def6214061fdd9ab9afd0bbc581e553c72eebe60bded36 \ - --hash=sha256:0a1dbb5c4d975a4ea568fb7686550aa225d94023191fb0cca8747dc5b5d77857 \ - --hash=sha256:0f43f1608518347fdcb9c8f443fa5cabedd33f94188b13e4196a3a7ba90d169c \ - --hash=sha256:11ce5fec5990e34e3981ed14897ba601c83957b577d77d395f1f8f878a179f98 \ - --hash=sha256:17a09e38fdc91e4857cf5a7ce82f3c0b229c3977490f2146513e366923fc256b \ - --hash=sha256:22d970cee5c096b9123415e183ae03702b2cd4d3ba3f0ced25c4e1aba3967167 \ - --hash=sha256:2a1793efcbae3a2264c5e0e492a2629eb10d895d6e5f17dbbd00eb8b489c6bda \ - --hash=sha256:30a8a148a0fe482cec1aaf942bbd0ade56ec197c14fe058b2a94318c57e1f991 \ - --hash=sha256:32fbbaf964c5184d3f3e349085b0536dd28184b02e2b014fc900f58bbc126339 \ - --hash=sha256:347d67faee36d449dc9632da411cc318df52959079062627f1243001b10dc227 \ - --hash=sha256:45f4b4e5461a041518baabc52340c249b60833aa84cea6377dc8016a2b33c666 \ - --hash=sha256:4717daec0035034b002d31c42e55431c970e3e38a78211f43990e1b7eaf19e28 \ - --hash=sha256:51a1ac9e7dda81da444fed8be558a60ec88dfc73b2aa4b0efa310e87acb75838 \ - --hash=sha256:53e9dcc8f14783f6300b70da325a50ac1b0a3dbaee323bd9dc3f71d409c197a1 \ - --hash=sha256:5519a2ed776e193688b7ddb61ab709303f6eb7d1237081e298283c72acc44271 \ - --hash=sha256:583450e8e80a0885c453211ed2bd69ceea634d8c904f23ff8687f677fe810e95 \ - --hash=sha256:60f862bd2a07133585a4fc2ce2b1a8ec24746b07ac44307d22ef2b767cb03435 \ - --hash=sha256:612091f1d3c84e723bec7cb855cf77576e646045744794c9a3f75ba80737762f \ - --hash=sha256:629a87b87c8203b8789ccefc7f2f2faecd2daaeb56bdd0b4e44cd89565f2db07 \ - --hash=sha256:6e56ec4c8938fb388b6f250ddd5e21c15e8f25a76e0ad0e2abae9afee09e67b4 \ - --hash=sha256:8e8092651844a11ec7fa534395f3dfe99256ce4edca06f128efc9d770d6e1dc1 \ - --hash=sha256:8f5f260629876603e08f3ce95c8ccd9b6b83bf9a921c41409046796267f7adc5 \ - --hash=sha256:9a6b74f38613f54c56bd759b411a352258f47489bbefd1d57c930a291498b35b \ - --hash=sha256:a5a13ebb52c4cd065fb673d8c94f39f30823428a4de19e1f3f828b63a8882d1e \ - --hash=sha256:a77ca778a476829876a3a70ae880073379160e4a465d057e3c4e1c79acdf1b8a \ - --hash=sha256:a9f7be3d19f79429c2118fd61bc2ec4fa095e93b56fb3a5f3009822402c4380f \ - --hash=sha256:dc15a467c4f9e4b43748ba2f97aea66f67812bfd581818284c47cadc81d4caec \ - --hash=sha256:e13cdeea23059f7577c230fd580d2c8178e67ebe10e360041abe86c33c316f1c \ - --hash=sha256:e45b85c8521bca6bdfaf57e4987743ade53e9f03529dd3adbc9524094c6d55c4 \ - --hash=sha256:e87f17867b260f57c88487f943eb4d46c90532652bb37046e764842c3b66cbb1 \ - --hash=sha256:ee40a5b156f6c1192bc3082e9d73d0479904433cdda83110546cd67f5a15a5be \ - --hash=sha256:ef63ffde3b267043579af8830fc97fc3b9b8a526a24e3ba23af9989d4e9e689a +pip==20.0.2 \ + --hash=sha256:4ae14a42d8adba3205ebeb38aa68cfc0b6c346e1ae2e699a0b3bad4da19cef5c \ + --hash=sha256:7db0c8ea4c7ea51c8049640e8e6e7fde949de672bfa4949920675563a5a6967f +pycryptodomex==3.9.7 \ + --hash=sha256:1537d2d15b604b303aef56e7f440895a1c81adbee786b91f1f06eddc34da5314 \ + --hash=sha256:1d20ab8369b7558168fc014a0745c678613f9f486dae468cca2d68145196b8a4 \ + --hash=sha256:1ecc9db7409db67765eb008e558879d298406642d33ade43a6488224d23e8081 \ + --hash=sha256:37033976f72af829fe15f7fe5fe1dbed308cc43a98d9dd9d2a0a76de8ca5ee78 \ + --hash=sha256:3c3dd9d4c9c1e279d3945ae422895c901f98987333acc132dc094faf52afec35 \ + --hash=sha256:3c9b3fba037ea52c626060c5a87ee6de7e86c99e8a7c6ee07302539985d2bd64 \ + --hash=sha256:45ee555fc5e28c119a46d44ce373f5237e54a35c61b750fb3a94446b09855dbc \ + --hash=sha256:4c93038ac011b36512cb0bf2ee3e2aec774e8bc81021d015917c89fe02bb0ee5 \ + --hash=sha256:50163324834edd0c9ce3e4512ded3e221c969086e10fdd5d3fdcaadac5e24a78 \ + --hash=sha256:59b0ea9cda5490f924771456912a225d8d9e678891f9f986661af718534719b2 \ + --hash=sha256:5cf306a17cccc327a33cdc3845629fa13f4573a4ec620ed607c79cf6785f2e27 \ + --hash=sha256:5fff8da399af16a1855f58771223acbbdac720b9969cd03fc5013d2e9a7bd9a4 \ + --hash=sha256:68650ce5b9f7152b8283302a4617269f821695a612692640dd247bd12ab21c0b \ + --hash=sha256:6b3a9a562688996f760b5077714c3ab8b62ca56061b6e9ab7906841e43e19f91 \ + --hash=sha256:7e938ed51a59e29431ea86fab60423ada2757728db0f78952329fa02a789bd31 \ + --hash=sha256:87aa70daad6f039e814790a06422a3189311198b674b62f13933a2bdcb6b1bcc \ + --hash=sha256:99be3a1df2b2b9f731ebe1c264a2c07c465e71cee68e35e1640b645b5213a755 \ + --hash=sha256:a3f2908666e6f74b8c4893f86dd02e16170f50e4a78ae7f3468b6208d54bc205 \ + --hash=sha256:ae3d44a639fd11dbdeca47e35e94febb1ee8bc15daf26673331add37146e0b85 \ + --hash=sha256:afb4c2fa3c6f492fd9a8b38d76e13f32d429b8e5e1e00238309391b5591cde0d \ + --hash=sha256:b1515ce3a8a2c3fa537d137c5ca5f8b7a902044d04e07d7c3aa26c3e026120fb \ + --hash=sha256:bf391b377413a197000b43ef2b74359974d8927d329a897c9f5ba7b63dca7b9c \ + --hash=sha256:c436919117c23355740c669f89720673578b9aa4569bbfe105f6c10101fc1966 \ + --hash=sha256:d2c3c280975638e2a2c2fd9cb36ab111980219757fa163a2755594b9448e4138 \ + --hash=sha256:e585d530764c459cbd5d460aed0288807bb881f376ca9a20e653645217895961 \ + --hash=sha256:e76e6638ead4a7d93262a24218f0ff3ff74de6b6c823b7e19dccb31b6a481978 \ + --hash=sha256:ebfc2f885cafda076c31ae30fa0dd81e7e919ec34059a88d3018ed66e83fcce3 \ + --hash=sha256:f5797a39933a3d41526da60856735e6684b2b71a8ca99d5f79555ca121be2f4b \ + --hash=sha256:f7e5fc5e124200b19a14be33fb0099e956e6ebb5e25d287b0829ef0a78ed76c7 \ + --hash=sha256:fb350e31e55211fec8ddc89fc0256f3b9bc3b44b68a8bde1cf44b3b4e80c0e42 PyQt5==5.11.3 \ --hash=sha256:517e4339135c4874b799af0d484bc2e8c27b54850113a68eec40a0b56534f450 \ --hash=sha256:ac1eb5a114b6e7788e8be378be41c5e54b17d5158994504e85e43b5fca006a39 \ @@ -52,9 +50,9 @@ PyQt5-sip==4.19.13 \ --hash=sha256:a91a308a5e0cc99de1e97afd8f09f46dd7ca20cfaa5890ef254113eebaa1adff \ --hash=sha256:b0342540da479d2713edc68fb21f307473f68da896ad5c04215dae97630e0069 \ --hash=sha256:f997e21b4e26a3397cb7b255b8d1db5b9772c8e0c94b6d870a5a0ab5c27eacaa -setuptools==42.0.2 \ - --hash=sha256:c5b372090d7c8709ce79a6a66872a91e518f7d65af97fca78135e1cb10d4b940 \ - --hash=sha256:c8abd0f3574bc23afd2f6fd2c415ba7d9e097c8a99b845473b0d957ba1e2dac6 -wheel==0.33.6 \ - --hash=sha256:10c9da68765315ed98850f8e048347c3eb06dd81822dc2ab1d4fde9dc9702646 \ - --hash=sha256:f4da1763d3becf2e2cd92a14a7c920f0f00eca30fdde9ea992c836685b9faf28 +setuptools==46.1.3 \ + --hash=sha256:4fe404eec2738c20ab5841fa2d791902d2a645f32318a7850ef26f8d7215a8ee \ + --hash=sha256:795e0475ba6cd7fa082b1ee6e90d552209995627a2a227a47c6ea93282f4bfb1 +wheel==0.34.2 \ + --hash=sha256:8788e9155fe14f54164c1b9eb0a319d98ef02c160725587ad60f14ddc57b6f96 \ + --hash=sha256:df277cb51e61359aba502208d680f90c0493adec6f0e848af94948778aed386e diff --git a/contrib/deterministic-build/requirements-hw.txt b/contrib/deterministic-build/requirements-hw.txt index 38513ebd60..17b38ce36b 100644 --- a/contrib/deterministic-build/requirements-hw.txt +++ b/contrib/deterministic-build/requirements-hw.txt @@ -1,66 +1,68 @@ btchip-python==0.1.28 \ --hash=sha256:da09d0d7a6180d428833795ea9a233c3b317ddfcccea8cc6f0eba59435e5dd83 -certifi==2019.11.28 \ - --hash=sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3 \ - --hash=sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f +certifi==2020.4.5.1 \ + --hash=sha256:1d987a998c75633c40847cc966fcf5904906c920a7f17ef374f5aa4282abd304 \ + --hash=sha256:51fcb31174be6e6664c5f69e3e1691a2d72a1a12e90f872cbdb1567eb47b6519 chardet==3.0.4 \ --hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae \ --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 -ckcc-protocol==0.8.0 \ - --hash=sha256:bad1d1448423472df95ba67621fdd0ad919e625fbe0a4d3ba93648f34ea286e0 \ - --hash=sha256:f0851c98b91825d19567d0d3bac1b28044d40a3d5f194c8b04c5338f114d7ad5 -click==7.0 \ - --hash=sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13 \ - --hash=sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7 -construct==2.9.45 \ - --hash=sha256:2271a0efd0798679dea825ff47e22a4c550456a5db0ba8baa82f7eae0af0118c -Cython==0.29.10 \ - --hash=sha256:0afa0b121b89de619e71587e25702e2b7068d7da2164c47e6eee80c17823a62f \ - --hash=sha256:1c608ba76f7a20cc9f0c021b7fe5cb04bc1a70327ae93a9298b1bc3e0edddebe \ - --hash=sha256:26229570d6787ff3caa932fe9d802960f51a89239b990d275ae845405ce43857 \ - --hash=sha256:2a9deafa437b6154cac2f25bb88e0bfd075a897c8dc847669d6f478d7e3ee6b1 \ - --hash=sha256:2f28396fbce6d9d68a40edbf49a6729cf9d92a4d39ff0f501947a89188e9099f \ - --hash=sha256:3983dd7b67297db299b403b29b328d9e03e14c4c590ea90aa1ad1d7b35fb178b \ - --hash=sha256:4100a3f8e8bbe47d499cdac00e56d5fe750f739701ea52dc049b6c56f5421d97 \ - --hash=sha256:51abfaa7b6c66f3f18028876713c8804e73d4c2b6ceddbcbcfa8ec62429377f0 \ - --hash=sha256:61c24f4554efdb8fb1ac6c8e75dab301bcdf2b7b739ed0c2b267493bb43163c5 \ - --hash=sha256:700ccf921b2fdc9b23910e95b5caae4b35767685e0812343fa7172409f1b5830 \ - --hash=sha256:7b41eb2e792822a790cb2a171df49d1a9e0baaa8e81f58077b7380a273b93d5f \ - --hash=sha256:803987d3b16d55faa997bfc12e8b97f1091f145930dee229b020487aed8a1f44 \ - --hash=sha256:99af5cfcd208c81998dcf44b3ca466dee7e17453cfb50e98b87947c3a86f8753 \ - --hash=sha256:9faea1cca34501c7e139bc7ef8e504d532b77865c58592493e2c154a003b450f \ - --hash=sha256:a7ba4c9a174db841cfee9a0b92563862a0301d7ca543334666c7266b541f141a \ - --hash=sha256:b26071c2313d1880599c69fd831a07b32a8c961ba69d7ccbe5db1cd8d319a4ca \ - --hash=sha256:b49dc8e1116abde13a3e6a9eb8da6ab292c5a3325155fb872e39011b110b37e6 \ - --hash=sha256:bd40def0fd013569887008baa6da9ca428e3d7247adeeaeada153006227bb2e7 \ - --hash=sha256:bfd0db770e8bd4e044e20298dcae6dfc42561f85d17ee546dcd978c8b23066ae \ - --hash=sha256:c2fad1efae5889925c8fd7867fdd61f59480e4e0b510f9db096c912e884704f1 \ - --hash=sha256:c81aea93d526ccf6bc0b842c91216ee9867cd8792f6725a00f19c8b5837e1715 \ - --hash=sha256:da786e039b4ad2bce3d53d4799438cf1f5e01a0108f1b8d78ac08e6627281b1a \ - --hash=sha256:deab85a069397540987082d251e9c89e0e5b2e3e044014344ff81f60e211fc4b \ - --hash=sha256:e3f1e6224c3407beb1849bdc5ae3150929e593e4cffff6ca41c6ec2b10942c80 \ - --hash=sha256:e74eb224e53aae3943d66e2d29fe42322d5753fd4c0641329bccb7efb3a46552 \ - --hash=sha256:ee697c7ea65cb14915a64f36874da8ffc2123df43cf8bc952172e04a26656cd6 \ - --hash=sha256:f37792b16d11606c28e428460bd6a3d14b8917b109e77cdbe4ca78b0b9a52c87 \ - --hash=sha256:fd2906b54cbf879c09d875ad4e4687c58d87f5ed03496063fec1c9065569fd5d -ecdsa==0.14.1 \ - --hash=sha256:64c613005f13efec6541bb0a33290d0d03c27abab5f15fbab20fb0ee162bdd8e \ - --hash=sha256:e108a5fe92c67639abae3260e43561af914e7fd0d27bae6d2ec1312ae7934dfe -hidapi==0.7.99.post21 \ - --hash=sha256:1ac170f4d601c340f2cd52fd06e85c5e77bad7ceac811a7bb54b529f7dc28c24 \ - --hash=sha256:6424ad75da0021ce8c1bcd78056a04adada303eff3c561f8d132b85d0a914cb3 \ - --hash=sha256:8d3be666f464347022e2b47caf9132287885d9eacc7895314fc8fefcb4e42946 \ - --hash=sha256:92878bad7324dee619b7832fbfc60b5360d378aa7c5addbfef0a410d8fd342c7 \ - --hash=sha256:b4b1f6aff0192e9be153fe07c1b7576cb7a1ff52e78e3f76d867be95301a8e87 \ - --hash=sha256:bf03f06f586ce7d8aeb697a94b7dba12dc9271aae92d7a8d4486360ff711a660 \ - --hash=sha256:c76de162937326fcd57aa399f94939ce726242323e65c15c67e183da1f6c26f7 \ - --hash=sha256:d4ad1e46aef98783a9e6274d523b8b1e766acfc3d72828cd44a337564d984cfa \ - --hash=sha256:d4b5787a04613503357606bb10e59c3e2c1114fa00ee328b838dd257f41cbd7b \ - --hash=sha256:e0be1aa6566979266a8fc845ab0e18613f4918cf2c977fe67050f5dc7e2a9a97 \ - --hash=sha256:edfb16b16a298717cf05b8c8a9ad1828b6ff3de5e93048ceccd74e6ae4ff0922 -idna==2.8 \ - --hash=sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407 \ - --hash=sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c +ckcc-protocol==1.0.1 \ + --hash=sha256:03f2e1a629d4f36842e5404b9a797305a7142ab65bdebbf2eec1fafe245c308e \ + --hash=sha256:6605889e28a80573738a94c86372869137ffb10e876135af12e50fb2de0a3688 +click==7.1.1 \ + --hash=sha256:8a18b4ea89d8820c5d0c7da8a64b2c324b4dabb695804dbfea19b9be9d88c0cc \ + --hash=sha256:e345d143d80bf5ee7534056164e5e112ea5e22716bbb1ce727941f4c8b471b9a +construct==2.10.56 \ + --hash=sha256:97ba13edcd98546f10f7555af41c8ce7ae9d8221525ec4062c03f9adbf940661 +Cython==0.29.16 \ + --hash=sha256:0542a6c4ff1be839b6479deffdbdff1a330697d7953dd63b6de99c078e3acd5f \ + --hash=sha256:0bcf7f87aa0ba8b62d4f3b6e0146e48779eaa4f39f92092d7ff90081ef6133e0 \ + --hash=sha256:13408a5e5574b322153a23f23eb9e69306d4d8216428b435b75fdab9538ad169 \ + --hash=sha256:1846a8f4366fb4041d34cd37c2d022421ab1a28bcf79ffa6cf33a45b5acba9af \ + --hash=sha256:1d32d0965c2fc1476af9c367e396c3ecc294d4bde2cfde6f1704e8787e3f0e1f \ + --hash=sha256:21d6abd25e0fcfa96edf164831f53ca20deb64221eb3b7d6d1c4d582f4c54c84 \ + --hash=sha256:232755284f942cbb3b43a06cd85974ef3c970a021aef19b5243c03ee2b08fa05 \ + --hash=sha256:245e69a1f367c89e3c8a1c2699bd20ab67b3d57053f3c71f0623d36def074308 \ + --hash=sha256:3a274c63a3575ae9d6cde5a31c2f5cb18d0a34d9bded96433ceb86d11dc0806d \ + --hash=sha256:3b400efb38d6092d2ee7f6d9835dd6dc4f99e804abf97652a5839ff9b1910f25 \ + --hash=sha256:4ab2054325a7856ed0df881b8ffdadae05b29cf3549635f741c18ce2c860f51b \ + --hash=sha256:4b5efb5bff2a1ed0c23dd131223566a0cc51c5266e70968082aed75b73f8c1e2 \ + --hash=sha256:54e7bf8a2a0c8536f4c42fc5ef54e6780153826279aef923317cf919671119f4 \ + --hash=sha256:59a0b01fc9376c2424eb3b09a0550f1cbd51681a59cee1e02c9d5c546c601679 \ + --hash=sha256:5ba06cf0cfc79686daedf9a7895cad4c993c453b86240fc54ecbe9b0c951504c \ + --hash=sha256:66768684fdee5f9395e6ee2daa9f770b37455fcb22d31960843bd72996aaa84f \ + --hash=sha256:772c13250aea33ac17eb042544b310f0dc3862bbde49b334f5c12f7d1b627476 \ + --hash=sha256:7d31c4b518b34b427b51e85c6827473b08f473df2fcba75969daad65ea2a5f6c \ + --hash=sha256:961f11eb427161a8f5b35e74285a5ff6651eee710dbe092072af3e9337e26825 \ + --hash=sha256:96342c9f934bcce22eaef739e4fca9ce5cc5347df4673f4de8e5dce5fe158444 \ + --hash=sha256:a507d507b45af9657b050cea780e668cbcb9280eb94a5755c634a48760b1d035 \ + --hash=sha256:ad318b60d13767838e99cf93f3571849946eb960c54da86c000b97b2ffa60128 \ + --hash=sha256:b137bb2f6e079bd04e6b3ea15e9f9b9c97982ec0b1037d48972940577d3a57bb \ + --hash=sha256:b3f95ba4d251400bfd38b0891128d9b6365a54f06bd4d58ba033ecb39d2788cc \ + --hash=sha256:c0937ab8185d7f55bf7145dbfa3cc27a9d69916d4274690b18b9d1022ac54fd8 \ + --hash=sha256:c2c28d22bfea830c0cdbd0d7f373d4f51366893a18a5bbd4dd8deb1e6bdd08c2 \ + --hash=sha256:e074e2be68b4cb1d17b9c63d89ae0592742bdbc320466f342e1e1ea77ec83c40 \ + --hash=sha256:e9abcc8013354f0059c16af9c917d19341a41981bb74dcc44e060f8a88db9123 \ + --hash=sha256:eb757a4076e7bb1ca3e73fba4ec2b1c07ca0634200904f1df8f7f899c57b17af \ + --hash=sha256:f4ecb562b5b6a2d80543ec36f7fbc7c1a4341bb837a5fc8bd3c352470508133c \ + --hash=sha256:f516d11179627f95471cc0674afe8710d4dc5de764297db7f5bdb34bd92caff9 \ + --hash=sha256:fd6496b41eb529349d58f3f6a09a64cceb156c9720f79cebdf975ea4fafc05f0 +ecdsa==0.15 \ + --hash=sha256:867ec9cf6df0b03addc8ef66b56359643cb5d0c1dc329df76ba7ecfe256c8061 \ + --hash=sha256:8f12ac317f8a1318efa75757ef0a651abe12e51fc1af8838fb91079445227277 +hidapi==0.9.0.post2 \ + --hash=sha256:03b9118749f6102a96af175b2b77832c0d6f8957acb46ced5aa7afcf358052bc \ + --hash=sha256:3b31b396b6e95b635db4db8e9649cdb0aa2c205dd4cd8aaf3ee9807dddb1ebb8 \ + --hash=sha256:448c2ba9f713a5ee754830b222c9bc54a4e0dca4ecd0d84e3bf14314949ec594 \ + --hash=sha256:4c712309e2534a249721feb2abe7baedb9bfe7b3cc0e06cf4b78329684480932 \ + --hash=sha256:9c4369499a322d91d9f697c6b84b78f78c42695743641cb8bf3b5fa8c3c9b09c \ + --hash=sha256:a71dd3c153cb6bb2b73d2612b5ab262830d78c6428f33f0c06818749e64c9320 \ + --hash=sha256:d8dd636b7da9dfeb4aa08da64aceb91fb311465faae347b885cb8b695b141364 \ + --hash=sha256:da40dcf99ea15d440f3f3667f4166addd5676c485acf331c6e7c6c7879e11633 \ + --hash=sha256:dc633b34e318ce4638b73beb531136ab02ab005bfb383c260a41b5dfd5d85f16 +idna==2.9 \ + --hash=sha256:7588d1c14ae4c77d74036e8c22ff447b26d0fde8f007354fd48a7814db15b7cb \ + --hash=sha256:a068a21ceac8a4d63dbfd964670474107f541babbd2250d61922f029858365fa keepkey==6.3.1 \ --hash=sha256:88e2b5291c85c8e8567732f675697b88241082884aa1aba32257f35ee722fc09 \ --hash=sha256:cef1e862e195ece3e42640a0f57d15a63086fd1dedc8b5ddfcbc9c2657f0bb1e \ @@ -70,51 +72,53 @@ libusb1==1.7.1 \ mnemonic==0.19 \ --hash=sha256:4e37eb02b2cbd56a0079cabe58a6da93e60e3e4d6e757a586d9f23d96abea931 \ --hash=sha256:a8d78c5100acfa7df9bab6b9db7390831b0e54490934b718ff9efd68f0d731a6 -pip==19.3.1 \ - --hash=sha256:21207d76c1031e517668898a6b46a9fb1501c7a4710ef5dfd6a40ad9e6757ea7 \ - --hash=sha256:6917c65fc3769ecdc61405d3dfd97afdedd75808d200b2838d7d961cebc0c2c7 -protobuf==3.11.1 \ - --hash=sha256:0265379852b9e1f76af6d3d3fe4b3c383a595cc937594bda8565cf69a96baabd \ - --hash=sha256:200b77e51f17fbc1d3049045f5835f60405dec3a00fe876b9b986592e46d908c \ - --hash=sha256:29bd1ed46b2536ad8959401a2f02d2d7b5a309f8e97518e4f92ca6c5ba74dbed \ - --hash=sha256:3175d45698edb9a07c1a78a1a4850e674ce8988f20596580158b1d0921d0f057 \ - --hash=sha256:34a7270940f86da7a28be466ac541c89b6dbf144a6348b9cf7ac6f56b71006ce \ - --hash=sha256:38cbc830a4a5ba9956763b0f37090bfd14dd74e72762be6225de2ceac55f4d03 \ - --hash=sha256:665194f5ad386511ac8d8a0bd57b9ab37b8dd2cd71969458777318e774b9cd46 \ - --hash=sha256:839bad7d115c77cdff29b488fae6a3ab503ce9a4192bd4c42302a6ea8e5d0f33 \ - --hash=sha256:934a9869a7f3b0d84eca460e386fba1f7ba2a0c1a120a2648bc41fadf50efd1c \ - --hash=sha256:aecdf12ef6dc7fd91713a6da93a86c2f2a8fe54840a3b1670853a2b7402e77c9 \ - --hash=sha256:c4e90bc27c0691c76e09b5dc506133451e52caee1472b8b3c741b7c912ce43ef \ - --hash=sha256:c65d135ea2d85d40309e268106dab02d3bea723db2db21c23ecad4163ced210b \ - --hash=sha256:c98dea04a1ff41a70aff2489610f280004831798cb36a068013eed04c698903d \ - --hash=sha256:d9049aa194378a426f0b2c784e2054565bf6f754d20fcafdee7102a6250556e8 \ - --hash=sha256:e028fee51c96de4e81924484c77111dfdea14010ecfc906ea5b252209b0c4de6 \ - --hash=sha256:e84ad26fb50091b1ea676403c0dd2bd47663099454aa6d88000b1dafecab0941 \ - --hash=sha256:e88a924b591b06d0191620e9c8aa75297b3111066bb09d49a24bae1054a10c13 +pip==20.0.2 \ + --hash=sha256:4ae14a42d8adba3205ebeb38aa68cfc0b6c346e1ae2e699a0b3bad4da19cef5c \ + --hash=sha256:7db0c8ea4c7ea51c8049640e8e6e7fde949de672bfa4949920675563a5a6967f +protobuf==3.11.3 \ + --hash=sha256:0bae429443cc4748be2aadfdaf9633297cfaeb24a9a02d0ab15849175ce90fab \ + --hash=sha256:24e3b6ad259544d717902777b33966a1a069208c885576254c112663e6a5bb0f \ + --hash=sha256:2affcaba328c4662f3bc3c0e9576ea107906b2c2b6422344cdad961734ff6b93 \ + --hash=sha256:310a7aca6e7f257510d0c750364774034272538d51796ca31d42c3925d12a52a \ + --hash=sha256:52e586072612c1eec18e1174f8e3bb19d08f075fc2e3f91d3b16c919078469d0 \ + --hash=sha256:73152776dc75f335c476d11d52ec6f0f6925774802cd48d6189f4d5d7fe753f4 \ + --hash=sha256:7774bbbaac81d3ba86de646c39f154afc8156717972bf0450c9dbfa1dc8dbea2 \ + --hash=sha256:82d7ac987715d8d1eb4068bf997f3053468e0ce0287e2729c30601feb6602fee \ + --hash=sha256:8eb9c93798b904f141d9de36a0ba9f9b73cc382869e67c9e642c0aba53b0fc07 \ + --hash=sha256:adf0e4d57b33881d0c63bb11e7f9038f98ee0c3e334c221f0858f826e8fb0151 \ + --hash=sha256:c40973a0aee65422d8cb4e7d7cbded95dfeee0199caab54d5ab25b63bce8135a \ + --hash=sha256:c77c974d1dadf246d789f6dad1c24426137c9091e930dbf50e0a29c1fcf00b1f \ + --hash=sha256:dd9aa4401c36785ea1b6fff0552c674bdd1b641319cb07ed1fe2392388e9b0d7 \ + --hash=sha256:e11df1ac6905e81b815ab6fd518e79be0a58b5dc427a2cf7208980f30694b956 \ + --hash=sha256:e2f8a75261c26b2f5f3442b0525d50fd79a71aeca04b5ec270fc123536188306 \ + --hash=sha256:e512b7f3a4dd780f59f1bf22c302740e27b10b5c97e858a6061772668cd6f961 \ + --hash=sha256:ef2c2e56aaf9ee914d3dccc3408d42661aaf7d9bb78eaa8f17b2e6282f214481 \ + --hash=sha256:fac513a9dc2a74b99abd2e17109b53945e364649ca03d9f7a0b96aa8d1807d0a \ + --hash=sha256:fdfb6ad138dbbf92b5dbea3576d7c8ba7463173f7d2cb0ca1bd336ec88ddbd80 pyaes==1.6.1 \ --hash=sha256:02c1b1405c38d3c370b085fb952dd8bea3fadcee6411ad99f312cc129c536d8f -requests==2.22.0 \ - --hash=sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4 \ - --hash=sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31 +requests==2.23.0 \ + --hash=sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee \ + --hash=sha256:b3f43d496c6daba4493e7c431722aeb7dbc6288f52a6e04e7b6023b0247817e6 safet==0.1.5 \ --hash=sha256:a7fd4b68bb1bc6185298af665c8e8e00e2bb2bcbddbb22844ead929b845c635e \ --hash=sha256:f966a23243312f64d14c7dfe02e8f13f6eeba4c3f51341f2c11ae57831f07de3 -setuptools==42.0.2 \ - --hash=sha256:c5b372090d7c8709ce79a6a66872a91e518f7d65af97fca78135e1cb10d4b940 \ - --hash=sha256:c8abd0f3574bc23afd2f6fd2c415ba7d9e097c8a99b845473b0d957ba1e2dac6 -six==1.13.0 \ - --hash=sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd \ - --hash=sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66 -trezor==0.11.5 \ - --hash=sha256:711137bb83e7e0aef4009745e0da1b7d258146f246b43e3f7f5b849405088ef1 \ - --hash=sha256:cd8aafd70a281daa644c4a3fb021ffac20b7a88e86226ecc8bb3e78e1734a184 -typing-extensions==3.7.4.1 \ - --hash=sha256:091ecc894d5e908ac75209f10d5b4f118fbdb2eb1ede6a63544054bb1edb41f2 \ - --hash=sha256:910f4656f54de5993ad9304959ce9bb903f90aadc7c67a0bef07e678014e892d \ - --hash=sha256:cf8b63fedea4d89bab840ecbb93e75578af28f76f66c35889bd7065f5af88575 -urllib3==1.25.7 \ - --hash=sha256:a8a318824cc77d1fd4b2bec2ded92646630d7fe8619497b142c84a9e6f5a7293 \ - --hash=sha256:f3c5fd51747d450d4dcf6f923c81f78f811aab8205fda64b0aba34a4e48b0745 -wheel==0.33.6 \ - --hash=sha256:10c9da68765315ed98850f8e048347c3eb06dd81822dc2ab1d4fde9dc9702646 \ - --hash=sha256:f4da1763d3becf2e2cd92a14a7c920f0f00eca30fdde9ea992c836685b9faf28 +setuptools==46.1.3 \ + --hash=sha256:4fe404eec2738c20ab5841fa2d791902d2a645f32318a7850ef26f8d7215a8ee \ + --hash=sha256:795e0475ba6cd7fa082b1ee6e90d552209995627a2a227a47c6ea93282f4bfb1 +six==1.14.0 \ + --hash=sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a \ + --hash=sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c +trezor==0.12.0 \ + --hash=sha256:da5b750ada03830fd1f0b9010f7d5d30e77ec3e1458230e3d08fe4588a0741b2 \ + --hash=sha256:f6bc821bddec06e67a1abd0be1d9fbc61c59b08272c736522ae2f6b225bf9579 +typing-extensions==3.7.4.2 \ + --hash=sha256:6e95524d8a547a91e08f404ae485bbb71962de46967e1b71a0cb89af24e761c5 \ + --hash=sha256:79ee589a3caca649a9bfd2a8de4709837400dfa00b6cc81962a1e6a1815969ae \ + --hash=sha256:f8d2bd89d25bc39dabe7d23df520442fa1d8969b82544370e03d88b5a591c392 +urllib3==1.25.8 \ + --hash=sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc \ + --hash=sha256:87716c2d2a7121198ebcb7ce7cccf6ce5e9ba539041cfbaeecfb641dc0bf6acc +wheel==0.34.2 \ + --hash=sha256:8788e9155fe14f54164c1b9eb0a319d98ef02c160725587ad60f14ddc57b6f96 \ + --hash=sha256:df277cb51e61359aba502208d680f90c0493adec6f0e848af94948778aed386e diff --git a/contrib/deterministic-build/requirements-wine-build.txt b/contrib/deterministic-build/requirements-wine-build.txt index 50df23c93f..147e9ff507 100644 --- a/contrib/deterministic-build/requirements-wine-build.txt +++ b/contrib/deterministic-build/requirements-wine-build.txt @@ -1,19 +1,19 @@ -altgraph==0.16.1 \ - --hash=sha256:d6814989f242b2b43025cba7161fc1b8fb487a62cd49c49245d6fd01c18ac997 \ - --hash=sha256:ddf5320017147ba7b810198e0b6619bd7b5563aa034da388cea8546b877f9b0c +altgraph==0.17 \ + --hash=sha256:1f05a47122542f97028caf78775a095fbe6a2699b5089de8477eb583167d69aa \ + --hash=sha256:c623e5f3408ca61d4016f23a681b9adb100802ca3e3da5e718915a9e4052cebe future==0.18.2 \ --hash=sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d pefile==2019.4.18 \ --hash=sha256:a5d6e8305c6b210849b47a6174ddf9c452b2888340b8177874b862ba6c207645 -pip==19.3.1 \ - --hash=sha256:21207d76c1031e517668898a6b46a9fb1501c7a4710ef5dfd6a40ad9e6757ea7 \ - --hash=sha256:6917c65fc3769ecdc61405d3dfd97afdedd75808d200b2838d7d961cebc0c2c7 +pip==20.0.2 \ + --hash=sha256:4ae14a42d8adba3205ebeb38aa68cfc0b6c346e1ae2e699a0b3bad4da19cef5c \ + --hash=sha256:7db0c8ea4c7ea51c8049640e8e6e7fde949de672bfa4949920675563a5a6967f pywin32-ctypes==0.2.0 \ --hash=sha256:24ffc3b341d457d48e8922352130cf2644024a4ff09762a2261fd34c36ee5942 \ --hash=sha256:9dc2d991b3479cc2df15930958b674a48a227d5361d413827a4cfd0b5876fc98 -setuptools==42.0.2 \ - --hash=sha256:c5b372090d7c8709ce79a6a66872a91e518f7d65af97fca78135e1cb10d4b940 \ - --hash=sha256:c8abd0f3574bc23afd2f6fd2c415ba7d9e097c8a99b845473b0d957ba1e2dac6 -wheel==0.33.6 \ - --hash=sha256:10c9da68765315ed98850f8e048347c3eb06dd81822dc2ab1d4fde9dc9702646 \ - --hash=sha256:f4da1763d3becf2e2cd92a14a7c920f0f00eca30fdde9ea992c836685b9faf28 +setuptools==46.1.3 \ + --hash=sha256:4fe404eec2738c20ab5841fa2d791902d2a645f32318a7850ef26f8d7215a8ee \ + --hash=sha256:795e0475ba6cd7fa082b1ee6e90d552209995627a2a227a47c6ea93282f4bfb1 +wheel==0.34.2 \ + --hash=sha256:8788e9155fe14f54164c1b9eb0a319d98ef02c160725587ad60f14ddc57b6f96 \ + --hash=sha256:df277cb51e61359aba502208d680f90c0493adec6f0e848af94948778aed386e \ No newline at end of file diff --git a/contrib/deterministic-build/requirements.txt b/contrib/deterministic-build/requirements.txt index be25855b1f..9cc3ed2971 100644 --- a/contrib/deterministic-build/requirements.txt +++ b/contrib/deterministic-build/requirements.txt @@ -11,9 +11,9 @@ aiohttp==3.6.2 \ --hash=sha256:65f31b622af739a802ca6fd1a3076fd0ae523f8485c52924a89561ba10c49b48 \ --hash=sha256:ae55bac364c405caa23a4f2d6cfecc6a0daada500274ffca4a9230e7129eac59 \ --hash=sha256:b778ce0c909a2653741cb4b1ac7015b5c130ab9c897611df43ae6a58523cb965 -aiohttp-socks==0.2.2 \ - --hash=sha256:e473ee222b001fe33798957b9ce3352b32c187cf41684f8e2259427925914993 \ - --hash=sha256:eebd8939a7c3c1e3e7e1b2552c60039b4c65ef6b8b2351efcbdd98290538e310 +aiohttp-socks==0.3.7 \ + --hash=sha256:43803a8eafed9c1eaccf2c6f09a485daf91663d653dd2bdf6732dcece0a4f803 \ + --hash=sha256:47912c72a645716e822159376905c4f0c71fa4858f37698bdd7c4ee40e6f68d4 aiorpcX==0.18.4 \ --hash=sha256:bec9c0feb328d62ba80b79931b07f7372c98f2891ad51300be0b7163d5ccfb4a \ --hash=sha256:d424a55bcf52ebf1b3610a7809c0748fac91ce926854ad33ce952463bc6017e8 @@ -29,9 +29,9 @@ bitstring==3.1.6 \ --hash=sha256:7b60b0c300d0d3d0a24ec84abfda4b0eaed3dc56dc90f6cbfe497166c9ad8443 \ --hash=sha256:c97a8e2a136e99b523b27da420736ae5cb68f83519d633794a6a11192f69f8bf \ --hash=sha256:e392819965e7e0246e3cf6a51d5a54e731890ae03ebbfa3cd0e4f74909072096 -certifi==2019.11.28 \ - --hash=sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3 \ - --hash=sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f +certifi==2020.4.5.1 \ + --hash=sha256:1d987a998c75633c40847cc966fcf5904906c920a7f17ef374f5aa4282abd304 \ + --hash=sha256:51fcb31174be6e6664c5f69e3e1691a2d72a1a12e90f872cbdb1567eb47b6519 chardet==3.0.4 \ --hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae \ --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 @@ -41,110 +41,109 @@ click==6.7 \ dnspython==1.16.0 \ --hash=sha256:36c5e8e38d4369a08b6780b7f27d790a292b2b08eea01607865bf0936c558e01 \ --hash=sha256:f69c21288a962f4da86e56c4905b49d11aba7938d3d740e80d9e366ee4f1632d -ecdsa==0.14.1 \ - --hash=sha256:64c613005f13efec6541bb0a33290d0d03c27abab5f15fbab20fb0ee162bdd8e \ - --hash=sha256:e108a5fe92c67639abae3260e43561af914e7fd0d27bae6d2ec1312ae7934dfe -idna==2.8 \ - --hash=sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407 \ - --hash=sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c +ecdsa==0.15 \ + --hash=sha256:867ec9cf6df0b03addc8ef66b56359643cb5d0c1dc329df76ba7ecfe256c8061 \ + --hash=sha256:8f12ac317f8a1318efa75757ef0a651abe12e51fc1af8838fb91079445227277 +idna==2.9 \ + --hash=sha256:7588d1c14ae4c77d74036e8c22ff447b26d0fde8f007354fd48a7814db15b7cb \ + --hash=sha256:a068a21ceac8a4d63dbfd964670474107f541babbd2250d61922f029858365fa idna_ssl==1.1.0 \ --hash=sha256:a933e3bb13da54383f9e8f35dc4f9cb9eb9b3b78c6b36f311254d6d0d92c6c7c -importlib-metadata==1.1.0 \ - --hash=sha256:b044f07694ef14a6683b097ba56bd081dbc7cdc7c7fe46011e499dfecc082f21 \ - --hash=sha256:e6ac600a142cf2db707b1998382cc7fc3b02befb7273876e01b8ad10b9652742 -jsonrpcclient==3.3.4 \ - --hash=sha256:c50860409b73af9f94b648439caae3b4af80d5ac937f2a8ac7783de3d1050ba9 -jsonrpcserver==4.0.5 \ - --hash=sha256:240c517f49b0fdd3bfa428c9a7cc581126a0c43eca60d29762da124017d9d9f4 +importlib-metadata==1.6.0 \ + --hash=sha256:2a688cbaa90e0cc587f1df48bdc97a6eadccdcd9c35fb3f976a09e3b5016d90f \ + --hash=sha256:34513a8a0c4962bc66d35b359558fd8a5e10cd472d37aec5f66858addef32c1e +jsonrpcclient==3.3.5 \ + --hash=sha256:a17d02a53061748384b15b4e9812e866d5f69771656ccf7031d6dc64d0c38099 +jsonrpcserver==4.1.2 \ + --hash=sha256:73db55d1cf245ebdfb96ca05c4cce01c51b61be845a2a981f539ea1e6a4e0c4a jsonschema==3.2.0 \ --hash=sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163 \ --hash=sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a -more-itertools==8.0.0 \ - --hash=sha256:53ff73f186307d9c8ef17a9600309154a6ae27f25579e80af4db8f047ba14bc2 \ - --hash=sha256:a0ea684c39bc4315ba7aae406596ef191fd84f873d2d2751f84d64e81a7a2d45 -multidict==4.6.1 \ - --hash=sha256:07f9a6bf75ad675d53956b2c6a2d4ef2fa63132f33ecc99e9c24cf93beb0d10b \ - --hash=sha256:0ffe4d4d28cbe9801952bfb52a8095dd9ffecebd93f84bdf973c76300de783c5 \ - --hash=sha256:1b605272c558e4c659dbaf0fb32a53bfede44121bcf77b356e6e906867b958b7 \ - --hash=sha256:205a011e636d885af6dd0029e41e3514a46e05bb2a43251a619a6e8348b96fc0 \ - --hash=sha256:250632316295f2311e1ed43e6b26a63b0216b866b45c11441886ac1543ca96e1 \ - --hash=sha256:2bc9c2579312c68a3552ee816311c8da76412e6f6a9cf33b15152e385a572d2a \ - --hash=sha256:318aadf1cfb6741c555c7dd83d94f746dc95989f4f106b25b8a83dfb547f2756 \ - --hash=sha256:42cdd649741a14b0602bf15985cad0dd4696a380081a3319cd1ead46fd0f0fab \ - --hash=sha256:5159c4975931a1a78bf6602bbebaa366747fce0a56cb2111f44789d2c45e379f \ - --hash=sha256:87e26d8b89127c25659e962c61a4c655ec7445d19150daea0759516884ecb8b4 \ - --hash=sha256:891b7e142885e17a894d9d22b0349b92bb2da4769b4e675665d0331c08719be5 \ - --hash=sha256:8d919034420378132d074bf89df148d0193e9780c9fe7c0e495e895b8af4d8a2 \ - --hash=sha256:9c890978e2b37dd0dc1bd952da9a5d9f245d4807bee33e3517e4119c48d66f8c \ - --hash=sha256:a37433ce8cdb35fc9e6e47e1606fa1bfd6d70440879038dca7d8dd023197eaa9 \ - --hash=sha256:c626029841ada34c030b94a00c573a0c7575fe66489cde148785b6535397d675 \ - --hash=sha256:cfec9d001a83dc73580143f3c77e898cf7ad78b27bb5e64dbe9652668fcafec7 \ - --hash=sha256:efaf1b18ea6c1f577b1371c0159edbe4749558bfe983e13aa24d0a0c01e1ad7b -pip==19.3.1 \ - --hash=sha256:21207d76c1031e517668898a6b46a9fb1501c7a4710ef5dfd6a40ad9e6757ea7 \ - --hash=sha256:6917c65fc3769ecdc61405d3dfd97afdedd75808d200b2838d7d961cebc0c2c7 -protobuf==3.11.1 \ - --hash=sha256:0265379852b9e1f76af6d3d3fe4b3c383a595cc937594bda8565cf69a96baabd \ - --hash=sha256:200b77e51f17fbc1d3049045f5835f60405dec3a00fe876b9b986592e46d908c \ - --hash=sha256:29bd1ed46b2536ad8959401a2f02d2d7b5a309f8e97518e4f92ca6c5ba74dbed \ - --hash=sha256:3175d45698edb9a07c1a78a1a4850e674ce8988f20596580158b1d0921d0f057 \ - --hash=sha256:34a7270940f86da7a28be466ac541c89b6dbf144a6348b9cf7ac6f56b71006ce \ - --hash=sha256:38cbc830a4a5ba9956763b0f37090bfd14dd74e72762be6225de2ceac55f4d03 \ - --hash=sha256:665194f5ad386511ac8d8a0bd57b9ab37b8dd2cd71969458777318e774b9cd46 \ - --hash=sha256:839bad7d115c77cdff29b488fae6a3ab503ce9a4192bd4c42302a6ea8e5d0f33 \ - --hash=sha256:934a9869a7f3b0d84eca460e386fba1f7ba2a0c1a120a2648bc41fadf50efd1c \ - --hash=sha256:aecdf12ef6dc7fd91713a6da93a86c2f2a8fe54840a3b1670853a2b7402e77c9 \ - --hash=sha256:c4e90bc27c0691c76e09b5dc506133451e52caee1472b8b3c741b7c912ce43ef \ - --hash=sha256:c65d135ea2d85d40309e268106dab02d3bea723db2db21c23ecad4163ced210b \ - --hash=sha256:c98dea04a1ff41a70aff2489610f280004831798cb36a068013eed04c698903d \ - --hash=sha256:d9049aa194378a426f0b2c784e2054565bf6f754d20fcafdee7102a6250556e8 \ - --hash=sha256:e028fee51c96de4e81924484c77111dfdea14010ecfc906ea5b252209b0c4de6 \ - --hash=sha256:e84ad26fb50091b1ea676403c0dd2bd47663099454aa6d88000b1dafecab0941 \ - --hash=sha256:e88a924b591b06d0191620e9c8aa75297b3111066bb09d49a24bae1054a10c13 +multidict==4.7.5 \ + --hash=sha256:317f96bc0950d249e96d8d29ab556d01dd38888fbe68324f46fd834b430169f1 \ + --hash=sha256:42f56542166040b4474c0c608ed051732033cd821126493cf25b6c276df7dd35 \ + --hash=sha256:4b7df040fb5fe826d689204f9b544af469593fb3ff3a069a6ad3409f742f5928 \ + --hash=sha256:544fae9261232a97102e27a926019100a9db75bec7b37feedd74b3aa82f29969 \ + --hash=sha256:620b37c3fea181dab09267cd5a84b0f23fa043beb8bc50d8474dd9694de1fa6e \ + --hash=sha256:6e6fef114741c4d7ca46da8449038ec8b1e880bbe68674c01ceeb1ac8a648e78 \ + --hash=sha256:7774e9f6c9af3f12f296131453f7b81dabb7ebdb948483362f5afcaac8a826f1 \ + --hash=sha256:85cb26c38c96f76b7ff38b86c9d560dea10cf3459bb5f4caf72fc1bb932c7136 \ + --hash=sha256:a326f4240123a2ac66bb163eeba99578e9d63a8654a59f4688a79198f9aa10f8 \ + --hash=sha256:ae402f43604e3b2bc41e8ea8b8526c7fa7139ed76b0d64fc48e28125925275b2 \ + --hash=sha256:aee283c49601fa4c13adc64c09c978838a7e812f85377ae130a24d7198c0331e \ + --hash=sha256:b51249fdd2923739cd3efc95a3d6c363b67bbf779208e9f37fd5e68540d1a4d4 \ + --hash=sha256:bb519becc46275c594410c6c28a8a0adc66fe24fef154a9addea54c1adb006f5 \ + --hash=sha256:c2c37185fb0af79d5c117b8d2764f4321eeb12ba8c141a95d0aa8c2c1d0a11dd \ + --hash=sha256:dc561313279f9d05a3d0ffa89cd15ae477528ea37aa9795c4654588a3287a9ab \ + --hash=sha256:e439c9a10a95cb32abd708bb8be83b2134fa93790a4fb0535ca36db3dda94d20 \ + --hash=sha256:fc3b4adc2ee8474cb3cd2a155305d5f8eda0a9c91320f83e55748e1fcb68f8e3 +pip==20.0.2 \ + --hash=sha256:4ae14a42d8adba3205ebeb38aa68cfc0b6c346e1ae2e699a0b3bad4da19cef5c \ + --hash=sha256:7db0c8ea4c7ea51c8049640e8e6e7fde949de672bfa4949920675563a5a6967f +protobuf==3.11.3 \ + --hash=sha256:0bae429443cc4748be2aadfdaf9633297cfaeb24a9a02d0ab15849175ce90fab \ + --hash=sha256:24e3b6ad259544d717902777b33966a1a069208c885576254c112663e6a5bb0f \ + --hash=sha256:2affcaba328c4662f3bc3c0e9576ea107906b2c2b6422344cdad961734ff6b93 \ + --hash=sha256:310a7aca6e7f257510d0c750364774034272538d51796ca31d42c3925d12a52a \ + --hash=sha256:52e586072612c1eec18e1174f8e3bb19d08f075fc2e3f91d3b16c919078469d0 \ + --hash=sha256:73152776dc75f335c476d11d52ec6f0f6925774802cd48d6189f4d5d7fe753f4 \ + --hash=sha256:7774bbbaac81d3ba86de646c39f154afc8156717972bf0450c9dbfa1dc8dbea2 \ + --hash=sha256:82d7ac987715d8d1eb4068bf997f3053468e0ce0287e2729c30601feb6602fee \ + --hash=sha256:8eb9c93798b904f141d9de36a0ba9f9b73cc382869e67c9e642c0aba53b0fc07 \ + --hash=sha256:adf0e4d57b33881d0c63bb11e7f9038f98ee0c3e334c221f0858f826e8fb0151 \ + --hash=sha256:c40973a0aee65422d8cb4e7d7cbded95dfeee0199caab54d5ab25b63bce8135a \ + --hash=sha256:c77c974d1dadf246d789f6dad1c24426137c9091e930dbf50e0a29c1fcf00b1f \ + --hash=sha256:dd9aa4401c36785ea1b6fff0552c674bdd1b641319cb07ed1fe2392388e9b0d7 \ + --hash=sha256:e11df1ac6905e81b815ab6fd518e79be0a58b5dc427a2cf7208980f30694b956 \ + --hash=sha256:e2f8a75261c26b2f5f3442b0525d50fd79a71aeca04b5ec270fc123536188306 \ + --hash=sha256:e512b7f3a4dd780f59f1bf22c302740e27b10b5c97e858a6061772668cd6f961 \ + --hash=sha256:ef2c2e56aaf9ee914d3dccc3408d42661aaf7d9bb78eaa8f17b2e6282f214481 \ + --hash=sha256:fac513a9dc2a74b99abd2e17109b53945e364649ca03d9f7a0b96aa8d1807d0a \ + --hash=sha256:fdfb6ad138dbbf92b5dbea3576d7c8ba7463173f7d2cb0ca1bd336ec88ddbd80 pyaes==1.6.1 \ --hash=sha256:02c1b1405c38d3c370b085fb952dd8bea3fadcee6411ad99f312cc129c536d8f -pyrsistent==0.15.6 \ - --hash=sha256:f3b280d030afb652f79d67c5586157c5c1355c9a58dfc7940566e28d28f3df1b +pyrsistent==0.16.0 \ + --hash=sha256:28669905fe725965daa16184933676547c5bb40a5153055a8dee2a4bd7933ad3 QDarkStyle==2.6.8 \ --hash=sha256:037a54bf0aa5153f8055b65b8b36ac0d0f7648f2fd906c011a4da22eb0f582a2 \ --hash=sha256:fd1abae37d3a0a004089178da7c0b26ec5eb29f965b3e573853b8f280b614dea qrcode==6.1 \ --hash=sha256:3996ee560fc39532910603704c82980ff6d4d5d629f9c3f25f34174ce8606cf5 \ --hash=sha256:505253854f607f2abf4d16092c61d4e9d511a3b4392e60bff957a68592b04369 -setuptools==42.0.2 \ - --hash=sha256:c5b372090d7c8709ce79a6a66872a91e518f7d65af97fca78135e1cb10d4b940 \ - --hash=sha256:c8abd0f3574bc23afd2f6fd2c415ba7d9e097c8a99b845473b0d957ba1e2dac6 -six==1.13.0 \ - --hash=sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd \ - --hash=sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66 -typing-extensions==3.7.4.1 \ - --hash=sha256:091ecc894d5e908ac75209f10d5b4f118fbdb2eb1ede6a63544054bb1edb41f2 \ - --hash=sha256:910f4656f54de5993ad9304959ce9bb903f90aadc7c67a0bef07e678014e892d \ - --hash=sha256:cf8b63fedea4d89bab840ecbb93e75578af28f76f66c35889bd7065f5af88575 -wheel==0.33.6 \ - --hash=sha256:10c9da68765315ed98850f8e048347c3eb06dd81822dc2ab1d4fde9dc9702646 \ - --hash=sha256:f4da1763d3becf2e2cd92a14a7c920f0f00eca30fdde9ea992c836685b9faf28 -yarl==1.4.1 \ - --hash=sha256:031e8f56cf085d3b3df6b6bce756369ea7052b82d35ea07b6045f209c819e0e5 \ - --hash=sha256:074958fe4578ef3a3d0bdaf96bbc25e4c4db82b7ff523594776fcf3d3f16c531 \ - --hash=sha256:2db667ee21f620b446a54a793e467714fc5a446fcc82d93a47e8bde01d69afab \ - --hash=sha256:326f2dbaaa17b858ae86f261ae73a266fd820a561fc5142cee9d0fc58448fbd7 \ - --hash=sha256:32a3885f542f74d0f4f87057050c6b45529ebd79d0639f56582e741521575bfe \ - --hash=sha256:56126ef061b913c3eefecace3404ca88917265d0550b8e32bbbeab29e5c830bf \ - --hash=sha256:589ac1e82add13fbdedc04eb0a83400db728e5f1af2bd273392088ca90de7062 \ - --hash=sha256:6076bce2ecc6ebf6c92919d77762f80f4c9c6ecc9c1fbaa16567ec59ad7d6f1d \ - --hash=sha256:63be649c535d18ab6230efbc06a07f7779cd4336a687672defe70c025349a47b \ - --hash=sha256:6642cbc92eaffa586180f669adc772f5c34977e9e849e93f33dc142351e98c9c \ - --hash=sha256:6fa05a25f2280e78a514041d4609d39962e7d51525f2439db9ad7a2ae7aac163 \ - --hash=sha256:7ed006a220422c33ff0889288be24db56ff0a3008ffe9eaead58a690715ad09b \ - --hash=sha256:80c9c213803b50899460cc355f47e66778c3c868f448b7b7de5b1f1858c82c2a \ - --hash=sha256:8bae18e2129850e76969b57869dacc72a66cccdbeebce1a28d7f3d439c21a7a3 \ - --hash=sha256:ab112fba996a8f48f427e26969f2066d50080df0c24007a8cc6d7ae865e19013 \ - --hash=sha256:b1c178ef813940c9a5cbad42ab7b8b76ac08b594b0a6bad91063c968e0466efc \ - --hash=sha256:d6eff151c3b23a56a5e4f496805619bc3bdf4f749f63a7a95ad50e8267c17475 -zipp==0.6.0 \ - --hash=sha256:3718b1cbcd963c7d4c5511a8240812904164b7f381b647143a89d3b98f9bcd8e \ - --hash=sha256:f06903e9f1f43b12d371004b4ac7b06ab39a44adc747266928ae6debfa7b3335 -colorama==0.4.1 \ - --hash=sha256:05eed71e2e327246ad6b38c540c4a3117230b19679b875190486ddd2d721422d \ - --hash=sha256:f8ac84de7840f5b9c4e3347b3c1eaa50f7e49c2b07596221daec5edaabbd7c48 +setuptools==46.1.3 \ + --hash=sha256:4fe404eec2738c20ab5841fa2d791902d2a645f32318a7850ef26f8d7215a8ee \ + --hash=sha256:795e0475ba6cd7fa082b1ee6e90d552209995627a2a227a47c6ea93282f4bfb1 +six==1.14.0 \ + --hash=sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a \ + --hash=sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c +typing-extensions==3.7.4.2 \ + --hash=sha256:6e95524d8a547a91e08f404ae485bbb71962de46967e1b71a0cb89af24e761c5 \ + --hash=sha256:79ee589a3caca649a9bfd2a8de4709837400dfa00b6cc81962a1e6a1815969ae \ + --hash=sha256:f8d2bd89d25bc39dabe7d23df520442fa1d8969b82544370e03d88b5a591c392 +wheel==0.34.2 \ + --hash=sha256:8788e9155fe14f54164c1b9eb0a319d98ef02c160725587ad60f14ddc57b6f96 \ + --hash=sha256:df277cb51e61359aba502208d680f90c0493adec6f0e848af94948778aed386e +yarl==1.4.2 \ + --hash=sha256:0c2ab325d33f1b824734b3ef51d4d54a54e0e7a23d13b86974507602334c2cce \ + --hash=sha256:0ca2f395591bbd85ddd50a82eb1fde9c1066fafe888c5c7cc1d810cf03fd3cc6 \ + --hash=sha256:2098a4b4b9d75ee352807a95cdf5f10180db903bc5b7270715c6bbe2551f64ce \ + --hash=sha256:25e66e5e2007c7a39541ca13b559cd8ebc2ad8fe00ea94a2aad28a9b1e44e5ae \ + --hash=sha256:26d7c90cb04dee1665282a5d1a998defc1a9e012fdca0f33396f81508f49696d \ + --hash=sha256:308b98b0c8cd1dfef1a0311dc5e38ae8f9b58349226aa0533f15a16717ad702f \ + --hash=sha256:3ce3d4f7c6b69c4e4f0704b32eca8123b9c58ae91af740481aa57d7857b5e41b \ + --hash=sha256:58cd9c469eced558cd81aa3f484b2924e8897049e06889e8ff2510435b7ef74b \ + --hash=sha256:5b10eb0e7f044cf0b035112446b26a3a2946bca9d7d7edb5e54a2ad2f6652abb \ + --hash=sha256:6faa19d3824c21bcbfdfce5171e193c8b4ddafdf0ac3f129ccf0cdfcb083e462 \ + --hash=sha256:944494be42fa630134bf907714d40207e646fd5a94423c90d5b514f7b0713fea \ + --hash=sha256:a161de7e50224e8e3de6e184707476b5a989037dcb24292b391a3d66ff158e70 \ + --hash=sha256:a4844ebb2be14768f7994f2017f70aca39d658a96c786211be5ddbe1c68794c1 \ + --hash=sha256:c2b509ac3d4b988ae8769901c66345425e361d518aecbe4acbfc2567e416626a \ + --hash=sha256:c9959d49a77b0e07559e579f38b2f3711c2b8716b8410b320bf9713013215a1b \ + --hash=sha256:d8cdee92bc930d8b09d8bd2043cedd544d9c8bd7436a77678dd602467a993080 \ + --hash=sha256:e15199cdb423316e15f108f51249e44eb156ae5dba232cb73be555324a1d49c2 +zipp==3.1.0 \ + --hash=sha256:aa36550ff0c0b7ef7fa639055d797116ee891440eac1a56f378e2d3179e0320b \ + --hash=sha256:c599e4d75c98f6798c509911d08a22e6c021d074469042177c8c86fb92eefd96 +colorama==0.4.3 \ + --hash=sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff \ + --hash=sha256:e96da0d330793e2cb9485e9ddfd918d456036c7149416295932478192f4436a1 From c798e5d9a19f42c86aa09bb48fda465ddc6e774d Mon Sep 17 00:00:00 2001 From: SomberNight Date: Tue, 7 Apr 2020 16:48:26 +0200 Subject: [PATCH 009/117] qt: introduce PasswordLineEdit(QLineEdit) --- electrum/gui/qt/confirm_tx_dialog.py | 6 +++--- electrum/gui/qt/installwizard.py | 5 ++--- electrum/gui/qt/network_dialog.py | 6 +++--- electrum/gui/qt/password_dialog.py | 15 ++++++--------- electrum/gui/qt/util.py | 6 ++++++ electrum/plugins/hw_wallet/qt.py | 6 +++--- electrum/plugins/ledger/auth2fa.py | 8 ++++---- electrum/plugins/trezor/qt.py | 8 +++----- 8 files changed, 30 insertions(+), 30 deletions(-) diff --git a/electrum/gui/qt/confirm_tx_dialog.py b/electrum/gui/qt/confirm_tx_dialog.py index ee5e35c2f8..8b10598fd1 100644 --- a/electrum/gui/qt/confirm_tx_dialog.py +++ b/electrum/gui/qt/confirm_tx_dialog.py @@ -34,7 +34,8 @@ from electrum.simple_config import FEERATE_WARNING_HIGH_FEE from electrum.wallet import InternalAddressCorruption -from .util import WindowModalDialog, ColorScheme, HelpLabel, Buttons, CancelButton, BlockingWaitingDialog +from .util import (WindowModalDialog, ColorScheme, HelpLabel, Buttons, CancelButton, + BlockingWaitingDialog, PasswordLineEdit) from .fee_slider import FeeSlider @@ -144,8 +145,7 @@ def __init__(self, *, window: 'ElectrumWindow', make_tx, output_value: Union[int grid.addWidget(self.message_label, 6, 0, 1, -1) self.pw_label = QLabel(_('Password')) self.pw_label.setVisible(self.password_required) - self.pw = QLineEdit() - self.pw.setEchoMode(2) + self.pw = PasswordLineEdit() self.pw.setVisible(self.password_required) grid.addWidget(self.pw_label, 8, 0) grid.addWidget(self.pw, 8, 1, 1, -1) diff --git a/electrum/gui/qt/installwizard.py b/electrum/gui/qt/installwizard.py index 762b931e72..834c8fb4ea 100644 --- a/electrum/gui/qt/installwizard.py +++ b/electrum/gui/qt/installwizard.py @@ -25,7 +25,7 @@ from .seed_dialog import SeedLayout, KeysLayout from .network_dialog import NetworkChoiceLayout from .util import (MessageBoxMixin, Buttons, icon_path, ChoicesLayout, WWLabel, - InfoButton, char_width_in_lineedit) + InfoButton, char_width_in_lineedit, PasswordLineEdit) from .password_dialog import PasswordLayout, PasswordLayoutForHW, PW_NEW from electrum.plugin import run_hook, Plugins @@ -196,9 +196,8 @@ def select_storage(self, path, get_wallet_from_daemon) -> Tuple[str, Optional[Wa msg_label = WWLabel('') vbox.addWidget(msg_label) hbox2 = QHBoxLayout() - pw_e = QLineEdit('', self) + pw_e = PasswordLineEdit('', self) pw_e.setFixedWidth(17 * char_width_in_lineedit()) - pw_e.setEchoMode(2) pw_label = QLabel(_('Password') + ':') hbox2.addWidget(pw_label) hbox2.addWidget(pw_e) diff --git a/electrum/gui/qt/network_dialog.py b/electrum/gui/qt/network_dialog.py index 92e3733d56..428df1b516 100644 --- a/electrum/gui/qt/network_dialog.py +++ b/electrum/gui/qt/network_dialog.py @@ -40,7 +40,8 @@ from electrum.network import Network from electrum.logging import get_logger -from .util import Buttons, CloseButton, HelpButton, read_QIcon, char_width_in_lineedit +from .util import (Buttons, CloseButton, HelpButton, read_QIcon, char_width_in_lineedit, + PasswordLineEdit) _logger = get_logger(__name__) @@ -267,9 +268,8 @@ def __init__(self, network: Network, config, wizard=False): self.proxy_port.setFixedWidth(fixed_width_port) self.proxy_user = QLineEdit() self.proxy_user.setPlaceholderText(_("Proxy user")) - self.proxy_password = QLineEdit() + self.proxy_password = PasswordLineEdit() self.proxy_password.setPlaceholderText(_("Password")) - self.proxy_password.setEchoMode(QLineEdit.Password) self.proxy_password.setFixedWidth(fixed_width_port) self.proxy_mode.currentIndexChanged.connect(self.set_proxy) diff --git a/electrum/gui/qt/password_dialog.py b/electrum/gui/qt/password_dialog.py index 851bbc5aaf..f35c479dab 100644 --- a/electrum/gui/qt/password_dialog.py +++ b/electrum/gui/qt/password_dialog.py @@ -33,7 +33,8 @@ from electrum.i18n import _ from electrum.plugin import run_hook -from .util import icon_path, WindowModalDialog, OkButton, CancelButton, Buttons +from .util import (icon_path, WindowModalDialog, OkButton, CancelButton, Buttons, + PasswordLineEdit) def check_password_strength(password): @@ -63,12 +64,9 @@ class PasswordLayout(object): def __init__(self, msg, kind, OK_button, wallet=None, force_disable_encrypt_cb=False): self.wallet = wallet - self.pw = QLineEdit() - self.pw.setEchoMode(2) - self.new_pw = QLineEdit() - self.new_pw.setEchoMode(2) - self.conf_pw = QLineEdit() - self.conf_pw.setEchoMode(2) + self.pw = PasswordLineEdit() + self.new_pw = PasswordLineEdit() + self.conf_pw = PasswordLineEdit() self.kind = kind self.OK_button = OK_button @@ -290,8 +288,7 @@ class PasswordDialog(WindowModalDialog): def __init__(self, parent=None, msg=None): msg = msg or _('Please enter your password') WindowModalDialog.__init__(self, parent, _("Enter Password")) - self.pw = pw = QLineEdit() - pw.setEchoMode(2) + self.pw = pw = PasswordLineEdit() vbox = QVBoxLayout() vbox.addWidget(QLabel(msg)) grid = QGridLayout() diff --git a/electrum/gui/qt/util.py b/electrum/gui/qt/util.py index 2229c0bca8..6b8bce4f50 100644 --- a/electrum/gui/qt/util.py +++ b/electrum/gui/qt/util.py @@ -748,6 +748,12 @@ def resizeEvent(self, e): return o +class PasswordLineEdit(QLineEdit): + def __init__(self, *args, **kwargs): + QLineEdit.__init__(self, *args, **kwargs) + self.setEchoMode(QLineEdit.Password) + + class TaskThread(QThread): '''Thread that runs background tasks. Callbacks are guaranteed to happen in the context of its parent.''' diff --git a/electrum/plugins/hw_wallet/qt.py b/electrum/plugins/hw_wallet/qt.py index c8432f29e2..b501a4a4f7 100644 --- a/electrum/plugins/hw_wallet/qt.py +++ b/electrum/plugins/hw_wallet/qt.py @@ -33,7 +33,8 @@ from electrum.gui.qt.password_dialog import PasswordLayout, PW_PASSPHRASE from electrum.gui.qt.util import (read_QIcon, WWLabel, OkButton, WindowModalDialog, - Buttons, CancelButton, TaskThread, char_width_in_lineedit) + Buttons, CancelButton, TaskThread, char_width_in_lineedit, + PasswordLineEdit) from electrum.gui.qt.main_window import StatusBarButton, ElectrumWindow from electrum.gui.qt.installwizard import InstallWizard @@ -142,8 +143,7 @@ def passphrase_dialog(self, msg, confirm): d.setLayout(vbox) passphrase = playout.new_password() if d.exec_() else None else: - pw = QLineEdit() - pw.setEchoMode(2) + pw = PasswordLineEdit() pw.setMinimumWidth(200) vbox = QVBoxLayout() vbox.addWidget(WWLabel(msg)) diff --git a/electrum/plugins/ledger/auth2fa.py b/electrum/plugins/ledger/auth2fa.py index ca34702788..8f6b55f260 100644 --- a/electrum/plugins/ledger/auth2fa.py +++ b/electrum/plugins/ledger/auth2fa.py @@ -5,6 +5,8 @@ from btchip.btchip import BTChipException +from electrum.gui.qt.util import PasswordLineEdit + from electrum.i18n import _ from electrum import constants, bitcoin from electrum.logging import get_logger @@ -79,8 +81,7 @@ def return_pin(): self.pinbox = QWidget() pinlayout = QHBoxLayout() self.pinbox.setLayout(pinlayout) - self.pintxt = QLineEdit() - self.pintxt.setEchoMode(2) + self.pintxt = PasswordLineEdit() self.pintxt.setMaxLength(4) self.pintxt.returnPressed.connect(return_pin) pinlayout.addWidget(QLabel(_("Enter PIN:"))) @@ -121,8 +122,7 @@ def pin_changed(s): pin_changed('') cardpin = QHBoxLayout() cardpin.addWidget(QLabel(_("Enter PIN:"))) - self.cardtxt = QLineEdit() - self.cardtxt.setEchoMode(2) + self.cardtxt = PasswordLineEdit() self.cardtxt.setMaxLength(len(self.idxs)) self.cardtxt.textChanged.connect(pin_changed) self.cardtxt.returnPressed.connect(return_pin) diff --git a/electrum/plugins/trezor/qt.py b/electrum/plugins/trezor/qt.py index ba0ccc5b3e..f6d252da63 100644 --- a/electrum/plugins/trezor/qt.py +++ b/electrum/plugins/trezor/qt.py @@ -8,7 +8,7 @@ QMessageBox, QFileDialog, QSlider, QTabWidget) from electrum.gui.qt.util import (WindowModalDialog, WWLabel, Buttons, CancelButton, - OkButton, CloseButton) + OkButton, CloseButton, PasswordLineEdit) from electrum.i18n import _ from electrum.plugin import hook from electrum.util import bh2u @@ -172,10 +172,8 @@ def passphrase_dialog(self, msg, confirm): OK_button = OkButton(d, _('Enter Passphrase')) OnDevice_button = QPushButton(_('Enter Passphrase on Device')) - new_pw = QLineEdit() - new_pw.setEchoMode(2) - conf_pw = QLineEdit() - conf_pw.setEchoMode(2) + new_pw = PasswordLineEdit() + conf_pw = PasswordLineEdit() vbox = QVBoxLayout() label = QLabel(msg + "\n") From 5259fcb6fd0cbb2ae77c55dc4efa1ca37350baab Mon Sep 17 00:00:00 2001 From: SomberNight Date: Tue, 7 Apr 2020 18:04:04 +0200 Subject: [PATCH 010/117] qt PasswordLineEdit: try to clear password from memory If an attacker has access to the process' memory, it's probably already game over, still we can make their life a bit harder. I really tried but failed to encapsulate this logic inside PasswordLineEdit. The destroyed signal arrives too late. deleteLater is not called. __del__ gets called too late. --- electrum/gui/qt/installwizard.py | 101 ++++++++++++++++------------- electrum/gui/qt/password_dialog.py | 23 +++++-- electrum/gui/qt/util.py | 6 ++ 3 files changed, 78 insertions(+), 52 deletions(-) diff --git a/electrum/gui/qt/installwizard.py b/electrum/gui/qt/installwizard.py index 834c8fb4ea..f94037b166 100644 --- a/electrum/gui/qt/installwizard.py +++ b/electrum/gui/qt/installwizard.py @@ -281,51 +281,57 @@ def on_filename(filename): name_e.textChanged.connect(on_filename) name_e.setText(os.path.basename(path)) - while True: - if self.loop.exec_() != 2: # 2 = next - raise UserCancelled - assert temp_storage - if temp_storage.file_exists() and not temp_storage.is_encrypted(): - break - if not temp_storage.file_exists(): - break - wallet_from_memory = get_wallet_from_daemon(temp_storage.path) - if wallet_from_memory: - raise WalletAlreadyOpenInMemory(wallet_from_memory) - if temp_storage.file_exists() and temp_storage.is_encrypted(): - if temp_storage.is_encrypted_with_user_pw(): - password = pw_e.text() - try: - temp_storage.decrypt(password) - break - except InvalidPassword as e: - self.show_message(title=_('Error'), msg=str(e)) - continue - except BaseException as e: - self.logger.exception('') - self.show_message(title=_('Error'), msg=repr(e)) - raise UserCancelled() - elif temp_storage.is_encrypted_with_hw_device(): - try: - self.run('choose_hw_device', HWD_SETUP_DECRYPT_WALLET, storage=temp_storage) - except InvalidPassword as e: - self.show_message(title=_('Error'), - msg=_('Failed to decrypt using this hardware device.') + '\n' + - _('If you use a passphrase, make sure it is correct.')) - self.reset_stack() - return self.select_storage(path, get_wallet_from_daemon) - except (UserCancelled, GoBack): - raise - except BaseException as e: - self.logger.exception('') - self.show_message(title=_('Error'), msg=repr(e)) - raise UserCancelled() - if temp_storage.is_past_initial_decryption(): - break + def run_user_interaction_loop(): + while True: + if self.loop.exec_() != 2: # 2 = next + raise UserCancelled + assert temp_storage + if temp_storage.file_exists() and not temp_storage.is_encrypted(): + break + if not temp_storage.file_exists(): + break + wallet_from_memory = get_wallet_from_daemon(temp_storage.path) + if wallet_from_memory: + raise WalletAlreadyOpenInMemory(wallet_from_memory) + if temp_storage.file_exists() and temp_storage.is_encrypted(): + if temp_storage.is_encrypted_with_user_pw(): + password = pw_e.text() + try: + temp_storage.decrypt(password) + break + except InvalidPassword as e: + self.show_message(title=_('Error'), msg=str(e)) + continue + except BaseException as e: + self.logger.exception('') + self.show_message(title=_('Error'), msg=repr(e)) + raise UserCancelled() + elif temp_storage.is_encrypted_with_hw_device(): + try: + self.run('choose_hw_device', HWD_SETUP_DECRYPT_WALLET, storage=temp_storage) + except InvalidPassword as e: + self.show_message(title=_('Error'), + msg=_('Failed to decrypt using this hardware device.') + '\n' + + _('If you use a passphrase, make sure it is correct.')) + self.reset_stack() + return self.select_storage(path, get_wallet_from_daemon) + except (UserCancelled, GoBack): + raise + except BaseException as e: + self.logger.exception('') + self.show_message(title=_('Error'), msg=repr(e)) + raise UserCancelled() + if temp_storage.is_past_initial_decryption(): + break + else: + raise UserCancelled() else: - raise UserCancelled() - else: - raise Exception('Unexpected encryption version') + raise Exception('Unexpected encryption version') + + try: + run_user_interaction_loop() + finally: + pw_e.clear() return temp_storage.path, (temp_storage if temp_storage.file_exists() else None) @@ -482,8 +488,11 @@ def pw_layout(self, msg, kind, force_disable_encrypt_cb): playout = PasswordLayout(msg=msg, kind=kind, OK_button=self.next_button, force_disable_encrypt_cb=force_disable_encrypt_cb) playout.encrypt_cb.setChecked(True) - self.exec_layout(playout.layout()) - return playout.new_password(), playout.encrypt_cb.isChecked() + try: + self.exec_layout(playout.layout()) + return playout.new_password(), playout.encrypt_cb.isChecked() + finally: + playout.clear_password_fields() @wizard_dialog def request_password(self, run_next, force_disable_encrypt_cb=False): diff --git a/electrum/gui/qt/password_dialog.py b/electrum/gui/qt/password_dialog.py index f35c479dab..e998c34166 100644 --- a/electrum/gui/qt/password_dialog.py +++ b/electrum/gui/qt/password_dialog.py @@ -25,6 +25,7 @@ import re import math +from functools import partial from PyQt5.QtCore import Qt from PyQt5.QtGui import QPixmap @@ -165,6 +166,10 @@ def new_password(self): pw = None return pw + def clear_password_fields(self): + for field in [self.pw, self.new_pw, self.conf_pw]: + field.clear() + class PasswordLayoutForHW(object): @@ -258,9 +263,12 @@ def create_password_layout(self, wallet, is_encrypted, OK_button): force_disable_encrypt_cb=not wallet.can_have_keystore_encryption()) def run(self): - if not self.exec_(): - return False, None, None, None - return True, self.playout.old_password(), self.playout.new_password(), self.playout.encrypt_cb.isChecked() + try: + if not self.exec_(): + return False, None, None, None + return True, self.playout.old_password(), self.playout.new_password(), self.playout.encrypt_cb.isChecked() + finally: + self.playout.clear_password_fields() class ChangePasswordDialogForHW(ChangePasswordDialogBase): @@ -301,6 +309,9 @@ def __init__(self, parent=None, msg=None): run_hook('password_dialog', pw, grid, 1) def run(self): - if not self.exec_(): - return - return self.pw.text() + try: + if not self.exec_(): + return + return self.pw.text() + finally: + self.pw.clear() diff --git a/electrum/gui/qt/util.py b/electrum/gui/qt/util.py index 6b8bce4f50..96a7ed08fc 100644 --- a/electrum/gui/qt/util.py +++ b/electrum/gui/qt/util.py @@ -753,6 +753,12 @@ def __init__(self, *args, **kwargs): QLineEdit.__init__(self, *args, **kwargs) self.setEchoMode(QLineEdit.Password) + def clear(self): + # Try to actually overwrite the memory. + # This is really just a best-effort thing... + self.setText(len(self.text()) * " ") + super().clear() + class TaskThread(QThread): '''Thread that runs background tasks. Callbacks are guaranteed From caefea19dd5ffb856b2d11b855e98a68ad517608 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Tue, 7 Apr 2020 18:56:14 +0200 Subject: [PATCH 011/117] trezor pin dialog: only show PIN "strength" when creating/changing fixes #4832 --- electrum/plugins/coldcard/cmdline.py | 2 +- electrum/plugins/hw_wallet/cmdline.py | 2 +- electrum/plugins/hw_wallet/plugin.py | 3 +++ electrum/plugins/keepkey/clientbase.py | 8 ++++++-- electrum/plugins/keepkey/qt.py | 10 +++++----- electrum/plugins/safe_t/clientbase.py | 8 ++++++-- electrum/plugins/safe_t/qt.py | 10 +++++----- electrum/plugins/trezor/clientbase.py | 4 +++- electrum/plugins/trezor/qt.py | 10 +++++----- 9 files changed, 35 insertions(+), 22 deletions(-) diff --git a/electrum/plugins/coldcard/cmdline.py b/electrum/plugins/coldcard/cmdline.py index 7df86f1f29..ab86f463c6 100644 --- a/electrum/plugins/coldcard/cmdline.py +++ b/electrum/plugins/coldcard/cmdline.py @@ -15,7 +15,7 @@ class ColdcardCmdLineHandler(CmdLineHandler): def get_passphrase(self, msg, confirm): raise NotImplementedError - def get_pin(self, msg): + def get_pin(self, msg, *, show_strength=True): raise NotImplementedError def prompt_auth(self, msg): diff --git a/electrum/plugins/hw_wallet/cmdline.py b/electrum/plugins/hw_wallet/cmdline.py index 5210267f11..b754fb4e0a 100644 --- a/electrum/plugins/hw_wallet/cmdline.py +++ b/electrum/plugins/hw_wallet/cmdline.py @@ -14,7 +14,7 @@ def get_passphrase(self, msg, confirm): print_stderr(msg) return getpass.getpass('') - def get_pin(self, msg): + def get_pin(self, msg, *, show_strength=True): t = { 'a':'7', 'b':'8', 'c':'9', 'd':'4', 'e':'5', 'f':'6', 'g':'1', 'h':'2', 'i':'3'} print_stderr(msg) print_stderr("a b c\nd e f\ng h i\n-----") diff --git a/electrum/plugins/hw_wallet/plugin.py b/electrum/plugins/hw_wallet/plugin.py index f1e28c6eb5..d5280802c6 100644 --- a/electrum/plugins/hw_wallet/plugin.py +++ b/electrum/plugins/hw_wallet/plugin.py @@ -251,6 +251,9 @@ def get_word(self, msg: str) -> str: def get_passphrase(self, msg: str, confirm: bool) -> Optional[str]: raise NotImplementedError() + def get_pin(self, msg: str, *, show_strength: bool = True) -> str: + raise NotImplementedError() + def is_any_tx_output_on_change_branch(tx: PartialTransaction) -> bool: return any([txout.is_change for txout in tx.outputs()]) diff --git a/electrum/plugins/keepkey/clientbase.py b/electrum/plugins/keepkey/clientbase.py index 8df1ac2505..6b71ca3204 100644 --- a/electrum/plugins/keepkey/clientbase.py +++ b/electrum/plugins/keepkey/clientbase.py @@ -1,5 +1,6 @@ import time from struct import pack +from typing import Optional from electrum import ecc from electrum.i18n import _ @@ -7,11 +8,12 @@ from electrum.keystore import bip39_normalize_passphrase from electrum.bip32 import BIP32Node, convert_bip32_path_to_list_of_uint32 from electrum.logging import Logger -from electrum.plugins.hw_wallet.plugin import HardwareClientBase +from electrum.plugins.hw_wallet.plugin import HardwareClientBase, HardwareHandlerBase class GuiMixin(object): # Requires: self.proto, self.device + handler: Optional[HardwareHandlerBase] messages = { 3: _("Confirm the transaction output on your {} device"), @@ -45,6 +47,7 @@ def callback_ButtonRequest(self, msg): return self.proto.ButtonAck() def callback_PinMatrixRequest(self, msg): + show_strength = True if msg.type == 2: msg = _("Enter a new PIN for your {}:") elif msg.type == 3: @@ -52,7 +55,8 @@ def callback_PinMatrixRequest(self, msg): "NOTE: the positions of the numbers have changed!")) else: msg = _("Enter your current {} PIN:") - pin = self.handler.get_pin(msg.format(self.device)) + show_strength = False + pin = self.handler.get_pin(msg.format(self.device), show_strength=show_strength) if len(pin) > 9: self.handler.show_error(_('The PIN cannot be longer than 9 characters.')) pin = '' # to cancel below diff --git a/electrum/plugins/keepkey/qt.py b/electrum/plugins/keepkey/qt.py index 72508ee73d..e70d430259 100644 --- a/electrum/plugins/keepkey/qt.py +++ b/electrum/plugins/keepkey/qt.py @@ -137,7 +137,7 @@ def get_char(self, word_pos, character_pos): class QtHandler(QtHandlerBase): char_signal = pyqtSignal(object) - pin_signal = pyqtSignal(object) + pin_signal = pyqtSignal(object, object) close_char_dialog_signal = pyqtSignal() def __init__(self, win, pin_matrix_widget_class, device): @@ -162,17 +162,17 @@ def _close_char_dialog(self): self.character_dialog.accept() self.character_dialog = None - def get_pin(self, msg): + def get_pin(self, msg, *, show_strength=True): self.done.clear() - self.pin_signal.emit(msg) + self.pin_signal.emit(msg, show_strength) self.done.wait() return self.response - def pin_dialog(self, msg): + def pin_dialog(self, msg, show_strength): # Needed e.g. when resetting a device self.clear_dialog() dialog = WindowModalDialog(self.top_level_window(), _("Enter PIN")) - matrix = self.pin_matrix_widget_class() + matrix = self.pin_matrix_widget_class(show_strength) vbox = QVBoxLayout() vbox.addWidget(QLabel(msg)) vbox.addWidget(matrix) diff --git a/electrum/plugins/safe_t/clientbase.py b/electrum/plugins/safe_t/clientbase.py index a6f733aaf0..8ff6662858 100644 --- a/electrum/plugins/safe_t/clientbase.py +++ b/electrum/plugins/safe_t/clientbase.py @@ -1,5 +1,6 @@ import time from struct import pack +from typing import Optional from electrum import ecc from electrum.i18n import _ @@ -7,11 +8,12 @@ from electrum.keystore import bip39_normalize_passphrase from electrum.bip32 import BIP32Node, convert_bip32_path_to_list_of_uint32 from electrum.logging import Logger -from electrum.plugins.hw_wallet.plugin import HardwareClientBase +from electrum.plugins.hw_wallet.plugin import HardwareClientBase, HardwareHandlerBase class GuiMixin(object): # Requires: self.proto, self.device + handler: Optional[HardwareHandlerBase] # ref: https://github.com/trezor/trezor-common/blob/44dfb07cfaafffada4b2ce0d15ba1d90d17cf35e/protob/types.proto#L89 messages = { @@ -47,6 +49,7 @@ def callback_ButtonRequest(self, msg): return self.proto.ButtonAck() def callback_PinMatrixRequest(self, msg): + show_strength = True if msg.type == 2: msg = _("Enter a new PIN for your {}:") elif msg.type == 3: @@ -54,7 +57,8 @@ def callback_PinMatrixRequest(self, msg): "NOTE: the positions of the numbers have changed!")) else: msg = _("Enter your current {} PIN:") - pin = self.handler.get_pin(msg.format(self.device)) + show_strength = False + pin = self.handler.get_pin(msg.format(self.device), show_strength=show_strength) if len(pin) > 9: self.handler.show_error(_('The PIN cannot be longer than 9 characters.')) pin = '' # to cancel below diff --git a/electrum/plugins/safe_t/qt.py b/electrum/plugins/safe_t/qt.py index d83663d537..6d3e9399ef 100644 --- a/electrum/plugins/safe_t/qt.py +++ b/electrum/plugins/safe_t/qt.py @@ -38,24 +38,24 @@ class QtHandler(QtHandlerBase): - pin_signal = pyqtSignal(object) + pin_signal = pyqtSignal(object, object) def __init__(self, win, pin_matrix_widget_class, device): super(QtHandler, self).__init__(win, device) self.pin_signal.connect(self.pin_dialog) self.pin_matrix_widget_class = pin_matrix_widget_class - def get_pin(self, msg): + def get_pin(self, msg, *, show_strength=True): self.done.clear() - self.pin_signal.emit(msg) + self.pin_signal.emit(msg, show_strength) self.done.wait() return self.response - def pin_dialog(self, msg): + def pin_dialog(self, msg, show_strength): # Needed e.g. when resetting a device self.clear_dialog() dialog = WindowModalDialog(self.top_level_window(), _("Enter PIN")) - matrix = self.pin_matrix_widget_class() + matrix = self.pin_matrix_widget_class(show_strength) vbox = QVBoxLayout() vbox.addWidget(QLabel(msg)) vbox.addWidget(matrix) diff --git a/electrum/plugins/trezor/clientbase.py b/electrum/plugins/trezor/clientbase.py index 430905b0a0..101f927b69 100644 --- a/electrum/plugins/trezor/clientbase.py +++ b/electrum/plugins/trezor/clientbase.py @@ -246,6 +246,7 @@ def button_request(self, code): self.handler.show_message(message.format(self.device), self.client.cancel) def get_pin(self, code=None): + show_strength = True if code == 2: msg = _("Enter a new PIN for your {}:") elif code == 3: @@ -253,7 +254,8 @@ def get_pin(self, code=None): "NOTE: the positions of the numbers have changed!")) else: msg = _("Enter your current {} PIN:") - pin = self.handler.get_pin(msg.format(self.device)) + show_strength = False + pin = self.handler.get_pin(msg.format(self.device), show_strength=show_strength) if not pin: raise Cancelled if len(pin) > 9: diff --git a/electrum/plugins/trezor/qt.py b/electrum/plugins/trezor/qt.py index f6d252da63..bf8911c056 100644 --- a/electrum/plugins/trezor/qt.py +++ b/electrum/plugins/trezor/qt.py @@ -108,7 +108,7 @@ def get_matrix(self, num): class QtHandler(QtHandlerBase): - pin_signal = pyqtSignal(object) + pin_signal = pyqtSignal(object, object) matrix_signal = pyqtSignal(object) close_matrix_dialog_signal = pyqtSignal() @@ -121,9 +121,9 @@ def __init__(self, win, pin_matrix_widget_class, device): self.matrix_dialog = None self.passphrase_on_device = False - def get_pin(self, msg): + def get_pin(self, msg, *, show_strength=True): self.done.clear() - self.pin_signal.emit(msg) + self.pin_signal.emit(msg, show_strength) self.done.wait() return self.response @@ -144,11 +144,11 @@ def _close_matrix_dialog(self): def close_matrix_dialog(self): self.close_matrix_dialog_signal.emit() - def pin_dialog(self, msg): + def pin_dialog(self, msg, show_strength): # Needed e.g. when resetting a device self.clear_dialog() dialog = WindowModalDialog(self.top_level_window(), _("Enter PIN")) - matrix = self.pin_matrix_widget_class() + matrix = self.pin_matrix_widget_class(show_strength) vbox = QVBoxLayout() vbox.addWidget(QLabel(msg)) vbox.addWidget(matrix) From 40389a21b6c87046175f792b93782e5cccbddb19 Mon Sep 17 00:00:00 2001 From: JeremyRand Date: Wed, 8 Apr 2020 03:09:08 +0000 Subject: [PATCH 012/117] Use specific Exception when chain isn't synced Makes it easier for calling code to know what error happened. --- electrum/commands.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/electrum/commands.py b/electrum/commands.py index 2cf1ab00db..02fb82b380 100644 --- a/electrum/commands.py +++ b/electrum/commands.py @@ -68,6 +68,10 @@ known_commands = {} # type: Dict[str, Command] +class NotSynchronizedException(Exception): + pass + + def satoshis(amount): # satoshi conversion must not be performed by the parser return int(COIN*Decimal(amount)) if amount not in ['!', None] else amount @@ -815,7 +819,7 @@ async def getminacceptablegap(self, wallet: Abstract_Wallet = None): if not isinstance(wallet, Deterministic_Wallet): raise Exception("This wallet is not deterministic.") if not wallet.is_up_to_date(): - raise Exception("Wallet not fully synchronized.") + raise NotSynchronizedException("Wallet not fully synchronized.") return wallet.min_acceptable_gap() @command('w') From 6307e13549486f31ac22e65f1a79b83e8dcba66d Mon Sep 17 00:00:00 2001 From: ThomasV Date: Wed, 8 Apr 2020 09:46:16 +0200 Subject: [PATCH 013/117] do not print the entire payment log again, this is redundant --- electrum/lnworker.py | 1 - 1 file changed, 1 deletion(-) diff --git a/electrum/lnworker.py b/electrum/lnworker.py index 78a24a31d6..bde0cdd65d 100644 --- a/electrum/lnworker.py +++ b/electrum/lnworker.py @@ -832,7 +832,6 @@ async def _pay(self, invoice, amount_sat=None, attempts=1) -> bool: self.network.trigger_callback('payment_succeeded', key) else: self.network.trigger_callback('payment_failed', key, reason) - self.logger.debug(f'payment attempts log for RHASH {key}: {repr(log)}') return success async def _pay_to_route(self, route: LNPaymentRoute, lnaddr: LnAddr) -> PaymentAttemptLog: From 789b78cab5218630b0c8e1c16494611860f101d4 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Wed, 8 Apr 2020 12:38:38 +0200 Subject: [PATCH 014/117] crypto: trivial clean-up of pw_encode/pw_decode functions --- electrum/crypto.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/electrum/crypto.py b/electrum/crypto.py index 58f66c2d1f..40101200e3 100644 --- a/electrum/crypto.py +++ b/electrum/crypto.py @@ -190,6 +190,7 @@ def _hash_password(password: Union[bytes, str], *, version: int) -> bytes: def pw_encode_bytes(data: bytes, password: Union[bytes, str], *, version: int) -> str: + """plaintext bytes -> base64 ciphertext""" if version not in KNOWN_PW_HASH_VERSIONS: raise UnexpectedPasswordHashVersion(version) # derive key from password @@ -199,7 +200,9 @@ def pw_encode_bytes(data: bytes, password: Union[bytes, str], *, version: int) - ciphertext_b64 = base64.b64encode(ciphertext) return ciphertext_b64.decode('utf8') + def pw_decode_bytes(data: str, password: Union[bytes, str], *, version: int) -> bytes: + """base64 ciphertext -> plaintext bytes""" if version not in KNOWN_PW_HASH_VERSIONS: raise UnexpectedPasswordHashVersion(version) data_bytes = bytes(base64.b64decode(data)) @@ -212,15 +215,22 @@ def pw_decode_bytes(data: str, password: Union[bytes, str], *, version: int) -> raise InvalidPassword() from e return d + def pw_encode(data: str, password: Union[bytes, str, None], *, version: int) -> str: + """plaintext str -> base64 ciphertext""" if not password: return data - return pw_encode_bytes(to_bytes(data, "utf8"), password, version=version) + plaintext_bytes = to_bytes(data, "utf8") + return pw_encode_bytes(plaintext_bytes, password, version=version) + def pw_decode(data: str, password: Union[bytes, str, None], *, version: int) -> str: + """base64 ciphertext -> plaintext str""" if password is None: return data - return to_string(pw_decode_bytes(data, password, version=version), "utf8") + plaintext_bytes = pw_decode_bytes(data, password, version=version) + plaintext_str = to_string(plaintext_bytes, "utf8") + return plaintext_str def sha256(x: Union[bytes, str]) -> bytes: From 1ea89af0129aa00aff8ed05c09d2712c275719bc Mon Sep 17 00:00:00 2001 From: SomberNight Date: Wed, 8 Apr 2020 12:49:50 +0200 Subject: [PATCH 015/117] crypto.pw_decode: fix one case of raising incorrect exception --- electrum/crypto.py | 5 ++++- electrum/tests/test_bitcoin.py | 5 +++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/electrum/crypto.py b/electrum/crypto.py index 40101200e3..62f7b91485 100644 --- a/electrum/crypto.py +++ b/electrum/crypto.py @@ -229,7 +229,10 @@ def pw_decode(data: str, password: Union[bytes, str, None], *, version: int) -> if password is None: return data plaintext_bytes = pw_decode_bytes(data, password, version=version) - plaintext_str = to_string(plaintext_bytes, "utf8") + try: + plaintext_str = to_string(plaintext_bytes, "utf8") + except UnicodeDecodeError as e: + raise InvalidPassword() from e return plaintext_str diff --git a/electrum/tests/test_bitcoin.py b/electrum/tests/test_bitcoin.py index 1cca2ab33e..35d7770c4f 100644 --- a/electrum/tests/test_bitcoin.py +++ b/electrum/tests/test_bitcoin.py @@ -254,6 +254,11 @@ def test_aes_decode_with_invalid_password(self): enc = crypto.pw_encode(payload, password, version=version) with self.assertRaises(InvalidPassword): crypto.pw_decode(enc, wrong_password, version=version) + # sometimes the PKCS7 padding gets removed cleanly, + # but then UnicodeDecodeError gets raised (internally): + enc = 'smJ7j6ccr8LnMOlx98s/ajgikv9s3R1PQuG3GyyIMmo=' + with self.assertRaises(InvalidPassword): + crypto.pw_decode(enc, wrong_password, version=1) @needs_test_with_all_chacha20_implementations def test_chacha20_poly1305_encrypt(self): From 7dabbdd08234ae036177d056a513a2189e9f14e1 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Wed, 8 Apr 2020 13:18:56 +0200 Subject: [PATCH 016/117] tests_lnpeer: trivial fix --- electrum/tests/test_lnpeer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/electrum/tests/test_lnpeer.py b/electrum/tests/test_lnpeer.py index 4089ef25b8..82b6a85474 100644 --- a/electrum/tests/test_lnpeer.py +++ b/electrum/tests/test_lnpeer.py @@ -75,7 +75,7 @@ async def broadcast_transaction(self, tx): await self.tx_queue.put(tx) async def try_broadcasting(self, tx, name): - self.broadcast_transaction(tx) + await self.broadcast_transaction(tx) class MockWallet: def set_label(self, x, y): From 9d0bb295e6f55a2bff9f5b6770fa744c16af6e8a Mon Sep 17 00:00:00 2001 From: SomberNight Date: Wed, 8 Apr 2020 14:43:01 +0200 Subject: [PATCH 017/117] hww: distinguish devices based on "soft device id" (not just labels) fixes #5759 --- electrum/base_wizard.py | 4 +++- electrum/keystore.py | 5 +++++ electrum/plugin.py | 14 +++++++++++--- electrum/plugins/hw_wallet/plugin.py | 12 +++++++++++- electrum/plugins/keepkey/clientbase.py | 3 +++ electrum/plugins/ledger/ledger.py | 12 +++++++++++- electrum/plugins/safe_t/clientbase.py | 3 +++ electrum/plugins/trezor/clientbase.py | 3 +++ 8 files changed, 50 insertions(+), 6 deletions(-) diff --git a/electrum/base_wizard.py b/electrum/base_wizard.py index 77230839af..d6a2b20bf9 100644 --- a/electrum/base_wizard.py +++ b/electrum/base_wizard.py @@ -438,6 +438,7 @@ def on_hw_derivation(self, name, device_info: 'DeviceInfo', derivation, xtype): if not client: raise Exception("failed to find client for device id") root_fingerprint = client.request_root_fingerprint_from_device() label = client.label() # use this as device_info.label might be outdated! + soft_device_id = client.get_soft_device_id() # use this as device_info.device_id might be outdated! except ScriptTypeNotSupported: raise # this is handled in derivation_dialog except BaseException as e: @@ -451,6 +452,7 @@ def on_hw_derivation(self, name, device_info: 'DeviceInfo', derivation, xtype): 'root_fingerprint': root_fingerprint, 'xpub': xpub, 'label': label, + 'soft_device_id': soft_device_id, } k = hardware_keystore(d) self.on_keystore(k) @@ -612,7 +614,7 @@ def create_storage(self, path): if os.path.exists(path): raise Exception('file already exists at path') if not self.pw_args: - return + return # FIXME pw_args = self.pw_args self.pw_args = None # clean-up so that it can get GC-ed storage = WalletStorage(path) diff --git a/electrum/keystore.py b/electrum/keystore.py index d82d3f260b..4f1fb9090d 100644 --- a/electrum/keystore.py +++ b/electrum/keystore.py @@ -724,6 +724,7 @@ def __init__(self, d): # device reconnects self.xpub = d.get('xpub') self.label = d.get('label') + self.soft_device_id = d.get('soft_device_id') # type: Optional[str] self.handler = None # type: Optional[HardwareHandlerBase] run_hook('init_keystore', self) @@ -747,6 +748,7 @@ def dump(self): 'derivation': self.get_derivation_prefix(), 'root_fingerprint': self.get_root_fingerprint(), 'label':self.label, + 'soft_device_id': self.soft_device_id, } def unpaired(self): @@ -788,6 +790,9 @@ def opportunistically_fill_in_missing_info_from_device(self, client: 'HardwareCl if self.label != client.label(): self.label = client.label() self.is_requesting_to_be_rewritten_to_wallet_file = True + if self.soft_device_id != client.get_soft_device_id(): + self.soft_device_id = client.get_soft_device_id() + self.is_requesting_to_be_rewritten_to_wallet_file = True KeyStoreWithMPK = Union[KeyStore, MasterPublicKeyMixin] # intersection really... diff --git a/electrum/plugin.py b/electrum/plugin.py index dbebf3a9f1..bf775187ab 100644 --- a/electrum/plugin.py +++ b/electrum/plugin.py @@ -306,6 +306,7 @@ class DeviceInfo(NamedTuple): initialized: Optional[bool] = None exception: Optional[Exception] = None plugin_name: Optional[str] = None # manufacturer, e.g. "trezor" + soft_device_id: Optional[str] = None # if available, used to distinguish same-type hw devices class HardwarePluginToScan(NamedTuple): @@ -548,7 +549,8 @@ def unpaired_device_infos(self, handler: Optional['HardwareHandlerBase'], plugin infos.append(DeviceInfo(device=device, label=client.label(), initialized=client.is_initialized(), - plugin_name=plugin.name)) + plugin_name=plugin.name, + soft_device_id=client.get_soft_device_id())) return infos @@ -575,6 +577,11 @@ def select_device(self, plugin: 'HW_PluginBase', handler: 'HardwareHandlerBase', devices = None if len(infos) == 1: return infos[0] + # select device by id + if keystore.soft_device_id: + for info in infos: + if info.soft_device_id == keystore.soft_device_id: + return info # select device by label automatically; # but only if not a placeholder label and only if there is no collision device_labels = [info.label for info in infos] @@ -583,7 +590,7 @@ def select_device(self, plugin: 'HW_PluginBase', handler: 'HardwareHandlerBase', for info in infos: if info.label == keystore.label: return info - # ask user to select device + # ask user to select device manually msg = _("Please select which {} device to use:").format(plugin.device) descriptions = ["{label} ({init}, {transport})" .format(label=info.label or _("An unnamed {}").format(info.plugin_name), @@ -594,8 +601,9 @@ def select_device(self, plugin: 'HW_PluginBase', handler: 'HardwareHandlerBase', if c is None: raise UserCancelled() info = infos[c] - # save new label + # save new label / soft_device_id keystore.set_label(info.label) + keystore.soft_device_id = info.soft_device_id wallet = handler.get_wallet() if wallet is not None: wallet.save_keystore() diff --git a/electrum/plugins/hw_wallet/plugin.py b/electrum/plugins/hw_wallet/plugin.py index d5280802c6..80f5b07f3c 100644 --- a/electrum/plugins/hw_wallet/plugin.py +++ b/electrum/plugins/hw_wallet/plugin.py @@ -167,7 +167,7 @@ def create_handler(self, window) -> 'HardwareHandlerBase': class HardwareClientBase: plugin: 'HW_PluginBase' - handler: Optional['HardwareHandlerBase'] + handler = None # type: Optional['HardwareHandlerBase'] def is_pairable(self) -> bool: raise NotImplementedError() @@ -191,6 +191,16 @@ def label(self) -> Optional[str]: """ raise NotImplementedError() + def get_soft_device_id(self) -> Optional[str]: + """An id-like string that is used to distinguish devices programmatically. + This is a long term id for the device, that does not change between reconnects. + This method should not prompt the user, i.e. no user interaction, as it is used + during USB device enumeration (called for each unpaired device). + Stored in the wallet file. + """ + # This functionality is optional. If not implemented just return None: + return None + def has_usable_connection_with_device(self) -> bool: raise NotImplementedError() diff --git a/electrum/plugins/keepkey/clientbase.py b/electrum/plugins/keepkey/clientbase.py index 6b71ca3204..92cae82078 100644 --- a/electrum/plugins/keepkey/clientbase.py +++ b/electrum/plugins/keepkey/clientbase.py @@ -119,6 +119,9 @@ def __str__(self): def label(self): return self.features.label + def get_soft_device_id(self): + return self.features.device_id + def is_initialized(self): return self.features.initialized diff --git a/electrum/plugins/ledger/ledger.py b/electrum/plugins/ledger/ledger.py index 8e0040f5a9..1866d735f7 100644 --- a/electrum/plugins/ledger/ledger.py +++ b/electrum/plugins/ledger/ledger.py @@ -66,6 +66,7 @@ def __init__(self, hidDevice, *, is_hw1: bool = False): self.dongleObject = btchip(hidDevice) self.preflightDone = False self._is_hw1 = is_hw1 + self._soft_device_id = None def is_pairable(self): return True @@ -82,6 +83,14 @@ def is_initialized(self): def label(self): return "" + def get_soft_device_id(self): + if self._soft_device_id is None: + # modern ledger can provide xpub without user interaction + # (hw1 would prompt for PIN) + if not self.is_hw1(): + self._soft_device_id = self.request_root_fingerprint_from_device() + return self._soft_device_id + def is_hw1(self) -> bool: return self._is_hw1 @@ -176,7 +185,8 @@ def perform_hw1_preflight(self): # Acquire the new client on the next run else: raise e - if self.has_detached_pin_support(self.dongleObject) and not self.is_pin_validated(self.dongleObject) and (self.handler is not None): + if self.has_detached_pin_support(self.dongleObject) and not self.is_pin_validated(self.dongleObject): + assert self.handler, "no handler for client" remaining_attempts = self.dongleObject.getVerifyPinRemainingAttempts() if remaining_attempts != 1: msg = "Enter your Ledger PIN - remaining attempts : " + str(remaining_attempts) diff --git a/electrum/plugins/safe_t/clientbase.py b/electrum/plugins/safe_t/clientbase.py index 8ff6662858..18bbdbab4b 100644 --- a/electrum/plugins/safe_t/clientbase.py +++ b/electrum/plugins/safe_t/clientbase.py @@ -121,6 +121,9 @@ def __str__(self): def label(self): return self.features.label + def get_soft_device_id(self): + return self.features.device_id + def is_initialized(self): return self.features.initialized diff --git a/electrum/plugins/trezor/clientbase.py b/electrum/plugins/trezor/clientbase.py index 101f927b69..19814b4c6f 100644 --- a/electrum/plugins/trezor/clientbase.py +++ b/electrum/plugins/trezor/clientbase.py @@ -98,6 +98,9 @@ def __str__(self): def label(self): return self.features.label + def get_soft_device_id(self): + return self.features.device_id + def is_initialized(self): return self.features.initialized From 4ef313a1acfb0be1045c9d81b5e02ae1e9be5c50 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Wed, 8 Apr 2020 16:39:46 +0200 Subject: [PATCH 018/117] hww: smarter auto-selection of which device to pair with scenario1: - 2of2 multisig wallet with trezor1 and trezor2 keystores - only trezor2 connected - previously we would pair first keystore with connected device and then display error. now we will pair the device with the correct keystore on the first try scenario2: - standard wallet with trezor1 keystore - trezor2 connected (different device) - previously we would pair trezor2 with the keystore and then display error. now we will prompt the user to select which device to pair with (out of one) related: #5789 --- electrum/plugin.py | 52 +++++++++++++------ electrum/plugins/coldcard/coldcard.py | 11 ++-- .../plugins/digitalbitbox/digitalbitbox.py | 9 ++-- electrum/plugins/hw_wallet/plugin.py | 11 +++- electrum/plugins/hw_wallet/qt.py | 35 +++++++++++-- electrum/plugins/keepkey/keepkey.py | 9 ++-- electrum/plugins/ledger/ledger.py | 9 ++-- electrum/plugins/safe_t/safe_t.py | 9 ++-- electrum/plugins/trezor/trezor.py | 9 ++-- 9 files changed, 106 insertions(+), 48 deletions(-) diff --git a/electrum/plugin.py b/electrum/plugin.py index bf775187ab..453f94c0bd 100644 --- a/electrum/plugin.py +++ b/electrum/plugin.py @@ -29,7 +29,7 @@ import threading import sys from typing import (NamedTuple, Any, Union, TYPE_CHECKING, Optional, Tuple, - Dict, Iterable, List) + Dict, Iterable, List, Sequence) from .i18n import _ from .util import (profiler, DaemonThread, UserCancelled, ThreadJob, UserFacingException) @@ -289,6 +289,7 @@ def settings_dialog(self): class DeviceUnpairableError(UserFacingException): pass class HardwarePluginLibraryUnavailable(Exception): pass +class CannotAutoSelectDevice(Exception): pass class Device(NamedTuple): @@ -460,19 +461,27 @@ def client_by_id(self, id_) -> Optional['HardwareClientBase']: @with_scan_lock def client_for_keystore(self, plugin: 'HW_PluginBase', handler: Optional['HardwareHandlerBase'], keystore: 'Hardware_KeyStore', - force_pair: bool) -> Optional['HardwareClientBase']: + force_pair: bool, *, + devices: Sequence['Device'] = None, + allow_user_interaction: bool = True) -> Optional['HardwareClientBase']: self.logger.info("getting client for keystore") if handler is None: raise Exception(_("Handler not found for") + ' ' + plugin.name + '\n' + _("A library is probably missing.")) handler.update_status(False) - devices = self.scan_devices() + if devices is None: + devices = self.scan_devices() xpub = keystore.xpub derivation = keystore.get_derivation_prefix() assert derivation is not None client = self.client_by_xpub(plugin, xpub, handler, devices) if client is None and force_pair: - info = self.select_device(plugin, handler, keystore, devices) - client = self.force_pair_xpub(plugin, handler, info, xpub, derivation) + try: + info = self.select_device(plugin, handler, keystore, devices, + allow_user_interaction=allow_user_interaction) + except CannotAutoSelectDevice: + pass + else: + client = self.force_pair_xpub(plugin, handler, info, xpub, derivation) if client: handler.update_status(True) if client: @@ -481,7 +490,7 @@ def client_for_keystore(self, plugin: 'HW_PluginBase', handler: Optional['Hardwa return client def client_by_xpub(self, plugin: 'HW_PluginBase', xpub, handler: 'HardwareHandlerBase', - devices: Iterable['Device']) -> Optional['HardwareClientBase']: + devices: Sequence['Device']) -> Optional['HardwareClientBase']: _id = self.xpub_id(xpub) client = self.client_lookup(_id) if client: @@ -523,7 +532,7 @@ def force_pair_xpub(self, plugin: 'HW_PluginBase', handler: 'HardwareHandlerBase 'receive will be unspendable.').format(plugin.device)) def unpaired_device_infos(self, handler: Optional['HardwareHandlerBase'], plugin: 'HW_PluginBase', - devices: List['Device'] = None, + devices: Sequence['Device'] = None, include_failing_clients=False) -> List['DeviceInfo']: '''Returns a list of DeviceInfo objects: one for each connected, unpaired device accepted by the plugin.''' @@ -555,15 +564,17 @@ def unpaired_device_infos(self, handler: Optional['HardwareHandlerBase'], plugin return infos def select_device(self, plugin: 'HW_PluginBase', handler: 'HardwareHandlerBase', - keystore: 'Hardware_KeyStore', devices: List['Device'] = None) -> 'DeviceInfo': - '''Ask the user to select a device to use if there is more than one, - and return the DeviceInfo for the device.''' + keystore: 'Hardware_KeyStore', devices: Sequence['Device'] = None, + *, allow_user_interaction: bool = True) -> 'DeviceInfo': + """Select the device to use for keystore.""" # ideally this should not be called from the GUI thread... # assert handler.get_gui_thread() != threading.current_thread(), 'must not be called from GUI thread' while True: infos = self.unpaired_device_infos(handler, plugin, devices) if infos: break + if not allow_user_interaction: + raise CannotAutoSelectDevice() msg = _('Please insert your {}').format(plugin.device) if keystore.label: msg += ' ({})'.format(keystore.label) @@ -575,21 +586,30 @@ def select_device(self, plugin: 'HW_PluginBase', handler: 'HardwareHandlerBase', if not handler.yes_no_question(msg): raise UserCancelled() devices = None - if len(infos) == 1: - return infos[0] - # select device by id + + # select device automatically. (but only if we have reasonable expectation it is the correct one) + # method 1: select device by id if keystore.soft_device_id: for info in infos: if info.soft_device_id == keystore.soft_device_id: return info - # select device by label automatically; - # but only if not a placeholder label and only if there is no collision + # method 2: select device by label + # but only if not a placeholder label and only if there is no collision device_labels = [info.label for info in infos] if (keystore.label not in PLACEHOLDER_HW_CLIENT_LABELS and device_labels.count(keystore.label) == 1): for info in infos: if info.label == keystore.label: return info + # method 3: if there is only one device connected, and we don't have useful label/soft_device_id + # saved for keystore anyway, select it + if (len(infos) == 1 + and keystore.label in PLACEHOLDER_HW_CLIENT_LABELS + and keystore.soft_device_id is None): + return infos[0] + + if not allow_user_interaction: + raise CannotAutoSelectDevice() # ask user to select device manually msg = _("Please select which {} device to use:").format(plugin.device) descriptions = ["{label} ({init}, {transport})" @@ -638,7 +658,7 @@ def _scan_devices_with_hid(self) -> List['Device']: return devices @with_scan_lock - def scan_devices(self) -> List['Device']: + def scan_devices(self) -> Sequence['Device']: self.logger.info("scanning devices...") # First see what's connected that we know about diff --git a/electrum/plugins/coldcard/coldcard.py b/electrum/plugins/coldcard/coldcard.py index 72bf07bf61..c0f34cfd77 100644 --- a/electrum/plugins/coldcard/coldcard.py +++ b/electrum/plugins/coldcard/coldcard.py @@ -4,7 +4,7 @@ # import os, time, io import traceback -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional import struct from electrum import bip32 @@ -536,11 +536,12 @@ def get_xpub(self, device_id, derivation, xtype, wizard): xpub = client.get_xpub(derivation, xtype) return xpub - def get_client(self, keystore, force_pair=True) -> 'CKCCClient': + def get_client(self, keystore, force_pair=True, *, + devices=None, allow_user_interaction=True) -> Optional['CKCCClient']: # Acquire a connection to the hardware device (via USB) - devmgr = self.device_manager() - handler = keystore.handler - client = devmgr.client_for_keystore(self, handler, keystore, force_pair) + client = super().get_client(keystore, force_pair, + devices=devices, + allow_user_interaction=allow_user_interaction) if client is not None: client.ping_check() diff --git a/electrum/plugins/digitalbitbox/digitalbitbox.py b/electrum/plugins/digitalbitbox/digitalbitbox.py index bf82942540..463a2f954d 100644 --- a/electrum/plugins/digitalbitbox/digitalbitbox.py +++ b/electrum/plugins/digitalbitbox/digitalbitbox.py @@ -741,10 +741,11 @@ def get_xpub(self, device_id, derivation, xtype, wizard): return xpub - def get_client(self, keystore, force_pair=True): - devmgr = self.device_manager() - handler = keystore.handler - client = devmgr.client_for_keystore(self, handler, keystore, force_pair) + def get_client(self, keystore, force_pair=True, *, + devices=None, allow_user_interaction=True): + client = super().get_client(keystore, force_pair, + devices=devices, + allow_user_interaction=allow_user_interaction) if client is not None: client.check_device_dialog() return client diff --git a/electrum/plugins/hw_wallet/plugin.py b/electrum/plugins/hw_wallet/plugin.py index 80f5b07f3c..4267e98def 100644 --- a/electrum/plugins/hw_wallet/plugin.py +++ b/electrum/plugins/hw_wallet/plugin.py @@ -83,8 +83,15 @@ def setup_device(self, device_info: DeviceInfo, wizard: 'BaseWizard', purpose): """ raise NotImplementedError() - def get_client(self, keystore: 'Hardware_KeyStore', force_pair: bool = True) -> Optional['HardwareClientBase']: - raise NotImplementedError() + def get_client(self, keystore: 'Hardware_KeyStore', force_pair: bool = True, *, + devices: Sequence['Device'] = None, + allow_user_interaction: bool = True) -> Optional['HardwareClientBase']: + devmgr = self.device_manager() + handler = keystore.handler + client = devmgr.client_for_keystore(self, handler, keystore, force_pair, + devices=devices, + allow_user_interaction=allow_user_interaction) + return client def show_address(self, wallet: 'Abstract_Wallet', address, keystore: 'Hardware_KeyStore' = None): pass # implemented in child classes diff --git a/electrum/plugins/hw_wallet/qt.py b/electrum/plugins/hw_wallet/qt.py index b501a4a4f7..56b8b62a2f 100644 --- a/electrum/plugins/hw_wallet/qt.py +++ b/electrum/plugins/hw_wallet/qt.py @@ -206,9 +206,11 @@ class QtPluginBase(object): @hook def load_wallet(self: Union['QtPluginBase', HW_PluginBase], wallet: 'Abstract_Wallet', window: ElectrumWindow): - for keystore in wallet.get_keystores(): - if not isinstance(keystore, self.keystore_class): - continue + relevant_keystores = [keystore for keystore in wallet.get_keystores() + if isinstance(keystore, self.keystore_class)] + if not relevant_keystores: + return + for keystore in relevant_keystores: if not self.libraries_available: message = keystore.plugin.get_library_not_available_message() window.show_error(message) @@ -224,8 +226,31 @@ def load_wallet(self: Union['QtPluginBase', HW_PluginBase], wallet: 'Abstract_Wa keystore.handler = handler keystore.thread = TaskThread(window, on_error=partial(self.on_task_thread_error, window, keystore)) self.add_show_address_on_hw_device_button_for_receive_addr(wallet, keystore, window) - # Trigger a pairing - keystore.thread.add(partial(self.get_client, keystore)) + # Trigger pairings + def trigger_pairings(): + devmgr = self.device_manager() + devices = devmgr.scan_devices() + # first pair with all devices that can be auto-selected + for keystore in relevant_keystores: + try: + self.get_client(keystore=keystore, + force_pair=True, + allow_user_interaction=False, + devices=devices) + except UserCancelled: + pass + # now do manual selections + for keystore in relevant_keystores: + try: + self.get_client(keystore=keystore, + force_pair=True, + allow_user_interaction=True, + devices=devices) + except UserCancelled: + pass + + some_keystore = relevant_keystores[0] + some_keystore.thread.add(trigger_pairings) def _on_status_bar_button_click(self, *, window: ElectrumWindow, keystore: 'Hardware_KeyStore'): try: diff --git a/electrum/plugins/keepkey/keepkey.py b/electrum/plugins/keepkey/keepkey.py index 3c76ef09a5..86eef60dca 100644 --- a/electrum/plugins/keepkey/keepkey.py +++ b/electrum/plugins/keepkey/keepkey.py @@ -179,10 +179,11 @@ def create_client(self, device, handler): return client - def get_client(self, keystore, force_pair=True) -> Optional['KeepKeyClient']: - devmgr = self.device_manager() - handler = keystore.handler - client = devmgr.client_for_keystore(self, handler, keystore, force_pair) + def get_client(self, keystore, force_pair=True, *, + devices=None, allow_user_interaction=True) -> Optional['KeepKeyClient']: + client = super().get_client(keystore, force_pair, + devices=devices, + allow_user_interaction=allow_user_interaction) # returns the client for a given keystore. can use xpub if client: client.used() diff --git a/electrum/plugins/ledger/ledger.py b/electrum/plugins/ledger/ledger.py index 1866d735f7..3522021f33 100644 --- a/electrum/plugins/ledger/ledger.py +++ b/electrum/plugins/ledger/ledger.py @@ -612,11 +612,12 @@ def get_xpub(self, device_id, derivation, xtype, wizard): xpub = client.get_xpub(derivation, xtype) return xpub - def get_client(self, keystore, force_pair=True): + def get_client(self, keystore, force_pair=True, *, + devices=None, allow_user_interaction=True): # All client interaction should not be in the main GUI thread - devmgr = self.device_manager() - handler = keystore.handler - client = devmgr.client_for_keystore(self, handler, keystore, force_pair) + client = super().get_client(keystore, force_pair, + devices=devices, + allow_user_interaction=allow_user_interaction) # returns the client for a given keystore. can use xpub #if client: # client.used() diff --git a/electrum/plugins/safe_t/safe_t.py b/electrum/plugins/safe_t/safe_t.py index 4bf6381e29..e3b6255954 100644 --- a/electrum/plugins/safe_t/safe_t.py +++ b/electrum/plugins/safe_t/safe_t.py @@ -141,10 +141,11 @@ def create_client(self, device, handler): return client - def get_client(self, keystore, force_pair=True) -> Optional['SafeTClient']: - devmgr = self.device_manager() - handler = keystore.handler - client = devmgr.client_for_keystore(self, handler, keystore, force_pair) + def get_client(self, keystore, force_pair=True, *, + devices=None, allow_user_interaction=True) -> Optional['SafeTClient']: + client = super().get_client(keystore, force_pair, + devices=devices, + allow_user_interaction=allow_user_interaction) # returns the client for a given keystore. can use xpub if client: client.used() diff --git a/electrum/plugins/trezor/trezor.py b/electrum/plugins/trezor/trezor.py index f1314f41ca..cac338866a 100644 --- a/electrum/plugins/trezor/trezor.py +++ b/electrum/plugins/trezor/trezor.py @@ -177,10 +177,11 @@ def create_client(self, device, handler): # note that this call can still raise! return TrezorClientBase(transport, handler, self) - def get_client(self, keystore, force_pair=True) -> Optional['TrezorClientBase']: - devmgr = self.device_manager() - handler = keystore.handler - client = devmgr.client_for_keystore(self, handler, keystore, force_pair) + def get_client(self, keystore, force_pair=True, *, + devices=None, allow_user_interaction=True) -> Optional['TrezorClientBase']: + client = super().get_client(keystore, force_pair, + devices=devices, + allow_user_interaction=allow_user_interaction) # returns the client for a given keystore. can use xpub if client: client.used() From e1996bde01edbae2619915eb216ede0639765946 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Wed, 8 Apr 2020 16:50:55 +0200 Subject: [PATCH 019/117] hww: select_device: only update label/dev_id after pairing succeeds --- electrum/plugin.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/electrum/plugin.py b/electrum/plugin.py index 453f94c0bd..8c81230cdc 100644 --- a/electrum/plugin.py +++ b/electrum/plugin.py @@ -485,6 +485,7 @@ def client_for_keystore(self, plugin: 'HW_PluginBase', handler: Optional['Hardwa if client: handler.update_status(True) if client: + # note: if select_device was called, we might also update label etc here: keystore.opportunistically_fill_in_missing_info_from_device(client) self.logger.info("end client for keystore") return client @@ -621,12 +622,7 @@ def select_device(self, plugin: 'HW_PluginBase', handler: 'HardwareHandlerBase', if c is None: raise UserCancelled() info = infos[c] - # save new label / soft_device_id - keystore.set_label(info.label) - keystore.soft_device_id = info.soft_device_id - wallet = handler.get_wallet() - if wallet is not None: - wallet.save_keystore() + # note: updated label/soft_device_id will be saved after pairing succeeds return info @with_scan_lock From db1ff4915fc613d30d4d73289337b38a2e2b0e2c Mon Sep 17 00:00:00 2001 From: SomberNight Date: Wed, 8 Apr 2020 17:25:18 +0200 Subject: [PATCH 020/117] hww: show model name in device enum lists (e.g. "Trezor T") --- electrum/base_wizard.py | 2 +- electrum/plugin.py | 9 ++++++--- electrum/plugins/hw_wallet/plugin.py | 6 ++++++ electrum/plugins/ledger/ledger.py | 22 ++++++++++++++++------ electrum/plugins/trezor/clientbase.py | 8 ++++++++ 5 files changed, 37 insertions(+), 10 deletions(-) diff --git a/electrum/base_wizard.py b/electrum/base_wizard.py index d6a2b20bf9..406907772d 100644 --- a/electrum/base_wizard.py +++ b/electrum/base_wizard.py @@ -336,7 +336,7 @@ def failed_getting_device_infos(name, e): label = info.label or _("An unnamed {}").format(name) try: transport_str = info.device.transport_ui_string[:20] except: transport_str = 'unknown transport' - descr = f"{label} [{name}, {state}, {transport_str}]" + descr = f"{label} [{info.model_name or name}, {state}, {transport_str}]" choices.append(((name, info), descr)) msg = _('Select a device') + ':' self.choice_dialog(title=title, message=msg, choices=choices, diff --git a/electrum/plugin.py b/electrum/plugin.py index 8c81230cdc..dcbd452579 100644 --- a/electrum/plugin.py +++ b/electrum/plugin.py @@ -308,6 +308,7 @@ class DeviceInfo(NamedTuple): exception: Optional[Exception] = None plugin_name: Optional[str] = None # manufacturer, e.g. "trezor" soft_device_id: Optional[str] = None # if available, used to distinguish same-type hw devices + model_name: Optional[str] = None # e.g. "Ledger Nano S" class HardwarePluginToScan(NamedTuple): @@ -560,7 +561,8 @@ def unpaired_device_infos(self, handler: Optional['HardwareHandlerBase'], plugin label=client.label(), initialized=client.is_initialized(), plugin_name=plugin.name, - soft_device_id=client.get_soft_device_id())) + soft_device_id=client.get_soft_device_id(), + model_name=client.device_model_name())) return infos @@ -613,10 +615,11 @@ def select_device(self, plugin: 'HW_PluginBase', handler: 'HardwareHandlerBase', raise CannotAutoSelectDevice() # ask user to select device manually msg = _("Please select which {} device to use:").format(plugin.device) - descriptions = ["{label} ({init}, {transport})" + descriptions = ["{label} ({maybe_model}{init}, {transport})" .format(label=info.label or _("An unnamed {}").format(info.plugin_name), init=(_("initialized") if info.initialized else _("wiped")), - transport=info.device.transport_ui_string) + transport=info.device.transport_ui_string, + maybe_model=f"{info.model_name}, " if info.model_name else "") for info in infos] c = handler.query_choice(msg, descriptions) if c is None: diff --git a/electrum/plugins/hw_wallet/plugin.py b/electrum/plugins/hw_wallet/plugin.py index 4267e98def..2fb7876edb 100644 --- a/electrum/plugins/hw_wallet/plugin.py +++ b/electrum/plugins/hw_wallet/plugin.py @@ -228,6 +228,12 @@ def get_password_for_storage_encryption(self) -> str: password = Xpub.get_pubkey_from_xpub(xpub, ()).hex() return password + def device_model_name(self) -> Optional[str]: + """Return the name of the model of this device, which might be displayed in the UI. + E.g. for Trezor, "Trezor One" or "Trezor T". + """ + return None + class HardwareHandlerBase: """An interface between the GUI and the device handling logic for handling I/O.""" diff --git a/electrum/plugins/ledger/ledger.py b/electrum/plugins/ledger/ledger.py index 3522021f33..54bf0a0235 100644 --- a/electrum/plugins/ledger/ledger.py +++ b/electrum/plugins/ledger/ledger.py @@ -2,7 +2,7 @@ import hashlib import sys import traceback -from typing import Optional +from typing import Optional, Tuple from electrum import ecc from electrum import bip32 @@ -62,10 +62,10 @@ def catch_exception(self, *args, **kwargs): class Ledger_Client(HardwareClientBase): - def __init__(self, hidDevice, *, is_hw1: bool = False): + def __init__(self, hidDevice, *, product_key: Tuple[int, int]): self.dongleObject = btchip(hidDevice) self.preflightDone = False - self._is_hw1 = is_hw1 + self._product_key = product_key self._soft_device_id = None def is_pairable(self): @@ -92,7 +92,18 @@ def get_soft_device_id(self): return self._soft_device_id def is_hw1(self) -> bool: - return self._is_hw1 + return self._product_key[0] == 0x2581 + + def device_model_name(self): + if self.is_hw1(): + return "Ledger HW.1" + if self._product_key == (0x2c97, 0x0000): + return "Ledger Blue" + if self._product_key == (0x2c97, 0x0001): + return "Ledger Nano S" + if self._product_key == (0x2c97, 0x0004): + return "Ledger Nano X" + return None def has_usable_connection_with_device(self): try: @@ -594,8 +605,7 @@ def create_client(self, device, handler): client = self.get_btchip_device(device) if client is not None: - is_hw1 = device.product_key[0] == 0x2581 - client = Ledger_Client(client, is_hw1=is_hw1) + client = Ledger_Client(client, product_key=device.product_key) return client def setup_device(self, device_info, wizard, purpose): diff --git a/electrum/plugins/trezor/clientbase.py b/electrum/plugins/trezor/clientbase.py index 19814b4c6f..829a245bca 100644 --- a/electrum/plugins/trezor/clientbase.py +++ b/electrum/plugins/trezor/clientbase.py @@ -196,6 +196,14 @@ def get_trezor_model(self): """Returns '1' for Trezor One, 'T' for Trezor T.""" return self.features.model + def device_model_name(self): + model = self.get_trezor_model() + if model == '1': + return "Trezor One" + elif model == 'T': + return "Trezor T" + return None + def show_address(self, address_str, script_type, multisig=None): coin_name = self.plugin.get_coin_name() address_n = parse_path(address_str) From bf067f7558657edd31b2dd7e6b30f0243cda528a Mon Sep 17 00:00:00 2001 From: SomberNight Date: Wed, 8 Apr 2020 18:28:21 +0200 Subject: [PATCH 021/117] HardwareClientBase: provide default implementation for label and add warning about placeholders --- electrum/plugins/digitalbitbox/digitalbitbox.py | 4 ---- electrum/plugins/hw_wallet/plugin.py | 4 +++- electrum/plugins/ledger/ledger.py | 3 --- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/electrum/plugins/digitalbitbox/digitalbitbox.py b/electrum/plugins/digitalbitbox/digitalbitbox.py index 463a2f954d..b0986dc1ed 100644 --- a/electrum/plugins/digitalbitbox/digitalbitbox.py +++ b/electrum/plugins/digitalbitbox/digitalbitbox.py @@ -88,10 +88,6 @@ def timeout(self, cutoff): pass - def label(self): - return " " - - def is_pairable(self): return True diff --git a/electrum/plugins/hw_wallet/plugin.py b/electrum/plugins/hw_wallet/plugin.py index 2fb7876edb..661fdbc10e 100644 --- a/electrum/plugins/hw_wallet/plugin.py +++ b/electrum/plugins/hw_wallet/plugin.py @@ -196,7 +196,9 @@ def label(self) -> Optional[str]: and they are also used as a fallback to distinguish devices programmatically. So ideally, different devices would have different labels. """ - raise NotImplementedError() + # When returning a constant here (i.e. not implementing the method in the way + # it is supposed to work), make sure the return value is in electrum.plugin.PLACEHOLDER_HW_CLIENT_LABELS + return " " def get_soft_device_id(self) -> Optional[str]: """An id-like string that is used to distinguish devices programmatically. diff --git a/electrum/plugins/ledger/ledger.py b/electrum/plugins/ledger/ledger.py index 54bf0a0235..5b9047c27c 100644 --- a/electrum/plugins/ledger/ledger.py +++ b/electrum/plugins/ledger/ledger.py @@ -80,9 +80,6 @@ def timeout(self, cutoff): def is_initialized(self): return True - def label(self): - return "" - def get_soft_device_id(self): if self._soft_device_id is None: # modern ledger can provide xpub without user interaction From 7a4acb05f2c9e21a738cffde712b5d8cd6eca783 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Wed, 8 Apr 2020 18:46:28 +0200 Subject: [PATCH 022/117] hww: fix threading issue in DeviceMgr: enumerate_func needs self.lock E | gui.qt.main_window.[test_ms_p2wsh_2of3_cc3132_trezort_cc3133] | on_error Traceback (most recent call last): File "...\electrum\electrum\gui\qt\util.py", line 794, in run result = task.task() File "...\electrum\electrum\plugins\hw_wallet\qt.py", line 232, in trigger_pairings devices = devmgr.scan_devices() File "...\electrum\electrum\plugin.py", line 376, in func_wrapper return func(self, *args, **kwargs) File "...\electrum\electrum\plugin.py", line 656, in scan_devices for f in self.enumerate_func: RuntimeError: Set changed size during iteration --- electrum/plugin.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/electrum/plugin.py b/electrum/plugin.py index dcbd452579..0db564efa2 100644 --- a/electrum/plugin.py +++ b/electrum/plugin.py @@ -364,7 +364,7 @@ def __init__(self, config: SimpleConfig): # pair. self.recognised_hardware = set() # Custom enumerate functions for devices we don't know about. - self.enumerate_func = set() + self._enumerate_func = set() # Needs self.lock. # locks: if you need to take multiple ones, acquire them in the order they are defined here! self._scan_lock = threading.RLock() self.lock = threading.RLock() @@ -395,7 +395,8 @@ def register_devices(self, device_pairs): self.recognised_hardware.add(pair) def register_enumerate_func(self, func): - self.enumerate_func.add(func) + with self.lock: + self._enumerate_func.add(func) def create_client(self, device: 'Device', handler: Optional['HardwareHandlerBase'], plugin: 'HW_PluginBase') -> Optional['HardwareClientBase']: @@ -664,7 +665,9 @@ def scan_devices(self) -> Sequence['Device']: devices = self._scan_devices_with_hid() # Let plugin handlers enumerate devices we don't know about - for f in self.enumerate_func: + with self.lock: + enumerate_funcs = list(self._enumerate_func) + for f in enumerate_funcs: try: new_devices = f() except BaseException as e: From 7c830cb22181e26c77d7819175e40d0c756618ce Mon Sep 17 00:00:00 2001 From: SomberNight Date: Wed, 8 Apr 2020 18:54:11 +0200 Subject: [PATCH 023/117] wizard hww: move devmgr.scan_devices() away from GUI thread --- electrum/base_wizard.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/electrum/base_wizard.py b/electrum/base_wizard.py index 406907772d..a03e814923 100644 --- a/electrum/base_wizard.py +++ b/electrum/base_wizard.py @@ -280,7 +280,8 @@ def failed_getting_device_infos(name, e): # scan devices try: - scanned_devices = devmgr.scan_devices() + scanned_devices = self.run_task_without_blocking_gui(task=devmgr.scan_devices, + msg=_("Scanning devices...")) except BaseException as e: self.logger.info('error scanning devices: {}'.format(repr(e))) debug_msg = ' {}:\n {}'.format(_('Error scanning devices'), e) From 756c7db888fdea7bdff616010193d006c6180958 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Thu, 9 Apr 2020 10:59:22 +0200 Subject: [PATCH 024/117] setup.py: specify lnwire as package_data fixes #6078 --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 8436c4f939..194517591c 100755 --- a/setup.py +++ b/setup.py @@ -82,6 +82,7 @@ 'electrum': [ 'wordlist/*.txt', 'locale/*/LC_MESSAGES/electrum.mo', + 'lnwire/*.csv', ], 'electrum.gui': [ 'icons/*', From 5efaaa523a30dd2ea9866ba3b96ce96a41098c1e Mon Sep 17 00:00:00 2001 From: ThomasV Date: Thu, 9 Apr 2020 15:16:07 +0200 Subject: [PATCH 025/117] lnworker: check chain_hash when decoding channel update. --- electrum/lnworker.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/electrum/lnworker.py b/electrum/lnworker.py index bde0cdd65d..6a697442f0 100644 --- a/electrum/lnworker.py +++ b/electrum/lnworker.py @@ -902,18 +902,24 @@ def handle_error_code_from_failed_htlc(self, failure_msg, sender_idx, route, pee # we try decoding both ways here. try: message_type, payload = decode_msg(channel_update_typed) + assert payload['chain_hash'] == constants.net.rev_genesis_bytes() payload['raw'] = channel_update_typed except: # FIXME: too broad message_type, payload = decode_msg(channel_update_as_received) payload['raw'] = channel_update_as_received + # sanity check + if payload['chain_hash'] != constants.net.rev_genesis_bytes(): + self.logger.info(f'could not decode channel_update for failed htlc: {channel_update_as_received.hex()}') + return True categorized_chan_upds = self.channel_db.add_channel_updates([payload]) blacklist = False + short_channel_id = ShortChannelID(payload['short_channel_id']) if categorized_chan_upds.good: - self.logger.info("applied channel update on our db") + self.logger.info(f"applied channel update to {short_channel_id}") peer.maybe_save_remote_update(payload) elif categorized_chan_upds.orphaned: # maybe it is a private channel (and data in invoice was outdated) - self.logger.info("maybe channel update is for private channel?") + self.logger.info(f"Could not find {short_channel_id}. maybe update is for private channel?") start_node_id = route[sender_idx].node_id self.channel_db.add_channel_update_for_private_channel(payload, start_node_id) elif categorized_chan_upds.expired: From 01dac92e19481690e31b06632c1b1833468cee0b Mon Sep 17 00:00:00 2001 From: SomberNight Date: Thu, 9 Apr 2020 17:55:42 +0200 Subject: [PATCH 026/117] wizard: fix crash when decrypting wallet hw device E | __main__ | daemon.run_gui errored Traceback (most recent call last): File ".../electrum/run_electrum", line 379, in d.run_gui(config, plugins) File "...\electrum\electrum\daemon.py", line 522, in run_gui self.gui_object.main() File "...\electrum\electrum\gui\qt\__init__.py", line 362, in main if not self.start_new_window(path, self.config.get('url'), app_is_starting=True): File "...\electrum\electrum\gui\qt\__init__.py", line 246, in wrapper return func(self, *args, **kwargs) File "...\electrum\electrum\gui\qt\__init__.py", line 270, in start_new_window wallet = self._start_wizard_to_select_or_create_wallet(path) File "...\electrum\electrum\gui\qt\__init__.py", line 308, in _start_wizard_to_select_or_create_wallet path, storage = wizard.select_storage(path, self.daemon.get_wallet) File "...\electrum\electrum\gui\qt\installwizard.py", line 334, in select_storage pw_e.clear() File "...\electrum\electrum\gui\qt\util.py", line 759, in clear self.setText(len(self.text()) * " ") RuntimeError: wrapped C/C++ object of type PasswordLineEdit has been deleted --- electrum/gui/qt/installwizard.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/electrum/gui/qt/installwizard.py b/electrum/gui/qt/installwizard.py index f94037b166..6918ab2857 100644 --- a/electrum/gui/qt/installwizard.py +++ b/electrum/gui/qt/installwizard.py @@ -331,7 +331,10 @@ def run_user_interaction_loop(): try: run_user_interaction_loop() finally: - pw_e.clear() + try: + pw_e.clear() + except RuntimeError: # wrapped C/C++ object has been deleted. + pass # happens when decrypting with hw device return temp_storage.path, (temp_storage if temp_storage.file_exists() else None) From 4b1d8353040513f6a3e371b7385711eb9f146a70 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Thu, 9 Apr 2020 18:00:35 +0200 Subject: [PATCH 027/117] wizard hww: scan devices fewer times and move away from GUI thread --- electrum/base_wizard.py | 7 +++---- electrum/plugin.py | 17 +++++++++-------- electrum/plugins/coldcard/coldcard.py | 1 + electrum/plugins/digitalbitbox/digitalbitbox.py | 1 + electrum/plugins/hw_wallet/plugin.py | 6 ++++-- electrum/plugins/keepkey/keepkey.py | 1 + electrum/plugins/ledger/ledger.py | 1 + electrum/plugins/safe_t/safe_t.py | 1 + electrum/plugins/trezor/trezor.py | 1 + 9 files changed, 22 insertions(+), 14 deletions(-) diff --git a/electrum/base_wizard.py b/electrum/base_wizard.py index a03e814923..4d97506c7c 100644 --- a/electrum/base_wizard.py +++ b/electrum/base_wizard.py @@ -348,7 +348,7 @@ def on_device(self, name, device_info: 'DeviceInfo', *, purpose, storage=None): assert isinstance(self.plugin, HW_PluginBase) devmgr = self.plugins.device_manager try: - self.plugin.setup_device(device_info, self, purpose) + client = self.plugin.setup_device(device_info, self, purpose) except OSError as e: self.show_error(_('We encountered an error while connecting to your device:') + '\n' + str(e) + '\n' @@ -376,19 +376,18 @@ def on_device(self, name, device_info: 'DeviceInfo', *, purpose, storage=None): self.show_error(str(e)) self.choose_hw_device(purpose, storage=storage) return + if purpose == HWD_SETUP_NEW_WALLET: def f(derivation, script_type): derivation = normalize_bip32_derivation(derivation) self.run('on_hw_derivation', name, device_info, derivation, script_type) self.derivation_and_script_type_dialog(f) elif purpose == HWD_SETUP_DECRYPT_WALLET: - client = devmgr.client_by_id(device_info.device.id_) password = client.get_password_for_storage_encryption() try: storage.decrypt(password) except InvalidPassword: # try to clear session so that user can type another passphrase - client = devmgr.client_by_id(device_info.device.id_) if hasattr(client, 'clear_session'): # FIXME not all hw wallet plugins have this client.clear_session() raise @@ -435,7 +434,7 @@ def on_hw_derivation(self, name, device_info: 'DeviceInfo', derivation, xtype): assert isinstance(self.plugin, HW_PluginBase) try: xpub = self.plugin.get_xpub(device_info.device.id_, derivation, xtype, self) - client = devmgr.client_by_id(device_info.device.id_) + client = devmgr.client_by_id(device_info.device.id_, scan_now=False) if not client: raise Exception("failed to find client for device id") root_fingerprint = client.request_root_fingerprint_from_device() label = client.label() # use this as device_info.label might be outdated! diff --git a/electrum/plugin.py b/electrum/plugin.py index 0db564efa2..d6136506f2 100644 --- a/electrum/plugin.py +++ b/electrum/plugin.py @@ -401,7 +401,7 @@ def register_enumerate_func(self, func): def create_client(self, device: 'Device', handler: Optional['HardwareHandlerBase'], plugin: 'HW_PluginBase') -> Optional['HardwareClientBase']: # Get from cache first - client = self.client_lookup(device.id_) + client = self._client_by_id(device.id_) if client: return client client = plugin.create_client(device, handler) @@ -437,7 +437,7 @@ def unpair_id(self, id_): self._close_client(id_) def _close_client(self, id_): - client = self.client_lookup(id_) + client = self._client_by_id(id_) self.clients.pop(client, None) if client: client.close() @@ -446,19 +446,20 @@ def pair_xpub(self, xpub, id_): with self.lock: self.xpub_ids[xpub] = id_ - def client_lookup(self, id_) -> Optional['HardwareClientBase']: + def _client_by_id(self, id_) -> Optional['HardwareClientBase']: with self.lock: for client, (path, client_id) in self.clients.items(): if client_id == id_: return client return None - def client_by_id(self, id_) -> Optional['HardwareClientBase']: + def client_by_id(self, id_, *, scan_now: bool = True) -> Optional['HardwareClientBase']: '''Returns a client for the device ID if one is registered. If a device is wiped or in bootloader mode pairing is impossible; in such cases we communicate by device ID and not wallet.''' - self.scan_devices() - return self.client_lookup(id_) + if scan_now: + self.scan_devices() + return self._client_by_id(id_) @with_scan_lock def client_for_keystore(self, plugin: 'HW_PluginBase', handler: Optional['HardwareHandlerBase'], @@ -495,7 +496,7 @@ def client_for_keystore(self, plugin: 'HW_PluginBase', handler: Optional['Hardwa def client_by_xpub(self, plugin: 'HW_PluginBase', xpub, handler: 'HardwareHandlerBase', devices: Sequence['Device']) -> Optional['HardwareClientBase']: _id = self.xpub_id(xpub) - client = self.client_lookup(_id) + client = self._client_by_id(_id) if client: # An unpaired client might have another wallet's handler # from a prior scan. Replace to fix dialog parenting. @@ -511,7 +512,7 @@ def force_pair_xpub(self, plugin: 'HW_PluginBase', handler: 'HardwareHandlerBase # The wallet has not been previously paired, so let the user # choose an unpaired device and compare its first address. xtype = bip32.xpub_type(xpub) - client = self.client_lookup(info.device.id_) + client = self._client_by_id(info.device.id_) if client and client.is_pairable(): # See comment above for same code client.handler = handler diff --git a/electrum/plugins/coldcard/coldcard.py b/electrum/plugins/coldcard/coldcard.py index c0f34cfd77..810fd91b37 100644 --- a/electrum/plugins/coldcard/coldcard.py +++ b/electrum/plugins/coldcard/coldcard.py @@ -524,6 +524,7 @@ def create_client(self, device, handler): def setup_device(self, device_info, wizard, purpose): device_id = device_info.device.id_ client = self.scan_and_create_client_for_device(device_id=device_id, wizard=wizard) + return client def get_xpub(self, device_id, derivation, xtype, wizard): # this seems to be part of the pairing process only, not during normal ops? diff --git a/electrum/plugins/digitalbitbox/digitalbitbox.py b/electrum/plugins/digitalbitbox/digitalbitbox.py index b0986dc1ed..b141955785 100644 --- a/electrum/plugins/digitalbitbox/digitalbitbox.py +++ b/electrum/plugins/digitalbitbox/digitalbitbox.py @@ -705,6 +705,7 @@ def setup_device(self, device_info, wizard, purpose): client.setupRunning = True wizard.run_task_without_blocking_gui( task=lambda: client.get_xpub("m/44'/0'", 'standard')) + return client def is_mobile_paired(self): diff --git a/electrum/plugins/hw_wallet/plugin.py b/electrum/plugins/hw_wallet/plugin.py index 661fdbc10e..68335f746d 100644 --- a/electrum/plugins/hw_wallet/plugin.py +++ b/electrum/plugins/hw_wallet/plugin.py @@ -25,6 +25,7 @@ # SOFTWARE. from typing import TYPE_CHECKING, Dict, List, Union, Tuple, Sequence, Optional, Type +from functools import partial from electrum.plugin import BasePlugin, hook, Device, DeviceMgr, DeviceInfo from electrum.i18n import _ @@ -67,14 +68,15 @@ def close_wallet(self, wallet: 'Abstract_Wallet'): def scan_and_create_client_for_device(self, *, device_id: str, wizard: 'BaseWizard') -> 'HardwareClientBase': devmgr = self.device_manager() - client = devmgr.client_by_id(device_id) + client = wizard.run_task_without_blocking_gui( + task=partial(devmgr.client_by_id, device_id)) if client is None: raise UserFacingException(_('Failed to create a client for this device.') + '\n' + _('Make sure it is in the correct state.')) client.handler = self.create_handler(wizard) return client - def setup_device(self, device_info: DeviceInfo, wizard: 'BaseWizard', purpose): + def setup_device(self, device_info: DeviceInfo, wizard: 'BaseWizard', purpose) -> 'HardwareClientBase': """Called when creating a new wallet or when using the device to decrypt an existing wallet. Select the device to use. If the device is uninitialized, go through the initialization process. diff --git a/electrum/plugins/keepkey/keepkey.py b/electrum/plugins/keepkey/keepkey.py index 86eef60dca..1722eaea64 100644 --- a/electrum/plugins/keepkey/keepkey.py +++ b/electrum/plugins/keepkey/keepkey.py @@ -283,6 +283,7 @@ def setup_device(self, device_info, wizard, purpose): wizard.run_task_without_blocking_gui( task=lambda: client.get_xpub("m", 'standard')) client.used() + return client def get_xpub(self, device_id, derivation, xtype, wizard): if xtype not in self.SUPPORTED_XTYPES: diff --git a/electrum/plugins/ledger/ledger.py b/electrum/plugins/ledger/ledger.py index 5b9047c27c..d5abc5dbef 100644 --- a/electrum/plugins/ledger/ledger.py +++ b/electrum/plugins/ledger/ledger.py @@ -610,6 +610,7 @@ def setup_device(self, device_info, wizard, purpose): client = self.scan_and_create_client_for_device(device_id=device_id, wizard=wizard) wizard.run_task_without_blocking_gui( task=lambda: client.get_xpub("m/44'/0'", 'standard')) # TODO replace by direct derivation once Nano S > 1.1 + return client def get_xpub(self, device_id, derivation, xtype, wizard): if xtype not in self.SUPPORTED_XTYPES: diff --git a/electrum/plugins/safe_t/safe_t.py b/electrum/plugins/safe_t/safe_t.py index e3b6255954..2a74e43e29 100644 --- a/electrum/plugins/safe_t/safe_t.py +++ b/electrum/plugins/safe_t/safe_t.py @@ -257,6 +257,7 @@ def setup_device(self, device_info, wizard, purpose): wizard.run_task_without_blocking_gui( task=lambda: client.get_xpub("m", 'standard')) client.used() + return client def get_xpub(self, device_id, derivation, xtype, wizard): if xtype not in self.SUPPORTED_XTYPES: diff --git a/electrum/plugins/trezor/trezor.py b/electrum/plugins/trezor/trezor.py index cac338866a..dd4bf2d126 100644 --- a/electrum/plugins/trezor/trezor.py +++ b/electrum/plugins/trezor/trezor.py @@ -288,6 +288,7 @@ def setup_device(self, device_info, wizard, purpose): wizard.run_task_without_blocking_gui( task=lambda: client.get_xpub('m', 'standard', creating=is_creating_wallet)) client.used() + return client def get_xpub(self, device_id, derivation, xtype, wizard): if xtype not in self.SUPPORTED_XTYPES: From a3e1b2e00c04b72e7375b121f094d34e721fe6ed Mon Sep 17 00:00:00 2001 From: SomberNight Date: Thu, 9 Apr 2020 19:04:52 +0200 Subject: [PATCH 028/117] wizard: hww creation flow: don't just swallow exception if we just return here, the calling code will try to create the storage and fail --- electrum/base_wizard.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/electrum/base_wizard.py b/electrum/base_wizard.py index 4d97506c7c..57c06209e2 100644 --- a/electrum/base_wizard.py +++ b/electrum/base_wizard.py @@ -263,7 +263,7 @@ def on_restore_from_key(self, text): k = keystore.from_master_key(text) self.on_keystore(k) - def choose_hw_device(self, purpose=HWD_SETUP_NEW_WALLET, *, storage=None): + def choose_hw_device(self, purpose=HWD_SETUP_NEW_WALLET, *, storage: WalletStorage = None): title = _('Hardware Keystore') # check available plugins supported_plugins = self.plugins.get_hardware_support() @@ -343,7 +343,7 @@ def failed_getting_device_infos(name, e): self.choice_dialog(title=title, message=msg, choices=choices, run_next=lambda *args: self.on_device(*args, purpose=purpose, storage=storage)) - def on_device(self, name, device_info: 'DeviceInfo', *, purpose, storage=None): + def on_device(self, name, device_info: 'DeviceInfo', *, purpose, storage: WalletStorage = None): self.plugin = self.plugins.get_plugin(name) assert isinstance(self.plugin, HW_PluginBase) devmgr = self.plugins.device_manager @@ -444,6 +444,7 @@ def on_hw_derivation(self, name, device_info: 'DeviceInfo', derivation, xtype): except BaseException as e: self.logger.exception('') self.show_error(e) + self.choose_hw_device() return d = { 'type': 'hardware', @@ -565,6 +566,7 @@ def create_wallet(self): except BaseException as e: self.logger.exception('') self.show_error(str(e)) + self.choose_hw_device() return self.request_storage_encryption( run_next=lambda encrypt_storage: self.on_password( From 08a79252353476aed746ff4c1085cf108f314a23 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Thu, 9 Apr 2020 19:08:17 +0200 Subject: [PATCH 029/117] wizard.create_storage: state API and abide by it none of the callers was handling the return None case properly... --- electrum/base_wizard.py | 12 ++++++------ electrum/storage.py | 1 - 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/electrum/base_wizard.py b/electrum/base_wizard.py index 57c06209e2..d8ea238ba0 100644 --- a/electrum/base_wizard.py +++ b/electrum/base_wizard.py @@ -28,7 +28,7 @@ import copy import traceback from functools import partial -from typing import List, TYPE_CHECKING, Tuple, NamedTuple, Any, Dict, Optional +from typing import List, TYPE_CHECKING, Tuple, NamedTuple, Any, Dict, Optional, Union from . import bitcoin from . import keystore @@ -611,12 +611,10 @@ def on_password(self, password, *, encrypt_storage: bool, encrypt_keystore=encrypt_keystore) self.terminate() - - def create_storage(self, path): + def create_storage(self, path) -> Tuple[WalletStorage, WalletDB]: if os.path.exists(path): raise Exception('file already exists at path') - if not self.pw_args: - return # FIXME + assert self.pw_args, f"pw_args not set?!" pw_args = self.pw_args self.pw_args = None # clean-up so that it can get GC-ed storage = WalletStorage(path) @@ -630,7 +628,9 @@ def create_storage(self, path): db.write(storage) return storage, db - def terminate(self, *, storage: Optional[WalletStorage], db: Optional[WalletDB] = None): + def terminate(self, *, storage: WalletStorage = None, + db: WalletDB = None, + aborted: bool = False) -> None: raise NotImplementedError() # implemented by subclasses def show_xpub_and_add_cosigners(self, xpub): diff --git a/electrum/storage.py b/electrum/storage.py index 17888b67e2..47b804588b 100644 --- a/electrum/storage.py +++ b/electrum/storage.py @@ -189,7 +189,6 @@ def decrypt(self, password) -> None: s = '' self.pubkey = ec_key.get_public_key_hex() self.decrypted = s - return s def encrypt_before_writing(self, plaintext: str) -> str: s = plaintext From 71eed1d4cb70b69767974d21430cfa28ba2cc6eb Mon Sep 17 00:00:00 2001 From: SomberNight Date: Thu, 9 Apr 2020 19:10:39 +0200 Subject: [PATCH 030/117] wizard: (trivial) add show_error to base class, document API --- electrum/base_wizard.py | 3 +++ electrum/gui/kivy/main_window.py | 9 +++++---- electrum/gui/kivy/uix/dialogs/installwizard.py | 7 ++++++- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/electrum/base_wizard.py b/electrum/base_wizard.py index d8ea238ba0..85effd6586 100644 --- a/electrum/base_wizard.py +++ b/electrum/base_wizard.py @@ -685,3 +685,6 @@ def confirm_passphrase(self, seed, passphrase): self.line_dialog(run_next=f, title=title, message=message, default='', test=lambda x: x==passphrase) else: f('') + + def show_error(self, msg: Union[str, BaseException]) -> None: + raise NotImplementedError() diff --git a/electrum/gui/kivy/main_window.py b/electrum/gui/kivy/main_window.py index e71c996651..aaa7d802ff 100644 --- a/electrum/gui/kivy/main_window.py +++ b/electrum/gui/kivy/main_window.py @@ -975,8 +975,8 @@ def on_ref_label(self, label): self.qr_dialog(label.name, label.data, True) def show_error(self, error, width='200dp', pos=None, arrow_pos=None, - exit=False, icon='atlas://electrum/gui/kivy/theming/light/error', duration=0, - modal=False): + exit=False, icon='atlas://electrum/gui/kivy/theming/light/error', duration=0, + modal=False): ''' Show an error Message Bubble. ''' self.show_info_bubble( text=error, icon=icon, width=width, @@ -984,7 +984,7 @@ def show_error(self, error, width='200dp', pos=None, arrow_pos=None, duration=duration, modal=modal) def show_info(self, error, width='200dp', pos=None, arrow_pos=None, - exit=False, duration=0, modal=False): + exit=False, duration=0, modal=False): ''' Show an Info Message Bubble. ''' self.show_error(error, icon='atlas://electrum/gui/kivy/theming/light/important', @@ -992,7 +992,7 @@ def show_info(self, error, width='200dp', pos=None, arrow_pos=None, arrow_pos=arrow_pos) def show_info_bubble(self, text=_('Hello World'), pos=None, duration=0, - arrow_pos='bottom_mid', width=None, icon='', modal=False, exit=False): + arrow_pos='bottom_mid', width=None, icon='', modal=False, exit=False): '''Method to show an Information Bubble .. parameters:: @@ -1002,6 +1002,7 @@ def show_info_bubble(self, text=_('Hello World'), pos=None, duration=0, width: width of the Bubble arrow_pos: arrow position for the bubble ''' + text = str(text) # so that we also handle e.g. Exception info_bubble = self.info_bubble if not info_bubble: info_bubble = self.info_bubble = Factory.InfoBubble() diff --git a/electrum/gui/kivy/uix/dialogs/installwizard.py b/electrum/gui/kivy/uix/dialogs/installwizard.py index 7b3fb6b894..487cf57eee 100644 --- a/electrum/gui/kivy/uix/dialogs/installwizard.py +++ b/electrum/gui/kivy/uix/dialogs/installwizard.py @@ -2,6 +2,7 @@ from functools import partial import threading import os +from typing import TYPE_CHECKING from kivy.app import App from kivy.clock import Clock @@ -24,6 +25,10 @@ from ...i18n import _ from .password_dialog import PasswordDialog +if TYPE_CHECKING: + from electrum.gui.kivy.main_window import ElectrumWindow + + # global Variables is_test = (platform == "linux") test_seed = "grape impose jazz bind spatial mind jelly tourist tank today holiday stomach" @@ -1153,7 +1158,7 @@ def show_xpub_dialog(self, **kwargs): ShowXpubDialog(self, **kwargs).open() def show_message(self, msg): self.show_error(msg) def show_error(self, msg): - app = App.get_running_app() + app = App.get_running_app() # type: ElectrumWindow Clock.schedule_once(lambda dt: app.show_error(msg)) def request_password(self, run_next, force_disable_encrypt_cb=False): From b6bac0182f23073b9136ebfd3aaf5a13d85f4def Mon Sep 17 00:00:00 2001 From: SomberNight Date: Thu, 9 Apr 2020 19:44:33 +0200 Subject: [PATCH 031/117] wizard hww: use exception handling to choose hw device again - no need to pass args, caller knows what it wanted - avoids deepening the call stack on every rescan (nicer tracebacks, no stack overflow) --- electrum/base_wizard.py | 40 ++++++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/electrum/base_wizard.py b/electrum/base_wizard.py index 85effd6586..ce8fcc6038 100644 --- a/electrum/base_wizard.py +++ b/electrum/base_wizard.py @@ -60,6 +60,9 @@ class ScriptTypeNotSupported(Exception): pass class GoBack(Exception): pass +class ChooseHwDeviceAgain(Exception): pass + + class WizardStackItem(NamedTuple): action: Any args: Any @@ -264,6 +267,15 @@ def on_restore_from_key(self, text): self.on_keystore(k) def choose_hw_device(self, purpose=HWD_SETUP_NEW_WALLET, *, storage: WalletStorage = None): + while True: + try: + self._choose_hw_device(purpose=purpose, storage=storage) + except ChooseHwDeviceAgain: + pass + else: + break + + def _choose_hw_device(self, *, purpose, storage: WalletStorage = None): title = _('Hardware Keystore') # check available plugins supported_plugins = self.plugins.get_hardware_support() @@ -327,8 +339,8 @@ def failed_getting_device_infos(name, e): msg += '\n\n' msg += _('Debug message') + '\n' + debug_msg self.confirm_dialog(title=title, message=msg, - run_next=lambda x: self.choose_hw_device(purpose, storage=storage)) - return + run_next=lambda x: None) + raise ChooseHwDeviceAgain() # select device self.devices = devices choices = [] @@ -355,27 +367,22 @@ def on_device(self, name, device_info: 'DeviceInfo', *, purpose, storage: Wallet + _('To try to fix this, we will now re-pair with your device.') + '\n' + _('Please try again.')) devmgr.unpair_id(device_info.device.id_) - self.choose_hw_device(purpose, storage=storage) - return + raise ChooseHwDeviceAgain() except OutdatedHwFirmwareException as e: if self.question(e.text_ignore_old_fw_and_continue(), title=_("Outdated device firmware")): self.plugin.set_ignore_outdated_fw() # will need to re-pair devmgr.unpair_id(device_info.device.id_) - self.choose_hw_device(purpose, storage=storage) - return + raise ChooseHwDeviceAgain() except (UserCancelled, GoBack): - self.choose_hw_device(purpose, storage=storage) - return + raise ChooseHwDeviceAgain() except UserFacingException as e: self.show_error(str(e)) - self.choose_hw_device(purpose, storage=storage) - return + raise ChooseHwDeviceAgain() except BaseException as e: self.logger.exception('') self.show_error(str(e)) - self.choose_hw_device(purpose, storage=storage) - return + raise ChooseHwDeviceAgain() if purpose == HWD_SETUP_NEW_WALLET: def f(derivation, script_type): @@ -444,8 +451,7 @@ def on_hw_derivation(self, name, device_info: 'DeviceInfo', derivation, xtype): except BaseException as e: self.logger.exception('') self.show_error(e) - self.choose_hw_device() - return + raise ChooseHwDeviceAgain() d = { 'type': 'hardware', 'hw_type': name, @@ -561,13 +567,11 @@ def create_wallet(self): except UserCancelled: devmgr = self.plugins.device_manager devmgr.unpair_xpub(k.xpub) - self.choose_hw_device() - return + raise ChooseHwDeviceAgain() except BaseException as e: self.logger.exception('') self.show_error(str(e)) - self.choose_hw_device() - return + raise ChooseHwDeviceAgain() self.request_storage_encryption( run_next=lambda encrypt_storage: self.on_password( password, From e5b1596b69a1c77b3d8d2b235b65e624a2383921 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Thu, 9 Apr 2020 22:11:25 +0200 Subject: [PATCH 032/117] build: add workaround for "pyinstaller with new setuptools" issue Traceback (most recent call last): File "site-packages\PyInstaller\loader\rthooks\pyi_rth_pkgres.py", line 13, in File "c:\python3\lib\site-packages\PyInstaller\loader\pyimod03_importers.py", line 623, in exec_module File "site-packages\pkg_resources\__init__.py", line 86, in ModuleNotFoundError: No module named 'pkg_resources.py2_warn' [7048] Failed to execute script pyi_rth_pkgres --- contrib/build-wine/deterministic.spec | 1 + contrib/osx/osx.spec | 1 + 2 files changed, 2 insertions(+) diff --git a/contrib/build-wine/deterministic.spec b/contrib/build-wine/deterministic.spec index 9ee6b04a73..2ad11294b0 100644 --- a/contrib/build-wine/deterministic.spec +++ b/contrib/build-wine/deterministic.spec @@ -16,6 +16,7 @@ home = 'C:\\electrum\\' # see https://github.com/pyinstaller/pyinstaller/issues/2005 hiddenimports = [] +hiddenimports += collect_submodules('pkg_resources') # workaround for https://github.com/pypa/setuptools/issues/1963 hiddenimports += collect_submodules('trezorlib') hiddenimports += collect_submodules('safetlib') hiddenimports += collect_submodules('btchip') diff --git a/contrib/osx/osx.spec b/contrib/osx/osx.spec index 2bcaac9fd2..f9ba4fec45 100644 --- a/contrib/osx/osx.spec +++ b/contrib/osx/osx.spec @@ -59,6 +59,7 @@ block_cipher = None # see https://github.com/pyinstaller/pyinstaller/issues/2005 hiddenimports = [] +hiddenimports += collect_submodules('pkg_resources') # workaround for https://github.com/pypa/setuptools/issues/1963 hiddenimports += collect_submodules('trezorlib') hiddenimports += collect_submodules('safetlib') hiddenimports += collect_submodules('btchip') From 8f41aeb783048ca01a2474d738ece1ee2ade28fb Mon Sep 17 00:00:00 2001 From: ThomasV Date: Fri, 13 Mar 2020 11:44:29 +0100 Subject: [PATCH 033/117] Replace wallet backup with channel backups - channels can be backed up individually - backups are added to lnwatcher - AbstractChannel ancestor class --- electrum/commands.py | 10 + electrum/gui/kivy/main_window.py | 20 +- .../kivy/uix/dialogs/lightning_channels.py | 126 +++++- electrum/gui/qt/__init__.py | 1 - electrum/gui/qt/channels_list.py | 51 ++- electrum/gui/qt/main_window.py | 59 ++- electrum/gui/qt/settings_dialog.py | 14 - electrum/lnchannel.py | 402 ++++++++++++------ electrum/lnpeer.py | 26 +- electrum/lnsweep.py | 2 +- electrum/lnutil.py | 70 ++- electrum/lnwatcher.py | 1 + electrum/lnworker.py | 108 ++++- electrum/transaction.py | 4 + electrum/wallet.py | 23 +- electrum/wallet_db.py | 4 +- 16 files changed, 705 insertions(+), 216 deletions(-) diff --git a/electrum/commands.py b/electrum/commands.py index c5255af79d..c74d1301e6 100644 --- a/electrum/commands.py +++ b/electrum/commands.py @@ -1050,6 +1050,16 @@ async def close_channel(self, channel_point, force=False, wallet: Abstract_Walle coro = wallet.lnworker.force_close_channel(chan_id) if force else wallet.lnworker.close_channel(chan_id) return await coro + @command('w') + async def export_channel_backup(self, channel_point, wallet: Abstract_Wallet = None): + txid, index = channel_point.split(':') + chan_id, _ = channel_id_from_funding_tx(txid, int(index)) + return wallet.lnworker.export_channel_backup(chan_id) + + @command('w') + async def import_channel_backup(self, encrypted, wallet: Abstract_Wallet = None): + return wallet.lnworker.import_channel_backup(encrypted) + @command('wn') async def get_channel_ctx(self, channel_point, iknowwhatimdoing=False, wallet: Abstract_Wallet = None): """ return the current commitment transaction of a channel """ diff --git a/electrum/gui/kivy/main_window.py b/electrum/gui/kivy/main_window.py index aaa7d802ff..373ae3804e 100644 --- a/electrum/gui/kivy/main_window.py +++ b/electrum/gui/kivy/main_window.py @@ -408,6 +408,9 @@ def on_qr(self, data): if data.startswith('bitcoin:'): self.set_URI(data) return + if data.startswith('channel_backup:'): + self.import_channel_backup(data[15:]) + return bolt11_invoice = maybe_extract_bolt11_invoice(data) if bolt11_invoice is not None: self.set_ln_invoice(bolt11_invoice) @@ -727,9 +730,6 @@ def lightning_open_channel_dialog(self): d.open() def lightning_channels_dialog(self): - if not self.wallet.has_lightning(): - self.show_error('Lightning not enabled on this wallet') - return if self._channels_dialog is None: self._channels_dialog = LightningChannelsDialog(self) self._channels_dialog.open() @@ -1303,3 +1303,17 @@ def show_private_key(addr, pk_label, password): self.show_error("Invalid PIN") return self.protected(_("Enter your PIN code in order to decrypt your private key"), show_private_key, (addr, pk_label)) + + def import_channel_backup(self, encrypted): + d = Question(_('Import Channel Backup?'), lambda b: self._import_channel_backup(b, encrypted)) + d.open() + + def _import_channel_backup(self, b, encrypted): + if not b: + return + try: + self.wallet.lnbackups.import_channel_backup(encrypted) + except Exception as e: + self.show_error("failed to import backup" + '\n' + str(e)) + return + self.lightning_channels_dialog() diff --git a/electrum/gui/kivy/uix/dialogs/lightning_channels.py b/electrum/gui/kivy/uix/dialogs/lightning_channels.py index 47d85e0d19..317925b8b3 100644 --- a/electrum/gui/kivy/uix/dialogs/lightning_channels.py +++ b/electrum/gui/kivy/uix/dialogs/lightning_channels.py @@ -198,9 +198,118 @@ text: _('Delete') on_release: root.remove_channel() disabled: not root.is_redeemed + +: + id: popuproot + data: [] + is_closed: False + is_redeemed: False + node_id:'' + short_id:'' + initiator:'' + capacity:'' + funding_txid:'' + closing_txid:'' + state:'' + is_open:False + BoxLayout: + padding: '12dp', '12dp', '12dp', '12dp' + spacing: '12dp' + orientation: 'vertical' + ScrollView: + scroll_type: ['bars', 'content'] + scroll_wheel_distance: dp(114) + BoxLayout: + orientation: 'vertical' + height: self.minimum_height + size_hint_y: None + spacing: '5dp' + BoxLabel: + text: _('Channel ID') + value: root.short_id + BoxLabel: + text: _('State') + value: root.state + BoxLabel: + text: _('Initiator') + value: root.initiator + BoxLabel: + text: _('Capacity') + value: root.capacity + Widget: + size_hint: 1, 0.1 + TopLabel: + text: _('Remote Node ID') + TxHashLabel: + data: root.node_id + name: _('Remote Node ID') + TopLabel: + text: _('Funding Transaction') + TxHashLabel: + data: root.funding_txid + name: _('Funding Transaction') + touch_callback: lambda: app.show_transaction(root.funding_txid) + TopLabel: + text: _('Closing Transaction') + opacity: int(bool(root.closing_txid)) + TxHashLabel: + opacity: int(bool(root.closing_txid)) + data: root.closing_txid + name: _('Closing Transaction') + touch_callback: lambda: app.show_transaction(root.closing_txid) + Widget: + size_hint: 1, 0.1 + Widget: + size_hint: 1, 0.05 + BoxLayout: + size_hint: 1, None + height: '48dp' + Button: + size_hint: 0.5, None + height: '48dp' + text: _('Request force-close') + on_release: root.request_force_close() + disabled: root.is_closed + Button: + size_hint: 0.5, None + height: '48dp' + text: _('Delete') + on_release: root.remove_backup() ''') +class ChannelBackupPopup(Popup): + + def __init__(self, chan, app, **kwargs): + super(ChannelBackupPopup,self).__init__(**kwargs) + self.chan = chan + self.app = app + + def request_force_close(self): + msg = _('Request force close?') + Question(msg, self._request_force_close).open() + + def _request_force_close(self, b): + if not b: + return + loop = self.app.wallet.network.asyncio_loop + coro = asyncio.run_coroutine_threadsafe(self.app.wallet.lnbackups.request_force_close(self.chan.channel_id), loop) + try: + coro.result(5) + self.app.show_info(_('Channel closed')) + except Exception as e: + self.app.show_info(_('Could not close channel: ') + repr(e)) # repr because str(Exception()) == '' + + def remove_backup(self): + msg = _('Delete backup?') + Question(msg, self._remove_backup).open() + + def _remove_backup(self, b): + if not b: + return + self.app.wallet.lnbackups.remove_channel_backup(self.chan.channel_id) + self.dismiss() + class ChannelDetailsPopup(Popup): def __init__(self, chan, app, **kwargs): @@ -282,7 +391,11 @@ def __init__(self, app: 'ElectrumWindow'): self.update() def show_item(self, obj): - p = ChannelDetailsPopup(obj._chan, self.app) + chan = obj._chan + if chan.is_backup(): + p = ChannelBackupPopup(chan, self.app) + else: + p = ChannelDetailsPopup(chan, self.app) p.open() def format_fields(self, chan): @@ -305,7 +418,7 @@ def format_fields(self, chan): def update_item(self, item): chan = item._chan item.status = chan.get_state_for_GUI() - item.short_channel_id = format_short_channel_id(chan.short_channel_id) + item.short_channel_id = chan.short_id_for_GUI() l, r = self.format_fields(chan) item.local_balance = _('Local') + ':' + l item.remote_balance = _('Remote') + ': ' + r @@ -317,10 +430,13 @@ def update(self): if not self.app.wallet: return lnworker = self.app.wallet.lnworker - for i in lnworker.channels.values(): + channels = list(lnworker.channels.values()) if lnworker else [] + lnbackups = self.app.wallet.lnbackups + backups = list(lnbackups.channel_backups.values()) + for i in channels + backups: item = Factory.LightningChannelItem() item.screen = self - item.active = i.node_id in lnworker.peers + item.active = i.node_id in (lnworker.peers if lnworker else []) item._chan = i self.update_item(item) channel_cards.add_widget(item) @@ -328,5 +444,7 @@ def update(self): def update_can_send(self): lnworker = self.app.wallet.lnworker + if not lnworker: + return self.can_send = self.app.format_amount_and_units(lnworker.num_sats_can_send()) self.can_receive = self.app.format_amount_and_units(lnworker.num_sats_can_receive()) diff --git a/electrum/gui/qt/__init__.py b/electrum/gui/qt/__init__.py index 1b2a6055f7..a64b126425 100644 --- a/electrum/gui/qt/__init__.py +++ b/electrum/gui/qt/__init__.py @@ -235,7 +235,6 @@ def _create_window_for_wallet(self, wallet): run_hook('on_new_window', w) w.warn_if_testnet() w.warn_if_watching_only() - w.warn_if_lightning_backup() return w def count_wizards_in_progress(func): diff --git a/electrum/gui/qt/channels_list.py b/electrum/gui/qt/channels_list.py index 609f24bf39..24bf0dd913 100644 --- a/electrum/gui/qt/channels_list.py +++ b/electrum/gui/qt/channels_list.py @@ -57,6 +57,7 @@ def __init__(self, parent): self.update_single_row.connect(self.do_update_single_row) self.network = self.parent.network self.lnworker = self.parent.wallet.lnworker + self.lnbackups = self.parent.wallet.lnbackups self.setSortingEnabled(True) def format_fields(self, chan): @@ -78,7 +79,7 @@ def format_fields(self, chan): else: node_alias = '' return [ - format_short_channel_id(chan.short_channel_id), + chan.short_id_for_GUI(), bh2u(chan.node_id), node_alias, '' if closed else labels[LOCAL], @@ -106,14 +107,11 @@ def task(): def force_close(self, channel_id): chan = self.lnworker.channels[channel_id] to_self_delay = chan.config[REMOTE].to_self_delay - if self.lnworker.wallet.is_lightning_backup(): - msg = _('WARNING: force-closing from an old state might result in fund loss.\nAre you sure?') - else: - msg = _('Force-close channel?') + '\n\n'\ - + _(f'Funds retrieved from this channel will not be available before {to_self_delay} blocks after forced closure.') + ' '\ - + _('After that delay, funds will be sent to an address derived from your wallet seed.') + '\n\n'\ - + _('In the meantime, channel funds will not be recoverable from your seed, and will be lost if you lose your wallet.') + ' '\ - + _('To prevent that, you should backup your wallet if you have not already done so.') + msg = _('Force-close channel?') + '\n\n'\ + + _(f'Funds retrieved from this channel will not be available before {to_self_delay} blocks after forced closure.') + ' '\ + + _('After that delay, funds will be sent to an address derived from your wallet seed.') + '\n\n'\ + + _('In the meantime, channel funds will not be recoverable from your seed, and might be lost if you lose your wallet.') + ' '\ + + _('To prevent that, you should have a backup of this channel on another device.') if self.parent.question(msg): def task(): coro = self.lnworker.force_close_channel(channel_id) @@ -124,6 +122,22 @@ def remove_channel(self, channel_id): if self.main_window.question(_('Are you sure you want to delete this channel? This will purge associated transactions from your wallet history.')): self.lnworker.remove_channel(channel_id) + def remove_channel_backup(self, channel_id): + if self.main_window.question(_('Remove channel backup?')): + self.lnbackups.remove_channel_backup(channel_id) + + def export_channel_backup(self, channel_id): + data = self.lnworker.export_channel_backup(channel_id) + self.main_window.show_qrcode('channel_backup:' + data, 'channel backup') + + def request_force_close(self, channel_id): + def task(): + coro = self.lnbackups.request_force_close(channel_id) + return self.network.run_from_another_thread(coro) + def on_success(b): + self.main_window.show_message('success') + WaitingDialog(self, 'please wait..', task, on_success, self.on_failure) + def create_menu(self, position): menu = QMenu() menu.setSeparatorsCollapsible(True) # consecutive separators are merged together @@ -140,6 +154,11 @@ def create_menu(self, position): if not item: return channel_id = idx.sibling(idx.row(), self.Columns.NODE_ID).data(ROLE_CHANNEL_ID) + if channel_id in self.lnbackups.channel_backups: + menu.addAction(_("Request force-close"), lambda: self.request_force_close(channel_id)) + menu.addAction(_("Delete"), lambda: self.remove_channel_backup(channel_id)) + menu.exec_(self.viewport().mapToGlobal(position)) + return chan = self.lnworker.channels[channel_id] menu.addAction(_("Details..."), lambda: self.parent.show_channel(channel_id)) cc = self.add_copy_menu(menu, idx) @@ -163,7 +182,6 @@ def create_menu(self, position): if chan.peer_state == peer_states.GOOD: menu.addAction(_("Close channel"), lambda: self.close_channel(channel_id)) menu.addAction(_("Force-close channel"), lambda: self.force_close(channel_id)) - menu.addSeparator() else: item = chan.get_closing_height() if item: @@ -171,6 +189,8 @@ def create_menu(self, position): closing_tx = self.lnworker.lnwatcher.db.get_transaction(txid) if closing_tx: menu.addAction(_("View closing transaction"), lambda: self.parent.show_transaction(closing_tx)) + menu.addSeparator() + menu.addAction(_("Export backup"), lambda: self.export_channel_backup(channel_id)) if chan.is_redeemed(): menu.addSeparator() menu.addAction(_("Delete"), lambda: self.remove_channel(channel_id)) @@ -195,13 +215,13 @@ def do_update_single_row(self, chan: Channel): def do_update_rows(self, wallet): if wallet != self.parent.wallet: return - lnworker = self.parent.wallet.lnworker - if not lnworker: - return - self.update_can_send(lnworker) + channels = list(wallet.lnworker.channels.values()) if wallet.lnworker else [] + backups = list(wallet.lnbackups.channel_backups.values()) + if wallet.lnworker: + self.update_can_send(wallet.lnworker) self.model().clear() self.update_headers(self.headers) - for chan in lnworker.channels.values(): + for chan in channels + backups: items = [QtGui.QStandardItem(x) for x in self.format_fields(chan)] self.set_editability(items) if self._default_item_bg_brush is None: @@ -212,6 +232,7 @@ def do_update_rows(self, wallet): items[self.Columns.REMOTE_BALANCE].setFont(QFont(MONOSPACE_FONT)) self._update_chan_frozen_bg(chan=chan, items=items) self.model().insertRow(0, items) + self.sortByColumn(self.Columns.SHORT_CHANID, Qt.DescendingOrder) def _update_chan_frozen_bg(self, *, chan: Channel, items: Sequence[QStandardItem]): diff --git a/electrum/gui/qt/main_window.py b/electrum/gui/qt/main_window.py index ba5b008716..3db2bbb9f7 100644 --- a/electrum/gui/qt/main_window.py +++ b/electrum/gui/qt/main_window.py @@ -221,8 +221,7 @@ def add_optional_tab(tabs, tab, icon, description, name): tabs.addTab(tab, icon, description.replace("&", "")) add_optional_tab(tabs, self.addresses_tab, read_QIcon("tab_addresses.png"), _("&Addresses"), "addresses") - if self.wallet.has_lightning(): - add_optional_tab(tabs, self.channels_tab, read_QIcon("lightning.png"), _("Channels"), "channels") + add_optional_tab(tabs, self.channels_tab, read_QIcon("lightning.png"), _("Channels"), "channels") add_optional_tab(tabs, self.utxo_tab, read_QIcon("tab_coins.png"), _("Co&ins"), "utxo") add_optional_tab(tabs, self.contacts_tab, read_QIcon("tab_contacts.png"), _("Con&tacts"), "contacts") add_optional_tab(tabs, self.console_tab, read_QIcon("tab_console.png"), _("Con&sole"), "console") @@ -524,18 +523,6 @@ def warn_if_watching_only(self): ]) self.show_warning(msg, title=_('Watch-only wallet')) - def warn_if_lightning_backup(self): - if self.wallet.is_lightning_backup(): - msg = '\n\n'.join([ - _("This file is a backup of a lightning wallet."), - _("You will not be able to perform lightning payments using this file, and the lightning balance displayed in this wallet might be outdated.") + ' ' + \ - _("If you have lost the original wallet file, you can use this file to trigger a forced closure of your channels."), - _("Do you want to have your channels force-closed?") - ]) - if self.question(msg, title=_('Lightning Backup')): - self.network.maybe_init_lightning() - self.wallet.lnworker.start_network(self.network) - def warn_if_testnet(self): if not constants.net.TESTNET: return @@ -572,14 +559,44 @@ def open_wallet(self): return self.gui_object.new_window(filename) + def select_backup_dir(self, b): + name = self.config.get('backup_dir', '') + dirname = QFileDialog.getExistingDirectory(self, "Select your SSL certificate file", name) + if dirname: + self.config.set_key('backup_dir', dirname) + self.backup_dir_e.setText(dirname) + def backup_wallet(self): + d = WindowModalDialog(self, _("File Backup")) + vbox = QVBoxLayout(d) + grid = QGridLayout() + backup_help = "" + backup_dir = self.config.get('backup_dir') + backup_dir_label = HelpLabel(_('Backup directory') + ':', backup_help) + msg = _('Please select a backup directory') + if self.wallet.lnworker and self.wallet.lnworker.channels: + msg += '\n\n' + ' '.join([ + _("Note that lightning channels will be converted to channel backups."), + _("You cannot use channel backups to perform lightning payments."), + _("Channel backups can only be used to request your channels to be closed.") + ]) + self.backup_dir_e = QPushButton(backup_dir) + self.backup_dir_e.clicked.connect(self.select_backup_dir) + grid.addWidget(backup_dir_label, 1, 0) + grid.addWidget(self.backup_dir_e, 1, 1) + vbox.addLayout(grid) + vbox.addWidget(WWLabel(msg)) + vbox.addLayout(Buttons(CancelButton(d), OkButton(d))) + if not d.exec_(): + return try: new_path = self.wallet.save_backup() except BaseException as reason: self.show_critical(_("Electrum was unable to copy your wallet file to the specified location.") + "\n" + str(reason), title=_("Unable to create backup")) return if new_path: - self.show_message(_("A copy of your wallet file was created in")+" '%s'" % str(new_path), title=_("Wallet backup created")) + msg = _("A copy of your wallet file was created in")+" '%s'" % str(new_path) + self.show_message(msg, title=_("Wallet backup created")) else: self.show_message(_("You need to configure a backup directory in your preferences"), title=_("Backup not created")) @@ -2524,6 +2541,15 @@ def tx_from_text(self, data: Union[str, bytes]) -> Union[None, 'PartialTransacti self.show_critical(_("Electrum was unable to parse your transaction") + ":\n" + repr(e)) return + def import_channel_backup(self, encrypted): + if not self.question('Import channel backup?'): + return + try: + self.wallet.lnbackups.import_channel_backup(encrypted) + except Exception as e: + self.show_error("failed to import backup" + '\n' + str(e)) + return + def read_tx_from_qrcode(self): from electrum import qrscanner try: @@ -2537,6 +2563,9 @@ def read_tx_from_qrcode(self): if str(data).startswith("bitcoin:"): self.pay_to_URI(data) return + if data.startswith('channel_backup:'): + self.import_channel_backup(data[15:]) + return # else if the user scanned an offline signed tx tx = self.tx_from_text(data) if not tx: diff --git a/electrum/gui/qt/settings_dialog.py b/electrum/gui/qt/settings_dialog.py index 0a3c005fb6..d95257e643 100644 --- a/electrum/gui/qt/settings_dialog.py +++ b/electrum/gui/qt/settings_dialog.py @@ -146,13 +146,6 @@ def on_batch_rbf(x): # lightning lightning_widgets = [] - backup_help = _("""If you configure a backup directory, a backup of your wallet file will be saved everytime you create a new channel.\n\nA backup file cannot be used as a wallet; it can only be used to retrieve the funds locked in your channels, by requesting your channels to be force closed (using data loss protection).\n\nIf the remote node is online, they will force-close your channels when you open the backup file. Note that a backup is not strictly necessary for that; if the remote party is online but they cannot reach you because you lost your wallet file, they should eventually close your channels, and your funds should be sent to an address recoverable from your seed (using static_remotekey).\n\nIf the remote node is not online, you can use the backup file to force close your channels, but only at the risk of losing all your funds in the channel, because you will be broadcasting an old state.""") - backup_dir = self.config.get('backup_dir') - backup_dir_label = HelpLabel(_('Backup directory') + ':', backup_help) - self.backup_dir_e = QPushButton(backup_dir) - self.backup_dir_e.clicked.connect(self.select_backup_dir) - lightning_widgets.append((backup_dir_label, self.backup_dir_e)) - help_persist = _("""If this option is checked, Electrum will persist as a daemon after you close all your wallet windows. Your local watchtower will keep running, and it will protect your channels even if your wallet is not @@ -554,13 +547,6 @@ def on_alias_edit(self): if alias: self.window.fetch_alias() - def select_backup_dir(self, b): - name = self.config.get('backup_dir', '') - dirname = QFileDialog.getExistingDirectory(self, "Select your SSL certificate file", name) - if dirname: - self.config.set_key('backup_dir', dirname) - self.backup_dir_e.setText(dirname) - def select_ssl_certfile(self, b): name = self.config.get('ssl_certfile', '') filename, __ = QFileDialog.getOpenFileName(self, "Select your SSL certificate file", name) diff --git a/electrum/lnchannel.py b/electrum/lnchannel.py index ab28dad292..7ab8d9759b 100644 --- a/electrum/lnchannel.py +++ b/electrum/lnchannel.py @@ -56,6 +56,8 @@ from .lnmsg import encode_msg, decode_msg from .address_synchronizer import TX_HEIGHT_LOCAL from .lnutil import CHANNEL_OPENING_TIMEOUT +from .lnutil import ChannelBackupStorage +from .lnutil import format_short_channel_id if TYPE_CHECKING: from .lnworker import LNWallet @@ -121,19 +123,256 @@ def htlcsum(htlcs): return sum([x.amount_msat for x in htlcs]) -class Channel(Logger): +class AbstractChannel(Logger): + + def set_short_channel_id(self, short_id: ShortChannelID) -> None: + self.short_channel_id = short_id + self.storage["short_channel_id"] = short_id + + def get_id_for_log(self) -> str: + scid = self.short_channel_id + if scid: + return str(scid) + return self.channel_id.hex() + + def set_state(self, state: channel_states) -> None: + """ set on-chain state """ + old_state = self._state + if (old_state, state) not in state_transitions: + raise Exception(f"Transition not allowed: {old_state.name} -> {state.name}") + self.logger.debug(f'Setting channel state: {old_state.name} -> {state.name}') + self._state = state + self.storage['state'] = self._state.name + + def get_state(self) -> channel_states: + return self._state + + def is_open(self): + return self.get_state() == channel_states.OPEN + + def is_closing(self): + return self.get_state() in [channel_states.CLOSING, channel_states.FORCE_CLOSING] + + def is_closed(self): + # the closing txid has been saved + return self.get_state() >= channel_states.CLOSED + + def is_redeemed(self): + return self.get_state() == channel_states.REDEEMED + + def save_funding_height(self, txid, height, timestamp): + self.storage['funding_height'] = txid, height, timestamp + + def get_funding_height(self): + return self.storage.get('funding_height') + + def delete_funding_height(self): + self.storage.pop('funding_height', None) + + def save_closing_height(self, txid, height, timestamp): + self.storage['closing_height'] = txid, height, timestamp + + def get_closing_height(self): + return self.storage.get('closing_height') + + def delete_closing_height(self): + self.storage.pop('closing_height', None) + + def create_sweeptxs_for_our_ctx(self, ctx): + return create_sweeptxs_for_our_ctx(chan=self, ctx=ctx, sweep_address=self.sweep_address) + + def create_sweeptxs_for_their_ctx(self, ctx): + return create_sweeptxs_for_their_ctx(chan=self, ctx=ctx, sweep_address=self.sweep_address) + + def is_backup(self): + return False + + def sweep_ctx(self, ctx: Transaction) -> Dict[str, SweepInfo]: + txid = ctx.txid() + if self.sweep_info.get(txid) is None: + our_sweep_info = self.create_sweeptxs_for_our_ctx(ctx) + their_sweep_info = self.create_sweeptxs_for_their_ctx(ctx) + if our_sweep_info is not None: + self.sweep_info[txid] = our_sweep_info + self.logger.info(f'we force closed') + elif their_sweep_info is not None: + self.sweep_info[txid] = their_sweep_info + self.logger.info(f'they force closed.') + else: + self.sweep_info[txid] = {} + self.logger.info(f'not sure who closed.') + return self.sweep_info[txid] + + # ancestor for Channel and ChannelBackup + def update_onchain_state(self, funding_txid, funding_height, closing_txid, closing_height, keep_watching): + # note: state transitions are irreversible, but + # save_funding_height, save_closing_height are reversible + if funding_height.height == TX_HEIGHT_LOCAL: + self.update_unfunded_state() + elif closing_height.height == TX_HEIGHT_LOCAL: + self.update_funded_state(funding_txid, funding_height) + else: + self.update_closed_state(funding_txid, funding_height, closing_txid, closing_height, keep_watching) + + def update_unfunded_state(self): + self.delete_funding_height() + self.delete_closing_height() + if self.get_state() in [channel_states.PREOPENING, channel_states.OPENING, channel_states.FORCE_CLOSING] and self.lnworker: + if self.is_initiator(): + # set channel state to REDEEMED so that it can be removed manually + # to protect ourselves against a server lying by omission, + # we check that funding_inputs have been double spent and deeply mined + inputs = self.storage.get('funding_inputs', []) + if not inputs: + self.logger.info(f'channel funding inputs are not provided') + self.set_state(channel_states.REDEEMED) + for i in inputs: + spender_txid = self.lnworker.wallet.db.get_spent_outpoint(*i) + if spender_txid is None: + continue + if spender_txid != self.funding_outpoint.txid: + tx_mined_height = self.lnworker.wallet.get_tx_height(spender_txid) + if tx_mined_height.conf > lnutil.REDEEM_AFTER_DOUBLE_SPENT_DELAY: + self.logger.info(f'channel is double spent {inputs}') + self.set_state(channel_states.REDEEMED) + break + else: + now = int(time.time()) + if now - self.storage.get('init_timestamp', 0) > CHANNEL_OPENING_TIMEOUT: + self.lnworker.remove_channel(self.channel_id) + + def update_funded_state(self, funding_txid, funding_height): + self.save_funding_height(funding_txid, funding_height.height, funding_height.timestamp) + self.delete_closing_height() + if self.get_state() == channel_states.OPENING: + if self.is_funding_tx_mined(funding_height): + self.set_state(channel_states.FUNDED) + self.set_short_channel_id(ShortChannelID.from_components( + funding_height.height, funding_height.txpos, self.funding_outpoint.output_index)) + self.logger.info(f"save_short_channel_id: {self.short_channel_id}") + + def update_closed_state(self, funding_txid, funding_height, closing_txid, closing_height, keep_watching): + self.save_funding_height(funding_txid, funding_height.height, funding_height.timestamp) + self.save_closing_height(closing_txid, closing_height.height, closing_height.timestamp) + if self.get_state() < channel_states.CLOSED: + conf = closing_height.conf + if conf > 0: + self.set_state(channel_states.CLOSED) + else: + # we must not trust the server with unconfirmed transactions + # if the remote force closed, we remain OPEN until the closing tx is confirmed + pass + if self.get_state() == channel_states.CLOSED and not keep_watching: + self.set_state(channel_states.REDEEMED) + + +class ChannelBackup(AbstractChannel): + """ + current capabilities: + - detect force close + - request force close + - sweep my ctx to_local + future: + - will need to sweep their ctx to_remote + """ + + def __init__(self, cb: ChannelBackupStorage, *, sweep_address=None, lnworker=None): + self.name = None + Logger.__init__(self) + self.cb = cb + self.sweep_info = {} # type: Dict[str, Dict[str, SweepInfo]] + self.sweep_address = sweep_address + self.storage = {} # dummy storage + self._state = channel_states.OPENING + self.config = {} + self.config[LOCAL] = LocalConfig.from_seed( + channel_seed=cb.channel_seed, + to_self_delay=cb.local_delay, + # dummy values + static_remotekey=None, + dust_limit_sat=None, + max_htlc_value_in_flight_msat=None, + max_accepted_htlcs=None, + initial_msat=None, + reserve_sat=None, + funding_locked_received=False, + was_announced=False, + current_commitment_signature=None, + current_htlc_signatures=b'', + htlc_minimum_msat=1, + ) + self.config[REMOTE] = RemoteConfig( + payment_basepoint=OnlyPubkeyKeypair(cb.remote_payment_pubkey), + revocation_basepoint=OnlyPubkeyKeypair(cb.remote_revocation_pubkey), + to_self_delay=cb.remote_delay, + # dummy values + multisig_key=OnlyPubkeyKeypair(None), + htlc_basepoint=OnlyPubkeyKeypair(None), + delayed_basepoint=OnlyPubkeyKeypair(None), + dust_limit_sat=None, + max_htlc_value_in_flight_msat=None, + max_accepted_htlcs=None, + initial_msat = None, + reserve_sat = None, + htlc_minimum_msat=None, + next_per_commitment_point=None, + current_per_commitment_point=None) + + self.node_id = cb.node_id + self.channel_id = cb.channel_id() + self.funding_outpoint = cb.funding_outpoint() + self.lnworker = lnworker + self.short_channel_id = None + + def is_backup(self): + return True + + def create_sweeptxs_for_their_ctx(self, ctx): + return {} + + def get_funding_address(self): + return self.cb.funding_address + + def short_id_for_GUI(self) -> str: + return 'BACKUP' + + def is_initiator(self): + return self.cb.is_initiator + + def get_state_for_GUI(self): + cs = self.get_state() + return cs.name + + def get_oldest_unrevoked_ctn(self, who): + return -1 + + def included_htlcs(self, subject, direction, ctn): + return [] + + def funding_txn_minimum_depth(self): + return 1 + + def is_funding_tx_mined(self, funding_height): + return funding_height.conf > 1 + + def balance_minus_outgoing_htlcs(self, whose: HTLCOwner, *, ctx_owner: HTLCOwner = HTLCOwner.LOCAL, ctn: int = None): + return 0 + + def balance(self, whose: HTLCOwner, *, ctx_owner=HTLCOwner.LOCAL, ctn: int = None) -> int: + return 0 + + def is_frozen_for_sending(self) -> bool: + return False + + def is_frozen_for_receiving(self) -> bool: + return False + + +class Channel(AbstractChannel): # note: try to avoid naming ctns/ctxs/etc as "current" and "pending". # they are ambiguous. Use "oldest_unrevoked" or "latest" or "next". # TODO enforce this ^ - def diagnostic_name(self): - if self.name: - return str(self.name) - try: - return f"lnchannel_{bh2u(self.channel_id[-4:])}" - except: - return super().diagnostic_name() - def __init__(self, state: 'StoredDict', *, sweep_address=None, name=None, lnworker=None, initial_feerate=None): self.name = name Logger.__init__(self) @@ -162,11 +401,22 @@ def __init__(self, state: 'StoredDict', *, sweep_address=None, name=None, lnwork self._receive_fail_reasons = {} # type: Dict[int, BarePaymentAttemptLog] self._ignore_max_htlc_value = False # used in tests - def get_id_for_log(self) -> str: - scid = self.short_channel_id - if scid: - return str(scid) - return self.channel_id.hex() + def short_id_for_GUI(self) -> str: + return format_short_channel_id(self.short_channel_id) + + def is_initiator(self): + return self.constraints.is_initiator + + def funding_txn_minimum_depth(self): + return self.constraints.funding_txn_minimum_depth + + def diagnostic_name(self): + if self.name: + return str(self.name) + try: + return f"lnchannel_{bh2u(self.channel_id[-4:])}" + except: + return super().diagnostic_name() def set_onion_key(self, key: int, value: bytes): self.onion_keys[key] = value @@ -269,10 +519,6 @@ def construct_channel_announcement_without_sigs(self) -> bytes: def is_static_remotekey_enabled(self) -> bool: return bool(self.storage.get('static_remotekey_enabled')) - def set_short_channel_id(self, short_id: ShortChannelID) -> None: - self.short_channel_id = short_id - self.storage["short_channel_id"] = short_id - def get_feerate(self, subject: HTLCOwner, *, ctn: int) -> int: # returns feerate in sat/kw return self.hm.get_feerate(subject, ctn) @@ -322,21 +568,11 @@ def open_with_first_pcp(self, remote_pcp: bytes, remote_sig: bytes) -> None: self.peer_state = peer_states.GOOD def set_state(self, state: channel_states) -> None: - """ set on-chain state """ - old_state = self._state - if (old_state, state) not in state_transitions: - raise Exception(f"Transition not allowed: {old_state.name} -> {state.name}") - self.logger.debug(f'Setting channel state: {old_state.name} -> {state.name}') - self._state = state - self.storage['state'] = self._state.name - + super().set_state(state) if self.lnworker: self.lnworker.save_channel(self) self.lnworker.network.trigger_callback('channel', self) - def get_state(self) -> channel_states: - return self._state - def get_state_for_GUI(self): # status displayed in the GUI cs = self.get_state() @@ -347,16 +583,6 @@ def get_state_for_GUI(self): return ps.name return cs.name - def is_open(self): - return self.get_state() == channel_states.OPEN - - def is_closing(self): - return self.get_state() in [channel_states.CLOSING, channel_states.FORCE_CLOSING] - - def is_closed(self): - # the closing txid has been saved - return self.get_state() >= channel_states.CLOSED - def set_can_send_ctx_updates(self, b: bool) -> None: self._can_send_ctx_updates = b @@ -373,27 +599,6 @@ def can_send_ctx_updates(self) -> bool: def can_send_update_add_htlc(self) -> bool: return self.can_send_ctx_updates() and not self.is_closing() - def save_funding_height(self, txid, height, timestamp): - self.storage['funding_height'] = txid, height, timestamp - - def get_funding_height(self): - return self.storage.get('funding_height') - - def delete_funding_height(self): - self.storage.pop('funding_height', None) - - def save_closing_height(self, txid, height, timestamp): - self.storage['closing_height'] = txid, height, timestamp - - def get_closing_height(self): - return self.storage.get('closing_height') - - def delete_closing_height(self): - self.storage.pop('closing_height', None) - - def is_redeemed(self): - return self.get_state() == channel_states.REDEEMED - def is_frozen_for_sending(self) -> bool: """Whether the user has marked this channel as frozen for sending. Frozen channels are not supposed to be used for new outgoing payments. @@ -1039,21 +1244,6 @@ def force_close_tx(self) -> PartialTransaction: assert tx.is_complete() return tx - def sweep_ctx(self, ctx: Transaction) -> Dict[str, SweepInfo]: - txid = ctx.txid() - if self.sweep_info.get(txid) is None: - our_sweep_info = create_sweeptxs_for_our_ctx(chan=self, ctx=ctx, sweep_address=self.sweep_address) - their_sweep_info = create_sweeptxs_for_their_ctx(chan=self, ctx=ctx, sweep_address=self.sweep_address) - if our_sweep_info is not None: - self.sweep_info[txid] = our_sweep_info - self.logger.info(f'we force closed.') - elif their_sweep_info is not None: - self.sweep_info[txid] = their_sweep_info - self.logger.info(f'they force closed.') - else: - self.sweep_info[txid] = {} - return self.sweep_info[txid] - def sweep_htlc(self, ctx: Transaction, htlc_tx: Transaction) -> Optional[SweepInfo]: # look at the output address, check if it matches return create_sweeptx_for_their_revoked_htlc(self, ctx, htlc_tx, self.sweep_address) @@ -1095,16 +1285,6 @@ def should_be_closed_due_to_expiring_htlcs(self, local_height) -> bool: 500_000) return total_value_sat > min_value_worth_closing_channel_over_sat - def update_onchain_state(self, funding_txid, funding_height, closing_txid, closing_height, keep_watching): - # note: state transitions are irreversible, but - # save_funding_height, save_closing_height are reversible - if funding_height.height == TX_HEIGHT_LOCAL: - self.update_unfunded_state() - elif closing_height.height == TX_HEIGHT_LOCAL: - self.update_funded_state(funding_txid, funding_height) - else: - self.update_closed_state(funding_txid, funding_height, closing_txid, closing_height, keep_watching) - def is_funding_tx_mined(self, funding_height): """ Checks if Funding TX has been mined. If it has, save the short channel ID in chan; @@ -1114,7 +1294,7 @@ def is_funding_tx_mined(self, funding_height): funding_txid = self.funding_outpoint.txid funding_idx = self.funding_outpoint.output_index conf = funding_height.conf - if conf < self.constraints.funding_txn_minimum_depth: + if conf < self.funding_txn_minimum_depth(): self.logger.info(f"funding tx is still not at sufficient depth. actual depth: {conf}") return False assert conf > 0 @@ -1132,53 +1312,3 @@ def is_funding_tx_mined(self, funding_height): return False return True - def update_unfunded_state(self): - self.delete_funding_height() - self.delete_closing_height() - if self.get_state() in [channel_states.PREOPENING, channel_states.OPENING, channel_states.FORCE_CLOSING] and self.lnworker: - if self.constraints.is_initiator: - # set channel state to REDEEMED so that it can be removed manually - # to protect ourselves against a server lying by omission, - # we check that funding_inputs have been double spent and deeply mined - inputs = self.storage.get('funding_inputs', []) - if not inputs: - self.logger.info(f'channel funding inputs are not provided') - self.set_state(channel_states.REDEEMED) - for i in inputs: - spender_txid = self.lnworker.wallet.db.get_spent_outpoint(*i) - if spender_txid is None: - continue - if spender_txid != self.funding_outpoint.txid: - tx_mined_height = self.lnworker.wallet.get_tx_height(spender_txid) - if tx_mined_height.conf > lnutil.REDEEM_AFTER_DOUBLE_SPENT_DELAY: - self.logger.info(f'channel is double spent {inputs}') - self.set_state(channel_states.REDEEMED) - break - else: - now = int(time.time()) - if now - self.storage.get('init_timestamp', 0) > CHANNEL_OPENING_TIMEOUT: - self.lnworker.remove_channel(self.channel_id) - - def update_funded_state(self, funding_txid, funding_height): - self.save_funding_height(funding_txid, funding_height.height, funding_height.timestamp) - self.delete_closing_height() - if self.get_state() == channel_states.OPENING: - if self.is_funding_tx_mined(funding_height): - self.set_state(channel_states.FUNDED) - self.set_short_channel_id(ShortChannelID.from_components( - funding_height.height, funding_height.txpos, self.funding_outpoint.output_index)) - self.logger.info(f"save_short_channel_id: {self.short_channel_id}") - - def update_closed_state(self, funding_txid, funding_height, closing_txid, closing_height, keep_watching): - self.save_funding_height(funding_txid, funding_height.height, funding_height.timestamp) - self.save_closing_height(closing_txid, closing_height.height, closing_height.timestamp) - if self.get_state() < channel_states.CLOSED: - conf = closing_height.conf - if conf > 0: - self.set_state(channel_states.CLOSED) - else: - # we must not trust the server with unconfirmed transactions - # if the remote force closed, we remain OPEN until the closing tx is confirmed - pass - if self.get_state() == channel_states.CLOSED and not keep_watching: - self.set_state(channel_states.REDEEMED) diff --git a/electrum/lnpeer.py b/electrum/lnpeer.py index 2bbb21eccd..1c059ffb02 100644 --- a/electrum/lnpeer.py +++ b/electrum/lnpeer.py @@ -44,7 +44,7 @@ MAXIMUM_REMOTE_TO_SELF_DELAY_ACCEPTED, RemoteMisbehaving, DEFAULT_TO_SELF_DELAY, NBLOCK_OUR_CLTV_EXPIRY_DELTA, format_short_channel_id, ShortChannelID, IncompatibleLightningFeatures, derive_payment_secret_from_payment_preimage) -from .lnutil import FeeUpdate +from .lnutil import FeeUpdate, channel_id_from_funding_tx from .lntransport import LNTransport, LNTransportBase from .lnmsg import encode_msg, decode_msg from .interface import GracefulDisconnect, NetworkException @@ -60,10 +60,6 @@ LN_P2P_NETWORK_TIMEOUT = 20 -def channel_id_from_funding_tx(funding_txid: str, funding_index: int) -> Tuple[bytes, bytes]: - funding_txid_bytes = bytes.fromhex(funding_txid)[::-1] - i = int.from_bytes(funding_txid_bytes, 'big') ^ funding_index - return i.to_bytes(32, 'big'), funding_txid_bytes class Peer(Logger): @@ -222,7 +218,7 @@ def on_init(self, payload): if constants.net.rev_genesis_bytes() not in their_chains: raise GracefulDisconnect(f"no common chain found with remote. (they sent: {their_chains})") # all checks passed - if isinstance(self.transport, LNTransport): + if self.channel_db and isinstance(self.transport, LNTransport): self.channel_db.add_recent_peer(self.transport.peer_addr) for chan in self.channels.values(): chan.add_or_update_peer_addr(self.transport.peer_addr) @@ -728,6 +724,17 @@ def validate_remote_reserve(self, remote_reserve_sat: int, dust_limit: int, fund raise Exception(f'reserve too high: {remote_reserve_sat}, funding_sat: {funding_sat}') return remote_reserve_sat + async def trigger_force_close(self, channel_id): + await self.initialized + latest_point = 0 + self.send_message( + "channel_reestablish", + channel_id=channel_id, + next_local_commitment_number=0, + next_remote_revocation_number=0, + your_last_per_commitment_secret=0, + my_current_per_commitment_point=latest_point) + async def reestablish_channel(self, chan: Channel): await self.initialized chan_id = chan.channel_id @@ -749,8 +756,7 @@ async def reestablish_channel(self, chan: Channel): next_remote_ctn = chan.get_next_ctn(REMOTE) assert self.features & LnFeatures.OPTION_DATA_LOSS_PROTECT_OPT # send message - srk_enabled = chan.is_static_remotekey_enabled() - if srk_enabled: + if chan.is_static_remotekey_enabled(): latest_secret, latest_point = chan.get_secret_and_point(LOCAL, 0) else: latest_secret, latest_point = chan.get_secret_and_point(LOCAL, latest_local_ctn) @@ -878,10 +884,6 @@ def are_datalossprotect_fields_valid() -> bool: self.logger.warning(f"channel_reestablish ({chan.get_id_for_log()}): we are ahead of remote! trying to force-close.") await self.lnworker.try_force_closing(chan_id) return - elif self.lnworker.wallet.is_lightning_backup(): - self.logger.warning(f"channel_reestablish ({chan.get_id_for_log()}): force-closing because we are a recent backup") - await self.lnworker.try_force_closing(chan_id) - return chan.peer_state = peer_states.GOOD # note: chan.short_channel_id being set implies the funding txn is already at sufficient depth diff --git a/electrum/lnsweep.py b/electrum/lnsweep.py index 02b70a5734..6f95444084 100644 --- a/electrum/lnsweep.py +++ b/electrum/lnsweep.py @@ -18,7 +18,7 @@ from .transaction import (Transaction, TxOutput, construct_witness, PartialTransaction, PartialTxInput, PartialTxOutput, TxOutpoint) from .simple_config import SimpleConfig -from .logging import get_logger +from .logging import get_logger, Logger if TYPE_CHECKING: from .lnchannel import Channel diff --git a/electrum/lnutil.py b/electrum/lnutil.py index c99c80a074..0d6acb55e8 100644 --- a/electrum/lnutil.py +++ b/electrum/lnutil.py @@ -24,6 +24,7 @@ from .i18n import _ from .lnaddr import lndecode from .bip32 import BIP32Node, BIP32_PRIME +from .transaction import BCDataStream if TYPE_CHECKING: from .lnchannel import Channel @@ -47,6 +48,11 @@ def ln_dummy_address(): from .json_db import StoredObject +def channel_id_from_funding_tx(funding_txid: str, funding_index: int) -> Tuple[bytes, bytes]: + funding_txid_bytes = bytes.fromhex(funding_txid)[::-1] + i = int.from_bytes(funding_txid_bytes, 'big') ^ funding_index + return i.to_bytes(32, 'big'), funding_txid_bytes + hex_to_bytes = lambda v: v if isinstance(v, bytes) else bytes.fromhex(v) if v is not None else None json_to_keypair = lambda v: v if isinstance(v, OnlyPubkeyKeypair) else Keypair(**v) if len(v)==2 else OnlyPubkeyKeypair(**v) @@ -116,6 +122,66 @@ class ChannelConstraints(StoredObject): is_initiator = attr.ib(type=bool) # note: sometimes also called "funder" funding_txn_minimum_depth = attr.ib(type=int) +@attr.s +class ChannelBackupStorage(StoredObject): + node_id = attr.ib(type=bytes, converter=hex_to_bytes) + privkey = attr.ib(type=bytes, converter=hex_to_bytes) + funding_txid = attr.ib(type=str) + funding_index = attr.ib(type=int, converter=int) + funding_address = attr.ib(type=str) + host = attr.ib(type=str) + port = attr.ib(type=int, converter=int) + is_initiator = attr.ib(type=bool) + channel_seed = attr.ib(type=bytes, converter=hex_to_bytes) + local_delay = attr.ib(type=int, converter=int) + remote_delay = attr.ib(type=int, converter=int) + remote_payment_pubkey = attr.ib(type=bytes, converter=hex_to_bytes) + remote_revocation_pubkey = attr.ib(type=bytes, converter=hex_to_bytes) + + def funding_outpoint(self): + return Outpoint(self.funding_txid, self.funding_index) + + def channel_id(self): + chan_id, _ = channel_id_from_funding_tx(self.funding_txid, self.funding_index) + return chan_id + + def to_bytes(self): + vds = BCDataStream() + vds.write_boolean(self.is_initiator) + vds.write_bytes(self.privkey, 32) + vds.write_bytes(self.channel_seed, 32) + vds.write_bytes(self.node_id, 33) + vds.write_bytes(bfh(self.funding_txid), 32) + vds.write_int16(self.funding_index) + vds.write_string(self.funding_address) + vds.write_bytes(self.remote_payment_pubkey, 33) + vds.write_bytes(self.remote_revocation_pubkey, 33) + vds.write_int16(self.local_delay) + vds.write_int16(self.remote_delay) + vds.write_string(self.host) + vds.write_int16(self.port) + return vds.input + + @staticmethod + def from_bytes(s): + vds = BCDataStream() + vds.write(s) + return ChannelBackupStorage( + is_initiator = bool(vds.read_bytes(1)), + privkey = vds.read_bytes(32).hex(), + channel_seed = vds.read_bytes(32).hex(), + node_id = vds.read_bytes(33).hex(), + funding_txid = vds.read_bytes(32).hex(), + funding_index = vds.read_int16(), + funding_address = vds.read_string(), + remote_payment_pubkey = vds.read_bytes(33).hex(), + remote_revocation_pubkey = vds.read_bytes(33).hex(), + local_delay = vds.read_int16(), + remote_delay = vds.read_int16(), + host = vds.read_string(), + port = vds.read_int16()) + + class ScriptHtlc(NamedTuple): redeem_script: bytes @@ -716,8 +782,8 @@ def extract_ctn_from_tx(tx: Transaction, txin_index: int, funder_payment_basepoi return get_obscured_ctn(obs, funder_payment_basepoint, fundee_payment_basepoint) def extract_ctn_from_tx_and_chan(tx: Transaction, chan: 'Channel') -> int: - funder_conf = chan.config[LOCAL] if chan.constraints.is_initiator else chan.config[REMOTE] - fundee_conf = chan.config[LOCAL] if not chan.constraints.is_initiator else chan.config[REMOTE] + funder_conf = chan.config[LOCAL] if chan.is_initiator() else chan.config[REMOTE] + fundee_conf = chan.config[LOCAL] if not chan.is_initiator() else chan.config[REMOTE] return extract_ctn_from_tx(tx, txin_index=0, funder_payment_basepoint=funder_conf.payment_basepoint.pubkey, fundee_payment_basepoint=fundee_conf.payment_basepoint.pubkey) diff --git a/electrum/lnwatcher.py b/electrum/lnwatcher.py index bb66af1874..624546ca5a 100644 --- a/electrum/lnwatcher.py +++ b/electrum/lnwatcher.py @@ -15,6 +15,7 @@ from .sql_db import SqlDB, sql from .wallet_db import WalletDB from .util import bh2u, bfh, log_exceptions, ignore_exceptions +from .lnutil import Outpoint from . import wallet from .storage import WalletStorage from .address_synchronizer import AddressSynchronizer, TX_HEIGHT_LOCAL, TX_HEIGHT_UNCONF_PARENT, TX_HEIGHT_UNCONFIRMED diff --git a/electrum/lnworker.py b/electrum/lnworker.py index 6a697442f0..88cf4f7f72 100644 --- a/electrum/lnworker.py +++ b/electrum/lnworker.py @@ -64,6 +64,9 @@ from .address_synchronizer import TX_HEIGHT_LOCAL from . import lnsweep from .lnwatcher import LNWalletWatcher +from .crypto import pw_encode_bytes, pw_decode_bytes, PW_HASH_VERSION_LATEST +from .lnutil import ChannelBackupStorage +from .lnchannel import ChannelBackup if TYPE_CHECKING: from .network import Network @@ -219,7 +222,8 @@ async def _add_peer(self, host, port, node_id) -> Peer: return peer def peer_closed(self, peer: Peer) -> None: - self.peers.pop(peer.pubkey) + if peer.pubkey in self.peers: + self.peers.pop(peer.pubkey) def num_peers(self) -> int: return sum([p.is_initialized() for p in self.peers.values()]) @@ -492,7 +496,8 @@ def start_network(self, network: 'Network'): self.lnwatcher = LNWalletWatcher(self, network) self.lnwatcher.start_network(network) self.network = network - for chan_id, chan in self.channels.items(): + + for chan in self.channels.values(): self.lnwatcher.add_channel(chan.funding_outpoint.to_str(), chan.get_funding_address()) super().start_network(network) @@ -763,8 +768,6 @@ def mktx_for_open_channel(self, *, coins: Sequence[PartialTxInput], funding_sat: def open_channel(self, *, connect_str: str, funding_tx: PartialTransaction, funding_sat: int, push_amt_sat: int, password: str = None, timeout: Optional[int] = 20) -> Tuple[Channel, PartialTransaction]: - if self.wallet.is_lightning_backup(): - raise Exception(_('Cannot create channel: this is a backup file')) if funding_sat > LN_MAX_FUNDING_SAT: raise Exception(_("Requested channel capacity is over protocol allowed maximum.")) coro = self._open_channel_coroutine(connect_str=connect_str, funding_tx=funding_tx, funding_sat=funding_sat, @@ -1319,3 +1322,100 @@ def current_feerate_per_kw(self): if feerate_per_kvbyte is None: feerate_per_kvbyte = FEERATE_FALLBACK_STATIC_FEE return max(253, feerate_per_kvbyte // 4) + + def create_channel_backup(self, channel_id): + chan = self.channels[channel_id] + peer_addresses = list(chan.get_peer_addresses()) + peer_addr = peer_addresses[0] + return ChannelBackupStorage( + node_id = chan.node_id, + privkey = self.node_keypair.privkey, + funding_txid = chan.funding_outpoint.txid, + funding_index = chan.funding_outpoint.output_index, + funding_address = chan.get_funding_address(), + host = peer_addr.host, + port = peer_addr.port, + is_initiator = chan.constraints.is_initiator, + channel_seed = chan.config[LOCAL].channel_seed, + local_delay = chan.config[LOCAL].to_self_delay, + remote_delay = chan.config[REMOTE].to_self_delay, + remote_revocation_pubkey = chan.config[REMOTE].revocation_basepoint.pubkey, + remote_payment_pubkey = chan.config[REMOTE].payment_basepoint.pubkey) + + def export_channel_backup(self, channel_id): + xpub = self.wallet.get_fingerprint() + backup_bytes = self.create_channel_backup(channel_id).to_bytes() + assert backup_bytes == ChannelBackupStorage.from_bytes(backup_bytes).to_bytes(), "roundtrip failed" + encrypted = pw_encode_bytes(backup_bytes, xpub, version=PW_HASH_VERSION_LATEST) + assert backup_bytes == pw_decode_bytes(encrypted, xpub, version=PW_HASH_VERSION_LATEST), "encrypt failed" + return encrypted + + +class LNBackups(Logger): + + def __init__(self, wallet: 'Abstract_Wallet'): + Logger.__init__(self) + self.features = LnFeatures(0) + self.features |= LnFeatures.OPTION_DATA_LOSS_PROTECT_OPT + self.features |= LnFeatures.OPTION_STATIC_REMOTEKEY_OPT + self.taskgroup = SilentTaskGroup() + self.lock = threading.RLock() + self.wallet = wallet + self.db = wallet.db + self.sweep_address = wallet.get_receiving_address() + self.channel_backups = {} + for channel_id, cb in self.db.get_dict("channel_backups").items(): + self.channel_backups[bfh(channel_id)] = ChannelBackup(cb, sweep_address=self.sweep_address, lnworker=self) + + def peer_closed(self, chan): + pass + + async def on_channel_update(self, chan): + pass + + def channel_by_txo(self, txo): + with self.lock: + channel_backups = list(self.channel_backups.values()) + for chan in channel_backups: + if chan.funding_outpoint.to_str() == txo: + return chan + + def start_network(self, network: 'Network'): + assert network + self.lnwatcher = LNWalletWatcher(self, network) + self.lnwatcher.start_network(network) + self.network = network + for cb in self.channel_backups.values(): + self.lnwatcher.add_channel(cb.funding_outpoint.to_str(), cb.get_funding_address()) + + def import_channel_backup(self, encrypted): + xpub = self.wallet.get_fingerprint() + x = pw_decode_bytes(encrypted, xpub, version=PW_HASH_VERSION_LATEST) + cb = ChannelBackupStorage.from_bytes(x) + channel_id = cb.channel_id().hex() + d = self.db.get_dict("channel_backups") + if channel_id in d: + raise Exception('Channel already in wallet') + d[channel_id] = cb + self.channel_backups[bfh(channel_id)] = ChannelBackup(cb, sweep_address=self.sweep_address, lnworker=self) + self.wallet.save_db() + self.network.trigger_callback('channels_updated', self.wallet) + + def remove_channel_backup(self, channel_id): + d = self.db.get_dict("channel_backups") + if channel_id.hex() not in d: + raise Exception('Channel not found') + d.pop(channel_id.hex()) + self.channel_backups.pop(channel_id) + self.wallet.save_db() + self.network.trigger_callback('channels_updated', self.wallet) + + @log_exceptions + async def request_force_close(self, channel_id): + cb = self.channel_backups[channel_id].cb + peer_addr = LNPeerAddr(cb.host, cb.port, cb.node_id) + transport = LNTransport(cb.privkey, peer_addr) + peer = Peer(self, cb.node_id, transport) + await self.taskgroup.spawn(peer._message_loop()) + await peer.initialized + await self.taskgroup.spawn(peer.trigger_force_close(channel_id)) diff --git a/electrum/transaction.py b/electrum/transaction.py index 0df616a292..5780d64b95 100644 --- a/electrum/transaction.py +++ b/electrum/transaction.py @@ -289,6 +289,10 @@ def read_bytes(self, length: int) -> bytes: else: raise SerializationError('attempt to read past end of buffer') + def write_bytes(self, _bytes: Union[bytes, bytearray], length: int): + assert len(_bytes) == length, len(_bytes) + self.write(_bytes) + def can_read_more(self) -> bool: if not self.input: return False diff --git a/electrum/wallet.py b/electrum/wallet.py index 9a5d71d30b..c8b4d7736c 100644 --- a/electrum/wallet.py +++ b/electrum/wallet.py @@ -72,7 +72,7 @@ from .interface import NetworkException from .mnemonic import Mnemonic from .logging import get_logger -from .lnworker import LNWallet +from .lnworker import LNWallet, LNBackups from .paymentrequest import PaymentRequest if TYPE_CHECKING: @@ -259,6 +259,7 @@ def __init__(self, db: WalletDB, storage: Optional[WalletStorage], *, config: Si # lightning ln_xprv = self.db.get('lightning_privkey2') self.lnworker = LNWallet(self, ln_xprv) if ln_xprv else None + self.lnbackups = LNBackups(self) def save_db(self): if self.storage: @@ -269,7 +270,14 @@ def save_backup(self): if backup_dir is None: return new_db = WalletDB(self.db.dump(), manual_upgrades=False) - new_db.put('is_backup', True) + + if self.lnworker: + channel_backups = new_db.get_dict('channel_backups') + for chan_id, chan in self.lnworker.channels.items(): + channel_backups[chan_id.hex()] = self.lnworker.create_channel_backup(chan_id) + new_db.put('channels', None) + new_db.put('lightning_privkey2', None) + new_path = os.path.join(backup_dir, self.basename() + '.backup') new_storage = WalletStorage(new_path) new_storage._encryption_version = self.storage._encryption_version @@ -305,9 +313,6 @@ def remove_lightning(self): self.db.put('lightning_privkey2', None) self.save_db() - def is_lightning_backup(self): - return self.has_lightning() and self.db.get('is_backup') - def stop_threads(self): super().stop_threads() if any([ks.is_requesting_to_be_rewritten_to_wallet_file for ks in self.get_keystores()]): @@ -324,9 +329,11 @@ def clear_history(self): def start_network(self, network): AddressSynchronizer.start_network(self, network) - if self.lnworker and network and not self.is_lightning_backup(): - network.maybe_init_lightning() - self.lnworker.start_network(network) + if network: + if self.lnworker: + network.maybe_init_lightning() + self.lnworker.start_network(network) + self.lnbackups.start_network(network) def load_and_cleanup(self): self.load_keystore() diff --git a/electrum/wallet_db.py b/electrum/wallet_db.py index e439b0dcc3..357be276c2 100644 --- a/electrum/wallet_db.py +++ b/electrum/wallet_db.py @@ -36,7 +36,7 @@ from .keystore import bip44_derivation from .transaction import Transaction, TxOutpoint, tx_from_any, PartialTransaction, PartialTxOutput from .logging import Logger -from .lnutil import LOCAL, REMOTE, FeeUpdate, UpdateAddHtlc, LocalConfig, RemoteConfig, Keypair, OnlyPubkeyKeypair, RevocationStore +from .lnutil import LOCAL, REMOTE, FeeUpdate, UpdateAddHtlc, LocalConfig, RemoteConfig, Keypair, OnlyPubkeyKeypair, RevocationStore, ChannelBackupStorage from .lnutil import ChannelConstraints, Outpoint, ShachainElement from .json_db import StoredDict, JsonDB, locked, modifier from .plugin import run_hook, plugin_loaders @@ -1101,6 +1101,8 @@ def _convert_dict(self, path, key, v): v = dict((k, UpdateAddHtlc.from_tuple(*x)) for k, x in v.items()) elif key == 'fee_updates': v = dict((k, FeeUpdate(**x)) for k, x in v.items()) + elif key == 'channel_backups': + v = dict((k, ChannelBackupStorage(**x)) for k, x in v.items()) elif key == 'tx_fees': v = dict((k, TxFeesValue(*x)) for k, x in v.items()) elif key == 'prevouts_by_scripthash': From 74517c88ad3f425003e99f8d97a64b1dafb9473d Mon Sep 17 00:00:00 2001 From: ThomasV Date: Fri, 10 Apr 2020 14:59:21 +0200 Subject: [PATCH 034/117] do not use short_channel_id as state, use channel state for that. display it as soon as the funding tx is mined. --- electrum/lnchannel.py | 20 ++++++++++---------- electrum/lnpeer.py | 11 +++++------ 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/electrum/lnchannel.py b/electrum/lnchannel.py index 7ab8d9759b..e88f285a89 100644 --- a/electrum/lnchannel.py +++ b/electrum/lnchannel.py @@ -135,6 +135,9 @@ def get_id_for_log(self) -> str: return str(scid) return self.channel_id.hex() + def short_id_for_GUI(self) -> str: + return format_short_channel_id(self.short_channel_id) + def set_state(self, state: channel_states) -> None: """ set on-chain state """ old_state = self._state @@ -147,6 +150,9 @@ def set_state(self, state: channel_states) -> None: def get_state(self) -> channel_states: return self._state + def is_funded(self): + return self.get_state() >= channel_states.FUNDED + def is_open(self): return self.get_state() == channel_states.OPEN @@ -244,12 +250,12 @@ def update_unfunded_state(self): def update_funded_state(self, funding_txid, funding_height): self.save_funding_height(funding_txid, funding_height.height, funding_height.timestamp) self.delete_closing_height() + if funding_height.conf>0: + self.set_short_channel_id(ShortChannelID.from_components( + funding_height.height, funding_height.txpos, self.funding_outpoint.output_index)) if self.get_state() == channel_states.OPENING: if self.is_funding_tx_mined(funding_height): self.set_state(channel_states.FUNDED) - self.set_short_channel_id(ShortChannelID.from_components( - funding_height.height, funding_height.txpos, self.funding_outpoint.output_index)) - self.logger.info(f"save_short_channel_id: {self.short_channel_id}") def update_closed_state(self, funding_txid, funding_height, closing_txid, closing_height, keep_watching): self.save_funding_height(funding_txid, funding_height.height, funding_height.timestamp) @@ -333,15 +339,12 @@ def create_sweeptxs_for_their_ctx(self, ctx): def get_funding_address(self): return self.cb.funding_address - def short_id_for_GUI(self) -> str: - return 'BACKUP' - def is_initiator(self): return self.cb.is_initiator def get_state_for_GUI(self): cs = self.get_state() - return cs.name + return 'BACKUP, ' + cs.name def get_oldest_unrevoked_ctn(self, who): return -1 @@ -401,9 +404,6 @@ def __init__(self, state: 'StoredDict', *, sweep_address=None, name=None, lnwork self._receive_fail_reasons = {} # type: Dict[int, BarePaymentAttemptLog] self._ignore_max_htlc_value = False # used in tests - def short_id_for_GUI(self) -> str: - return format_short_channel_id(self.short_channel_id) - def is_initiator(self): return self.constraints.is_initiator diff --git a/electrum/lnpeer.py b/electrum/lnpeer.py index 1c059ffb02..b9160e434f 100644 --- a/electrum/lnpeer.py +++ b/electrum/lnpeer.py @@ -886,11 +886,10 @@ def are_datalossprotect_fields_valid() -> bool: return chan.peer_state = peer_states.GOOD - # note: chan.short_channel_id being set implies the funding txn is already at sufficient depth - if their_next_local_ctn == next_local_ctn == 1 and chan.short_channel_id: + if chan.is_funded() and their_next_local_ctn == next_local_ctn == 1: self.send_funding_locked(chan) # checks done - if chan.config[LOCAL].funding_locked_received and chan.short_channel_id: + if chan.is_funded() and chan.config[LOCAL].funding_locked_received: self.mark_open(chan) self.network.trigger_callback('channel', chan) if chan.get_state() == channel_states.CLOSING: @@ -903,7 +902,7 @@ def send_funding_locked(self, chan: Channel): get_per_commitment_secret_from_seed(chan.config[LOCAL].per_commitment_secret_seed, per_commitment_secret_index), 'big')) # note: if funding_locked was not yet received, we might send it multiple times self.send_message("funding_locked", channel_id=channel_id, next_per_commitment_point=per_commitment_point_second) - if chan.config[LOCAL].funding_locked_received and chan.short_channel_id: + if chan.is_funded() and chan.config[LOCAL].funding_locked_received: self.mark_open(chan) def on_funding_locked(self, chan: Channel, payload): @@ -913,7 +912,7 @@ def on_funding_locked(self, chan: Channel, payload): chan.config[REMOTE].next_per_commitment_point = their_next_point chan.config[LOCAL].funding_locked_received = True self.lnworker.save_channel(chan) - if chan.short_channel_id: + if chan.is_funded(): self.mark_open(chan) def on_network_update(self, chan: Channel, funding_tx_depth: int): @@ -970,7 +969,7 @@ async def handle_announcements(self, chan: Channel): ) def mark_open(self, chan: Channel): - assert chan.short_channel_id is not None + assert chan.is_funded() # only allow state transition from "FUNDED" to "OPEN" old_state = chan.get_state() if old_state == channel_states.OPEN: From e50f6d29ed9c558845531a4abac0304564095efd Mon Sep 17 00:00:00 2001 From: ThomasV Date: Fri, 10 Apr 2020 19:50:20 +0200 Subject: [PATCH 035/117] export channel backup from kivy gui --- .../gui/kivy/uix/dialogs/lightning_channels.py | 12 ++++++++++++ electrum/lnchannel.py | 8 ++------ electrum/lnworker.py | 18 +++++++++++++----- electrum/tests/test_lnpeer.py | 3 +++ 4 files changed, 30 insertions(+), 11 deletions(-) diff --git a/electrum/gui/kivy/uix/dialogs/lightning_channels.py b/electrum/gui/kivy/uix/dialogs/lightning_channels.py index 317925b8b3..c5e8e08855 100644 --- a/electrum/gui/kivy/uix/dialogs/lightning_channels.py +++ b/electrum/gui/kivy/uix/dialogs/lightning_channels.py @@ -180,6 +180,11 @@ BoxLayout: size_hint: 1, None height: '48dp' + Button: + size_hint: 0.5, None + height: '48dp' + text: _('Backup') + on_release: root.export_backup() Button: size_hint: 0.5, None height: '48dp' @@ -284,6 +289,9 @@ def __init__(self, chan, app, **kwargs): super(ChannelBackupPopup,self).__init__(**kwargs) self.chan = chan self.app = app + self.short_id = format_short_channel_id(chan.short_channel_id) + self.state = chan.get_state_for_GUI() + self.title = _('Channel Backup') def request_force_close(self): msg = _('Request force close?') @@ -363,6 +371,10 @@ def _remove_channel(self, b): self.app._trigger_update_history() self.dismiss() + def export_backup(self): + text = self.app.wallet.lnworker.export_channel_backup(self.chan.channel_id) + self.app.qr_dialog(_("Channel Backup " + self.chan.short_id_for_GUI()), 'channel_backup:'+text) + def force_close(self): Question(_('Force-close channel?'), self._force_close).open() diff --git a/electrum/lnchannel.py b/electrum/lnchannel.py index e88f285a89..3331546093 100644 --- a/electrum/lnchannel.py +++ b/electrum/lnchannel.py @@ -146,6 +146,8 @@ def set_state(self, state: channel_states) -> None: self.logger.debug(f'Setting channel state: {old_state.name} -> {state.name}') self._state = state self.storage['state'] = self._state.name + if self.lnworker: + self.lnworker.channel_state_changed(self) def get_state(self) -> channel_states: return self._state @@ -567,12 +569,6 @@ def open_with_first_pcp(self, remote_pcp: bytes, remote_sig: bytes) -> None: self.hm.channel_open_finished() self.peer_state = peer_states.GOOD - def set_state(self, state: channel_states) -> None: - super().set_state(state) - if self.lnworker: - self.lnworker.save_channel(self) - self.lnworker.network.trigger_callback('channel', self) - def get_state_for_GUI(self): # status displayed in the GUI cs = self.get_state() diff --git a/electrum/lnworker.py b/electrum/lnworker.py index 88cf4f7f72..709aaee95b 100644 --- a/electrum/lnworker.py +++ b/electrum/lnworker.py @@ -657,6 +657,10 @@ def channels_for_peer(self, node_id): with self.lock: return {x: y for (x, y) in self.channels.items() if y.node_id == node_id} + def channel_state_changed(self, chan): + self.save_channel(chan) + self.network.trigger_callback('channel', chan) + def save_channel(self, chan): assert type(chan) is Channel if chan.config[REMOTE].next_per_commitment_point == chan.config[REMOTE].current_per_commitment_point: @@ -1367,6 +1371,9 @@ def __init__(self, wallet: 'Abstract_Wallet'): for channel_id, cb in self.db.get_dict("channel_backups").items(): self.channel_backups[bfh(channel_id)] = ChannelBackup(cb, sweep_address=self.sweep_address, lnworker=self) + def channel_state_changed(self, chan): + self.network.trigger_callback('channel', chan) + def peer_closed(self, chan): pass @@ -1390,16 +1397,17 @@ def start_network(self, network: 'Network'): def import_channel_backup(self, encrypted): xpub = self.wallet.get_fingerprint() - x = pw_decode_bytes(encrypted, xpub, version=PW_HASH_VERSION_LATEST) - cb = ChannelBackupStorage.from_bytes(x) - channel_id = cb.channel_id().hex() + decrypted = pw_decode_bytes(encrypted, xpub, version=PW_HASH_VERSION_LATEST) + cb_storage = ChannelBackupStorage.from_bytes(decrypted) + channel_id = cb_storage.channel_id().hex() d = self.db.get_dict("channel_backups") if channel_id in d: raise Exception('Channel already in wallet') - d[channel_id] = cb - self.channel_backups[bfh(channel_id)] = ChannelBackup(cb, sweep_address=self.sweep_address, lnworker=self) + d[channel_id] = cb_storage + self.channel_backups[bfh(channel_id)] = cb = ChannelBackup(cb_storage, sweep_address=self.sweep_address, lnworker=self) self.wallet.save_db() self.network.trigger_callback('channels_updated', self.wallet) + self.lnwatcher.add_channel(cb.funding_outpoint.to_str(), cb.get_funding_address()) def remove_channel_backup(self, channel_id): d = self.db.get_dict("channel_backups") diff --git a/electrum/tests/test_lnpeer.py b/electrum/tests/test_lnpeer.py index 82b6a85474..9500ec0574 100644 --- a/electrum/tests/test_lnpeer.py +++ b/electrum/tests/test_lnpeer.py @@ -124,6 +124,9 @@ def get_channel_by_short_id(self, short_channel_id): if chan.short_channel_id == short_channel_id: return chan + def channel_state_changed(self, chan): + pass + def save_channel(self, chan): print("Ignoring channel save") From 7a11c0591667fd15995e36c77af1fd344a61c061 Mon Sep 17 00:00:00 2001 From: ThomasV Date: Sat, 11 Apr 2020 10:56:43 +0200 Subject: [PATCH 036/117] fix #6075 --- electrum/commands.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/electrum/commands.py b/electrum/commands.py index c74d1301e6..a71acde91a 100644 --- a/electrum/commands.py +++ b/electrum/commands.py @@ -788,6 +788,7 @@ async def getrequest(self, key, wallet: Abstract_Wallet = None): async def list_requests(self, pending=False, expired=False, paid=False, wallet: Abstract_Wallet = None): """List the payment requests you made.""" out = wallet.get_sorted_requests() + out = list(map(self._format_request, out)) if pending: f = PR_UNPAID elif expired: @@ -798,7 +799,7 @@ async def list_requests(self, pending=False, expired=False, paid=False, wallet: f = None if f is not None: out = list(filter(lambda x: x.get('status')==f, out)) - return list(map(self._format_request, out)) + return out @command('w') async def createnewaddress(self, wallet: Abstract_Wallet = None): From 312ef15cd612a8235cf7be29dcf6f3b8a11b0ab7 Mon Sep 17 00:00:00 2001 From: ThomasV Date: Sat, 11 Apr 2020 12:02:38 +0200 Subject: [PATCH 037/117] fix #6056 --- electrum/commands.py | 5 +++++ electrum/gui/kivy/uix/screens.py | 4 ++-- electrum/gui/qt/main_window.py | 3 ++- electrum/lnaddr.py | 28 +++++++++++++++++++++++----- electrum/lnworker.py | 14 -------------- 5 files changed, 32 insertions(+), 22 deletions(-) diff --git a/electrum/commands.py b/electrum/commands.py index a71acde91a..bb85fe3708 100644 --- a/electrum/commands.py +++ b/electrum/commands.py @@ -58,6 +58,7 @@ from .plugin import run_hook from .version import ELECTRUM_VERSION from .simple_config import SimpleConfig +from .lnaddr import parse_lightning_invoice if TYPE_CHECKING: @@ -981,6 +982,10 @@ async def open_channel(self, connection_string, amount, push_amount=0, password= password=password) return chan.funding_outpoint.to_str() + @command('') + async def decode_invoice(self, invoice): + return parse_lightning_invoice(invoice) + @command('wn') async def lnpay(self, invoice, attempts=1, timeout=10, wallet: Abstract_Wallet = None): lnworker = wallet.lnworker diff --git a/electrum/gui/kivy/uix/screens.py b/electrum/gui/kivy/uix/screens.py index 75de262e4b..3bc6927a7b 100644 --- a/electrum/gui/kivy/uix/screens.py +++ b/electrum/gui/kivy/uix/screens.py @@ -33,7 +33,7 @@ from electrum.plugin import run_hook from electrum.wallet import InternalAddressCorruption from electrum import simple_config -from electrum.lnaddr import lndecode +from electrum.lnaddr import lndecode, parse_lightning_invoice from electrum.lnutil import RECEIVED, SENT, PaymentFailure from .dialogs.question import Question @@ -299,7 +299,7 @@ def read_invoice(self): return message = self.message if self.is_lightning: - return self.app.wallet.lnworker.parse_bech32_invoice(address) + return parse_lightning_invoice(address) else: # on-chain if self.payment_request: outputs = self.payment_request.get_outputs() diff --git a/electrum/gui/qt/main_window.py b/electrum/gui/qt/main_window.py index 3db2bbb9f7..eb9040d8f5 100644 --- a/electrum/gui/qt/main_window.py +++ b/electrum/gui/qt/main_window.py @@ -76,6 +76,7 @@ from electrum.util import PR_PAID, PR_FAILED from electrum.util import pr_expiration_values from electrum.lnutil import ln_dummy_address +from electrum.lnaddr import parse_lightning_invoice from .exception_window import Exception_Hook from .amountedit import AmountEdit, BTCAmountEdit, FreezableLineEdit, FeerateEdit @@ -1492,7 +1493,7 @@ def read_invoice(self): if not self.wallet.lnworker: self.show_error(_('Lightning is disabled')) return - invoice_dict = self.wallet.lnworker.parse_bech32_invoice(invoice) + invoice_dict = parse_lightning_invoice(invoice) if invoice_dict.get('amount') is None: amount = self.amount_e.get_amount() if amount: diff --git a/electrum/lnaddr.py b/electrum/lnaddr.py index a100540552..ace941db7c 100644 --- a/electrum/lnaddr.py +++ b/electrum/lnaddr.py @@ -13,6 +13,8 @@ from .segwit_addr import bech32_encode, bech32_decode, CHARSET from . import constants from . import ecc +from .util import PR_TYPE_LN +from .bitcoin import COIN # BOLT #11: @@ -307,6 +309,11 @@ def is_expired(self) -> bool: class LnDecodeException(Exception): pass +class SerializableKey: + def __init__(self, pubkey): + self.pubkey = pubkey + def serialize(self): + return self.pubkey.get_public_key_bytes(True) def lndecode(invoice: str, *, verbose=False, expected_hrp=None) -> LnAddr: if expected_hrp is None: @@ -460,11 +467,22 @@ class WrappedBytesKey: return addr -class SerializableKey: - def __init__(self, pubkey): - self.pubkey = pubkey - def serialize(self): - return self.pubkey.get_public_key_bytes(True) + + + +def parse_lightning_invoice(invoice): + lnaddr = lndecode(invoice, expected_hrp=constants.net.SEGWIT_HRP) + amount = int(lnaddr.amount * COIN) if lnaddr.amount else None + return { + 'type': PR_TYPE_LN, + 'invoice': invoice, + 'amount': amount, + 'message': lnaddr.get_description(), + 'time': lnaddr.date, + 'exp': lnaddr.get_expiry(), + 'pubkey': lnaddr.pubkey.serialize().hex(), + 'rhash': lnaddr.paymenthash.hex(), + } if __name__ == '__main__': # run using diff --git a/electrum/lnworker.py b/electrum/lnworker.py index 709aaee95b..e73ae7ebc0 100644 --- a/electrum/lnworker.py +++ b/electrum/lnworker.py @@ -529,20 +529,6 @@ def get_settled_payments(self): out[k].append(v) return out - def parse_bech32_invoice(self, invoice): - lnaddr = lndecode(invoice, expected_hrp=constants.net.SEGWIT_HRP) - amount = int(lnaddr.amount * COIN) if lnaddr.amount else None - return { - 'type': PR_TYPE_LN, - 'invoice': invoice, - 'amount': amount, - 'message': lnaddr.get_description(), - 'time': lnaddr.date, - 'exp': lnaddr.get_expiry(), - 'pubkey': bh2u(lnaddr.pubkey.serialize()), - 'rhash': lnaddr.paymenthash.hex(), - } - def get_lightning_history(self): out = {} for key, plist in self.get_settled_payments().items(): From e2544b893a401128bc28678cf463694418ad65d1 Mon Sep 17 00:00:00 2001 From: ThomasV Date: Sat, 11 Apr 2020 15:26:29 +0200 Subject: [PATCH 038/117] rm dead code: wallet.wait_until_synchronized --- electrum/wallet.py | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/electrum/wallet.py b/electrum/wallet.py index c8b4d7736c..af17309d8f 100644 --- a/electrum/wallet.py +++ b/electrum/wallet.py @@ -1170,33 +1170,6 @@ def set_frozen_state_of_coins(self, utxos: Sequence[PartialTxInput], freeze: boo self.frozen_coins -= set(utxos) self.db.put('frozen_coins', list(self.frozen_coins)) - def wait_until_synchronized(self, callback=None): - def wait_for_wallet(): - self.set_up_to_date(False) - while not self.is_up_to_date(): - if callback: - msg = "{}\n{} {}".format( - _("Please wait..."), - _("Addresses generated:"), - len(self.get_addresses())) - callback(msg) - time.sleep(0.1) - def wait_for_network(): - while not self.network.is_connected(): - if callback: - msg = "{} \n".format(_("Connecting...")) - callback(msg) - time.sleep(0.1) - # wait until we are connected, because the user - # might have selected another server - if self.network: - self.logger.info("waiting for network...") - wait_for_network() - self.logger.info("waiting while wallet is syncing...") - wait_for_wallet() - else: - self.synchronize() - def can_export(self): return not self.is_watching_only() and hasattr(self.keystore, 'get_private_key') From 08118ca1674f58860b5a5a639372c95bc012eb80 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Sat, 11 Apr 2020 15:50:12 +0200 Subject: [PATCH 039/117] qt wizard: tweak GoBack behaviour to recalc inputs to previous dialog follow-up f13f46c555b979b265c7da9b6e340b6342f9e4b0 When on dialog n user presses "Back", - previously, we went back to when dialog n-1 appeared - now, go back to just after dialog n-2 finishes This way, any calculations between when dialog n-2 finishes and dialog n-1 appears will rerun, potentially populating dialog n-1 differently. Namely if the user presses back on the confirm_seed_dialog, we want to go back to the show_seed_dialog but with a freshly generated seed. --- electrum/base_wizard.py | 3 +++ electrum/gui/qt/installwizard.py | 22 ++++++++++++++++++---- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/electrum/base_wizard.py b/electrum/base_wizard.py index ce8fcc6038..9144a9ace2 100644 --- a/electrum/base_wizard.py +++ b/electrum/base_wizard.py @@ -60,6 +60,9 @@ class ScriptTypeNotSupported(Exception): pass class GoBack(Exception): pass +class ReRunDialog(Exception): pass + + class ChooseHwDeviceAgain(Exception): pass diff --git a/electrum/gui/qt/installwizard.py b/electrum/gui/qt/installwizard.py index 6918ab2857..b71d9f5db8 100644 --- a/electrum/gui/qt/installwizard.py +++ b/electrum/gui/qt/installwizard.py @@ -19,7 +19,7 @@ from electrum.wallet import Wallet, Abstract_Wallet from electrum.storage import WalletStorage, StorageReadWriteError from electrum.util import UserCancelled, InvalidPassword, WalletFileException, get_new_wallet_name -from electrum.base_wizard import BaseWizard, HWD_SETUP_DECRYPT_WALLET, GoBack +from electrum.base_wizard import BaseWizard, HWD_SETUP_DECRYPT_WALLET, GoBack, ReRunDialog from electrum.i18n import _ from .seed_dialog import SeedLayout, KeysLayout @@ -97,6 +97,7 @@ def func_wrapper(*args, **kwargs): run_next = kwargs['run_next'] wizard = args[0] # type: InstallWizard while True: + #wizard.logger.debug(f"dialog stack. len: {len(wizard._stack)}. stack: {wizard._stack}") wizard.back_button.setText(_('Back') if wizard.can_go_back() else _('Cancel')) # current dialog try: @@ -110,11 +111,24 @@ def func_wrapper(*args, **kwargs): raise # next dialog try: - run_next(*out) - except GoBack: + while True: + try: + run_next(*out) + except ReRunDialog: + # restore state, and then let the loop re-run next + wizard.go_back(rerun_previous=False) + else: + break + except GoBack as e: # to go back from the next dialog, we ask the wizard to restore state wizard.go_back(rerun_previous=False) - # and we re-run the current dialog (by continuing) + # and we re-run the current dialog + if wizard.can_go_back(): + # also rerun any calculations that might have populated the inputs to the current dialog, + # by going back to just after the *previous* dialog finished + raise ReRunDialog() from e + else: + continue else: break return func_wrapper From 9a88c13b3de000fa3a65e1039cf19c5635d83c9f Mon Sep 17 00:00:00 2001 From: SomberNight Date: Sat, 11 Apr 2020 16:33:45 +0200 Subject: [PATCH 040/117] translations: add note that f-strings cannot be translated and replace current usage --- electrum/gui/kivy/uix/screens.py | 4 ++-- electrum/gui/qt/channels_list.py | 2 +- electrum/gui/qt/lightning_dialog.py | 8 ++++---- electrum/i18n.py | 3 +++ electrum/lnworker.py | 2 +- 5 files changed, 11 insertions(+), 8 deletions(-) diff --git a/electrum/gui/kivy/uix/screens.py b/electrum/gui/kivy/uix/screens.py index 3bc6927a7b..3da88ccac4 100644 --- a/electrum/gui/kivy/uix/screens.py +++ b/electrum/gui/kivy/uix/screens.py @@ -404,7 +404,7 @@ def callback(c): self.app.wallet.delete_invoice(key) self.update() n = len(invoices) - d = Question(_(f'Delete {n} invoices?'), callback) + d = Question(_('Delete {} invoices?').format(n), callback) d.open() @@ -522,7 +522,7 @@ def callback(c): self.app.wallet.delete_request(key) self.update() n = len(requests) - d = Question(_(f'Delete {n} requests?'), callback) + d = Question(_('Delete {} requests?').format(n), callback) d.open() diff --git a/electrum/gui/qt/channels_list.py b/electrum/gui/qt/channels_list.py index 24bf0dd913..543840dbc7 100644 --- a/electrum/gui/qt/channels_list.py +++ b/electrum/gui/qt/channels_list.py @@ -108,7 +108,7 @@ def force_close(self, channel_id): chan = self.lnworker.channels[channel_id] to_self_delay = chan.config[REMOTE].to_self_delay msg = _('Force-close channel?') + '\n\n'\ - + _(f'Funds retrieved from this channel will not be available before {to_self_delay} blocks after forced closure.') + ' '\ + + _('Funds retrieved from this channel will not be available before {} blocks after forced closure.').format(to_self_delay) + ' '\ + _('After that delay, funds will be sent to an address derived from your wallet seed.') + '\n\n'\ + _('In the meantime, channel funds will not be recoverable from your seed, and might be lost if you lose your wallet.') + ' '\ + _('To prevent that, you should have a backup of this channel on another device.') diff --git a/electrum/gui/qt/lightning_dialog.py b/electrum/gui/qt/lightning_dialog.py index c04e1fea31..cf467e7fe2 100644 --- a/electrum/gui/qt/lightning_dialog.py +++ b/electrum/gui/qt/lightning_dialog.py @@ -66,14 +66,14 @@ def __init__(self, gui_object: 'ElectrumGui'): self.set_unknown_channels('', len(self.network.lngossip.unknown_ids)) def on_channel_db(self, event, num_nodes, num_channels, num_policies): - self.num_nodes.setText(_(f'{num_nodes} nodes')) - self.num_channels.setText(_(f'{num_channels} channels')) + self.num_nodes.setText(_('{} nodes').format(num_nodes)) + self.num_channels.setText(_('{} channels').format(num_channels)) def set_num_peers(self, event, num_peers): - self.num_peers.setText(_(f'Connected to {num_peers} peers')) + self.num_peers.setText(_('Connected to {} peers').format(num_peers)) def set_unknown_channels(self, event, unknown): - self.status.setText(_(f'Requesting {unknown} channels...') if unknown else '') + self.status.setText(_('Requesting {} channels...').format(unknown) if unknown else '') def is_hidden(self): return self.isMinimized() or self.isHidden() diff --git a/electrum/i18n.py b/electrum/i18n.py index 9c6fad995c..3c5ba35721 100644 --- a/electrum/i18n.py +++ b/electrum/i18n.py @@ -30,6 +30,9 @@ language = gettext.translation('electrum', LOCALE_DIR, fallback=True) +# note: f-strings cannot be translated! see https://stackoverflow.com/q/49797658 +# So this does not work: _(f"My name: {name}") +# instead use .format: _("My name: {}").format(name) def _(x): global language return language.gettext(x) diff --git a/electrum/lnworker.py b/electrum/lnworker.py index e73ae7ebc0..0c2192e58e 100644 --- a/electrum/lnworker.py +++ b/electrum/lnworker.py @@ -819,7 +819,7 @@ async def _pay(self, invoice, amount_sat=None, attempts=1) -> bool: if success: break else: - reason = _(f'Failed after {attempts} attempts') + reason = _('Failed after {} attempts').format(attempts) self.network.trigger_callback('invoice_status', key) if success: self.network.trigger_callback('payment_succeeded', key) From 99f933401a618e6b8e32b90314cf0c890213e631 Mon Sep 17 00:00:00 2001 From: ThomasV Date: Sun, 12 Apr 2020 12:29:46 +0200 Subject: [PATCH 041/117] add more logging shortcuts --- electrum/address_synchronizer.py | 1 + electrum/lnpeer.py | 1 + electrum/lnworker.py | 1 + 3 files changed, 3 insertions(+) diff --git a/electrum/address_synchronizer.py b/electrum/address_synchronizer.py index e0575d606c..8e586cf1aa 100644 --- a/electrum/address_synchronizer.py +++ b/electrum/address_synchronizer.py @@ -616,6 +616,7 @@ def set_up_to_date(self, up_to_date): self.up_to_date = up_to_date if self.network: self.network.notify('status') + self.logger.info(f'set_up_to_date: {up_to_date}') def is_up_to_date(self): with self.lock: return self.up_to_date diff --git a/electrum/lnpeer.py b/electrum/lnpeer.py index b9160e434f..9eb5aab3dd 100644 --- a/electrum/lnpeer.py +++ b/electrum/lnpeer.py @@ -62,6 +62,7 @@ class Peer(Logger): + LOGGING_SHORTCUT = 'P' def __init__(self, lnworker: Union['LNGossip', 'LNWallet'], pubkey:bytes, transport: LNTransportBase): self._sent_init = False # type: bool diff --git a/electrum/lnworker.py b/electrum/lnworker.py index 0c2192e58e..f1bf2a9c04 100644 --- a/electrum/lnworker.py +++ b/electrum/lnworker.py @@ -366,6 +366,7 @@ def choose_preferred_address(addr_list: Sequence[Tuple[str, int, int]]) -> Tuple class LNGossip(LNWorker): max_age = 14*24*3600 + LOGGING_SHORTCUT = 'g' def __init__(self, network): seed = os.urandom(32) From 27949cb0e557085f205035fa67ef57fd71f650ac Mon Sep 17 00:00:00 2001 From: ThomasV Date: Sun, 12 Apr 2020 12:48:44 +0200 Subject: [PATCH 042/117] add list_peer command. (fix #6057) --- electrum/commands.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/electrum/commands.py b/electrum/commands.py index bb85fe3708..76fe1d7c90 100644 --- a/electrum/commands.py +++ b/electrum/commands.py @@ -969,6 +969,15 @@ async def add_peer(self, connection_string, timeout=20, wallet: Abstract_Wallet await wallet.lnworker.add_peer(connection_string) return True + @command('wn') + async def list_peers(self, wallet: Abstract_Wallet = None): + return [{ + 'node_id':p.pubkey.hex(), + 'address':p.transport.name(), + 'initialized':p.is_initialized(), + 'channels': [c.funding_outpoint.to_str() for c in p.channels.values()], + } for p in wallet.lnworker.peers.values()] + @command('wpn') async def open_channel(self, connection_string, amount, push_amount=0, password=None, wallet: Abstract_Wallet = None): funding_sat = satoshis(amount) From bfffc7cb1ec3372d771371b5731226d1a5c8bafa Mon Sep 17 00:00:00 2001 From: ThomasV Date: Sun, 12 Apr 2020 12:57:07 +0200 Subject: [PATCH 043/117] Rename 'On-chain' button, add tooltips (see #6053) --- electrum/gui/qt/main_window.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/electrum/gui/qt/main_window.py b/electrum/gui/qt/main_window.py index eb9040d8f5..51fee20b74 100644 --- a/electrum/gui/qt/main_window.py +++ b/electrum/gui/qt/main_window.py @@ -1064,8 +1064,9 @@ def on_expiry(i): self.clear_invoice_button = QPushButton(_('Clear')) self.clear_invoice_button.clicked.connect(self.clear_receive_tab) - self.create_invoice_button = QPushButton(_('On-chain')) + self.create_invoice_button = QPushButton(_('Request')) self.create_invoice_button.setIcon(read_QIcon("bitcoin.png")) + self.create_invoice_button.setToolTip('Create on-chain request') self.create_invoice_button.clicked.connect(lambda: self.create_invoice(False)) self.receive_buttons = buttons = QHBoxLayout() buttons.addStretch(1) @@ -1073,6 +1074,7 @@ def on_expiry(i): buttons.addWidget(self.create_invoice_button) if self.wallet.has_lightning(): self.create_lightning_invoice_button = QPushButton(_('Lightning')) + self.create_lightning_invoice_button.setToolTip('Create lightning request') self.create_lightning_invoice_button.setIcon(read_QIcon("lightning.png")) self.create_lightning_invoice_button.clicked.connect(lambda: self.create_invoice(True)) buttons.addWidget(self.create_lightning_invoice_button) From a4fe14bb82d5853cd26351c7e2ef829debbd1520 Mon Sep 17 00:00:00 2001 From: TheCharlatan Date: Tue, 18 Feb 2020 15:31:05 +0100 Subject: [PATCH 044/117] BitBox02 Electrum plugin support This commit adds support for the BitBox02 hardware wallet. It supports both single and multisig for the electrum gui wallet. To use the plugin a local installation of the BitBox02 python library is required. It can be found on PiPy under the name 'bitbox02' and can be installed from the bitbox02-firmware repository in the py/bitbox02 directory. All communication to and from the BitBox02 is noise encrypted, the keys required for this are stored in the wallet config file under the bitbox02 key. The BitBox02 registers a multisig configuration before allowing transaction signing. This multisig configuration includes the threshold, cosigner xpubs, keypath, a variable to indicate for mainnet and testnet, and a name that the user can choose during configuration registration. The user is asked to register the multisig configuration either during address verification or during transaction signing. The check the xpub of the BitBox02 for other hardware wallets, a button is added in the wallet info dialog. The wallet encryption key is fetched in a separate api call, requiring a slightly tweaked override version of the wallet encryption password. --- electrum/bitcoin.py | 15 + electrum/gui/icons/bitbox02.png | Bin 0 -> 1622 bytes electrum/gui/icons/bitbox02_unpaired.png | Bin 0 -> 1629 bytes electrum/gui/qt/main_window.py | 10 +- electrum/gui/qt/util.py | 2 + electrum/plugin.py | 4 + electrum/plugins/bitbox02/__init__.py | 14 + electrum/plugins/bitbox02/bitbox02.py | 620 +++++++++++++++++++++++ electrum/plugins/bitbox02/qt.py | 174 +++++++ electrum/plugins/coldcard/qt.py | 2 +- 10 files changed, 838 insertions(+), 3 deletions(-) create mode 100644 electrum/gui/icons/bitbox02.png create mode 100644 electrum/gui/icons/bitbox02_unpaired.png create mode 100644 electrum/plugins/bitbox02/__init__.py create mode 100644 electrum/plugins/bitbox02/bitbox02.py create mode 100644 electrum/plugins/bitbox02/qt.py diff --git a/electrum/bitcoin.py b/electrum/bitcoin.py index 16d2fda9ca..c1c2d73be3 100644 --- a/electrum/bitcoin.py +++ b/electrum/bitcoin.py @@ -432,6 +432,21 @@ def address_to_script(addr: str, *, net=None) -> str: raise BitcoinException(f'unknown address type: {addrtype}') return script +def address_to_hash(addr: str, *, net=None) -> Tuple[int, bytes]: + """Return the pubkey hash / witness program of an address""" + if net is None: net = constants.net + if not is_address(addr, net=net): + raise BitcoinException(f"invalid bitcoin address: {addr}") + witver, witprog = segwit_addr.decode(net.SEGWIT_HRP, addr) + if witprog is not None: + if len(witprog) == 20: + return WIF_SCRIPT_TYPES['p2wpkh'], bytes(witprog) + return WIF_SCRIPT_TYPES['p2wsh'], bytes(witprog) + addrtype, hash_160_ = b58_address_to_hash160(addr) + if addrtype == net.ADDRTYPE_P2PKH: + return WIF_SCRIPT_TYPES['p2pkh'], hash_160_ + return WIF_SCRIPT_TYPES['p2sh'], hash_160_ + def address_to_scripthash(addr: str) -> str: script = address_to_script(addr) return script_to_scripthash(script) diff --git a/electrum/gui/icons/bitbox02.png b/electrum/gui/icons/bitbox02.png new file mode 100644 index 0000000000000000000000000000000000000000..3900c5425ef5059c6ee451759911bb9521838c7c GIT binary patch literal 1622 zcmV-c2C4apP)k=-6$kfA;%NQe4TNaRqW9zr6UfOJSf zf)wIIVEew4T_LY#wKKc3iX_h$2HDo`?9ON3d-LXpmoXzFBO@cDS(0U0aViBWEh;rC z8AC-5T#`zKO4k||VCD=ZF}MtsswF4DEy7BTo03J@mK9X6*5qAu(~^gq232+Da`m&UD5}DKjAu00J;L#VK9@2an0+a>C<0RGcY*X!z#l<}Dzq0G*Ofr>#n*ayI~? zp=hudxF7(X0{1T#i(YYZ`T+upAGk08or2@W32@GggA70e&;T?5jSN5o&;T?5jSN5o z&;T?5jSN5o@QwqVz%_^;rvjj}bV)z9CJdmnGFZ^PQa7goT7T)1eyvg2oKSCrSCsH@ zb*|LSg>h4RsvO`d_6<&hV1YRsB-Reb`@n5Ctw^tMdn{eDh}J}y`jV#=m3xce6=d(o zZ|O2hfJ|-4^VV%g4Y269!vHtz>@WygbOCe&w4BtyEFg?1ZW2b z2mDha*+#g!qK`uYc|28<2^5>Sm5BZx)Z!?n^5O$e1hZo2h#4)76Llg;55~|t2I}pR zhbru9^;1FDC`21Th5Sr8-^2s)Q~Oifde?o|FEalXFcC2jZiRS67QzUk`e2sMAwayG zot?3xqa#+Y*Kfr;0xUeYd(&kvTF=>=k4<*-<@3<@srygZ+QT)r9AEaD)p<>EvHVQ{ z0$4tumySh9H2`}QWYmdrb#=uK4-YxWYPBjn_WbQRyZG?u$ZIw~zTpKRcOLJsh4}^d zA=vxf@1-N}L>y}&H{}hJfp>g-%(B@mTUl9Qi9~|!?d|#IZ2%3c!*$%iZ@%2Hr++>5 zh35wO?A0^Y`L84Tx)OOWj0TlTCEg%3Ow_Xus@Lmr3tC!Q;=l%75Ego$d*6K@xIKV){`L>iSG!sP!c7~E2G6$;ll%lrf<*xaEKEg!yuZKSw^lFcu$<@A zVT9Go&P$OiBO^)1RMQxH+FmKp54tJ9!Bt0DS`fk75i;Bj|NWm+E%mQ1uCi_FR9xB% z-3!Z%5C99Q1}sg+?Se>DZ}Z=#w)&CwY(E|j^^vAAh*>8mC(14ww9|4cOzayD2cLOJ zaHTctkpKiRSe&jYqcDxusWx>e?gU#aK*ToL-f>o>HI3Qb-Q~S;kl|SrTKxWj*7^j) zyPj5ns>!qNcE4LDZisNX<-~I6x)nnUu{g<9 zIW!7pe8Rp?dZxWVew|@yv-YP>Kji_q)DKTs>V9fe{aM*HVd=$p8QV literal 0 HcmV?d00001 diff --git a/electrum/gui/icons/bitbox02_unpaired.png b/electrum/gui/icons/bitbox02_unpaired.png new file mode 100644 index 0000000000000000000000000000000000000000..66fe1109208cf6b644f8e29e1efb397a1b7dd1bb GIT binary patch literal 1629 zcmV-j2BP_iP)6cIFui?jixG^s+;ToZ}A`U<$_q^lqa6}kvlxI*}38Yp5% zk&G09LLi4?E{QYWGJ}@&u6Jg3W?930(v!i4@y<8zZ{ECJLM$vSEG#Up6B><1H<|y3?O#_T58RdY82BfRX|&_H_=QN1I$zv zE7#snN}8!Ewy%|+R58;^rP4S)K5lxyR;xAY^}6sdarw!VLGl*q3$!705@%;;WOsL$ z?Ck82lamuNIXOvsdwVba&Dz=;IXE~V0|NtOaBz@xb#;kc#u1fOD&MA3sj7*2-ZHp2 zb76k~vOpTh1BoI;m&;{G`uqDGGQC85(9IU{S1JqbMqnWJ_xBwVs#SHR*xTE4q^GBc z3=IuAgkA+=B@hz^afgS8+Q!6Kg+hUBZEZOug3OMNj@aguj#KL}QJv&@-2q5kUtf<^ z1JlI95@-pu1X=5ysle)A9(k_lJ>8~49K1ryx!76Z4Skc+hkd9)eID2K?QTCL`sy$Bq60+)p$?B3GhvsOvd+pRgFWx;l#v*Y~TQyOKc06OS}e#MaQwRF{iOoCfaASS(3}; zqIPS6NbFeVsWv|Jekj$U%2r~-{T@#bRDg-8_CDnfyFSz1iva z@d%I7#tFnkKpZqq)J%D@%e9Jko~i~CXxj1jUppo%mE*nhKIzQQIjdIN|C#)?w5VnL+aKKX)53gY4HVJ1ILcl1dcAqUwu|~oVkL~# zL}1ov9*C3zq>1=99l50!ulCB?#)mtAc!J4?MhG-C9x`P z3R_v4S;Dq^GFJ99~V8CL$kRp&~-UV`g z?6)ei*`J=CCgbDdLDgTaz`exHsmDj!#t#^)AngMgi3QdgF-t;jsz#y<;X4lGfhK$; z#zfF^r&TaHNR*w3GiMrkkq3Is7hFWXW1ThYpgrOnBGBF;vgegWt11s7=bztw8EJe} zWqdTysSRqq<)7_TW{F^fk+Z&~{PYXIj~-I;=B3M7ahbPYE~#3|@|yGMv&W%6G2QdFqjdpQ=_)xds2&+7(8s(% zWKPDsaY83i<-w$usXQU)EqB8~L_#C~rLVlmiZlhpb&T9`s?jaRzbkrXY>LQM4;WKL zQ}-!r{M^>~9k|}%#B6~)kGNAs%LYmI9V?oih?FHs1xR3Ih-!jVi!K|#qNsHXZL={K zcz$4{CUMrwy&|Y_Lf@52D3QF6d6q@JN7Qkz8VG9Lu0T^|Qj%@+wXm?Tu&}VOu()~n bUw{Ds?LzhZYyirM00000NkvXXu0mjf)YtAc literal 0 HcmV?d00001 diff --git a/electrum/gui/qt/main_window.py b/electrum/gui/qt/main_window.py index 51fee20b74..3125e20c38 100644 --- a/electrum/gui/qt/main_window.py +++ b/electrum/gui/qt/main_window.py @@ -2273,7 +2273,9 @@ def show_wallet_info(self): def show_mpk(index): mpk_text.setText(mpk_list[index]) mpk_text.repaint() # macOS hack for #4777 - + + # declare this value such that the hooks can later figure out what to do + labels_clayout = None # only show the combobox in case multiple accounts are available if len(mpk_list) > 1: # only show the combobox if multiple master keys are defined @@ -2288,6 +2290,7 @@ def label(idx, ks): on_click = lambda clayout: show_mpk(clayout.selected_index()) labels_clayout = ChoicesLayout(_("Master Public Keys"), labels, on_click) vbox.addLayout(labels_clayout.layout()) + labels_clayout.selected_index() else: vbox.addWidget(QLabel(_("Master Public Key"))) @@ -2295,7 +2298,10 @@ def label(idx, ks): vbox.addWidget(mpk_text) vbox.addStretch(1) - btns = run_hook('wallet_info_buttons', self, dialog) or Buttons(CloseButton(dialog)) + btn_export_info = run_hook('wallet_info_buttons', self, dialog) + btn_show_xpub = run_hook('show_xpub_button', self, dialog, labels_clayout) + btn_close = CloseButton(dialog) + btns = Buttons(btn_export_info, btn_show_xpub, btn_close) vbox.addLayout(btns) dialog.setLayout(vbox) dialog.exec_() diff --git a/electrum/gui/qt/util.py b/electrum/gui/qt/util.py index 96a7ed08fc..1449d344ad 100644 --- a/electrum/gui/qt/util.py +++ b/electrum/gui/qt/util.py @@ -161,6 +161,8 @@ def __init__(self, *buttons): QHBoxLayout.__init__(self) self.addStretch(1) for b in buttons: + if b is None: + continue self.addWidget(b) class CloseButton(QPushButton): diff --git a/electrum/plugin.py b/electrum/plugin.py index d6136506f2..084063cd9b 100644 --- a/electrum/plugin.py +++ b/electrum/plugin.py @@ -650,6 +650,10 @@ def _scan_devices_with_hid(self) -> List['Device']: if len(id_) == 0: id_ = str(d['path']) id_ += str(interface_number) + str(usage_page) + # The BitBox02's product_id is not unique per device, thus use the path instead to + # distinguish devices. + if d["product_id"] == 0x2403: + id_ = d['path'] devices.append(Device(path=d['path'], interface_number=interface_number, id_=id_, diff --git a/electrum/plugins/bitbox02/__init__.py b/electrum/plugins/bitbox02/__init__.py new file mode 100644 index 0000000000..86812d564a --- /dev/null +++ b/electrum/plugins/bitbox02/__init__.py @@ -0,0 +1,14 @@ +from electrum.i18n import _ + +fullname = "BitBox02" +description = ( + "Provides support for the BitBox02 hardware wallet" +) +requires = [ + ( + "bitbox02", + "https://github.com/digitalbitbox/bitbox02-firmware/tree/master/py/bitbox02", + ) +] +registers_keystore = ("hardware", "bitbox02", _("BitBox02")) +available_for = ["qt"] diff --git a/electrum/plugins/bitbox02/bitbox02.py b/electrum/plugins/bitbox02/bitbox02.py new file mode 100644 index 0000000000..c7098d6318 --- /dev/null +++ b/electrum/plugins/bitbox02/bitbox02.py @@ -0,0 +1,620 @@ +# +# BitBox02 Electrum plugin code. +# + +import hid +import hashlib +from typing import TYPE_CHECKING, Dict, Tuple, Optional, List, Any + +from electrum import bip32, constants +from electrum.i18n import _ +from electrum.keystore import Hardware_KeyStore, Xpub +from electrum.transaction import PartialTransaction +from electrum.wallet import Standard_Wallet, Multisig_Wallet, Deterministic_Wallet +from electrum.util import bh2u, UserFacingException +from electrum.base_wizard import ScriptTypeNotSupported, BaseWizard +from electrum.logging import get_logger +from electrum.crypto import hmac_oneshot +from electrum.plugin import Device, DeviceInfo +from electrum.simple_config import SimpleConfig +from electrum.json_db import StoredDict +from electrum.storage import get_derivation_used_for_hw_device_encryption + +import electrum.bitcoin as bitcoin +import electrum.ecc as ecc + +from ..hw_wallet import HW_PluginBase, HardwareClientBase +from ..hw_wallet.plugin import LibraryFoundButUnusable + + +try: + from bitbox02 import bitbox02 + from bitbox02 import util + from bitbox02.communication import ( + devices, + HARDENED, + u2fhid, + bitbox_api_protocol, + ) + requirements_ok = True +except ImportError: + requirements_ok = False + + +_logger = get_logger(__name__) + + +class BitBox02Client(HardwareClientBase): + # handler is a BitBox02_Handler, importing it would lead to a circular dependency + def __init__(self, handler: Any, device: Device, config: SimpleConfig): + self.bitbox02_device = None + self.handler = handler + self.device_descriptor = device + self.config = config + self.bitbox_hid_info = None + if self.config.get("bitbox02") is None: + bitbox02_config: dict = { + "remote_static_noise_keys": [], + "noise_privkey": None, + } + self.config.set_key("bitbox02", bitbox02_config) + + bitboxes = devices.get_any_bitbox02s() + for bitbox in bitboxes: + if ( + bitbox["path"] == self.device_descriptor.path + and bitbox["interface_number"] + == self.device_descriptor.interface_number + ): + self.bitbox_hid_info = bitbox + if self.bitbox_hid_info is None: + raise Exception("No BitBox02 detected") + + def label(self) -> str: + return "BitBox02" + + def is_initialized(self) -> bool: + return True + + def close(self): + try: + self.bitbox02_device.close() + except: + pass + + def has_usable_connection_with_device(self) -> bool: + if self.bitbox_hid_info is None: + return False + return True + + def pairing_dialog(self, wizard: bool = True): + def pairing_step(code): + msg = "Please compare and confirm the pairing code on your BitBox02:\n" + choice = [code] + if wizard == True: + return self.handler.win.query_choice(msg, choice) + self.handler.pairing_code_dialog(code) + + def exists_remote_static_pubkey(pubkey: bytes) -> bool: + bitbox02_config = self.config.get("bitbox02") + noise_keys = bitbox02_config.get("remote_static_noise_keys") + if noise_keys is not None: + if pubkey.hex() in [noise_key for noise_key in noise_keys]: + return True + return False + + def set_remote_static_pubkey(pubkey: bytes) -> None: + if not exists_remote_static_pubkey(pubkey): + bitbox02_config = self.config.get("bitbox02") + if bitbox02_config.get("remote_static_noise_keys") is not None: + bitbox02_config["remote_static_noise_keys"].append(pubkey.hex()) + else: + bitbox02_config["remote_static_noise_keys"] = [pubkey.hex()] + self.config.set_key("bitbox02", bitbox02_config) + + def get_noise_privkey() -> Optional[bytes]: + bitbox02_config = self.config.get("bitbox02") + privkey = bitbox02_config.get("noise_privkey") + if privkey is not None: + return bytes.fromhex(privkey) + return None + + def set_noise_privkey(privkey: bytes) -> None: + bitbox02_config = self.config.get("bitbox02") + bitbox02_config["noise_privkey"] = privkey.hex() + self.config.set_key("bitbox02", bitbox02_config) + + def attestation_warning() -> None: + self.handler.attestation_failed_warning( + "The BitBox02 attestation failed.\nTry reconnecting the BitBox02.\nWarning: The device might not be genuine, if the\n problem persists please contact Shift support." + ) + + class NoiseConfig(bitbox_api_protocol.BitBoxNoiseConfig): + """NoiseConfig extends BitBoxNoiseConfig""" + + def show_pairing(self, code: str) -> bool: + choice = [code] + try: + reply = pairing_step(code) + except: + # Close the hid device on exception + hid_device.close() + raise + return True + + def attestation_check(self, result: bool) -> None: + if not result: + attestation_warning() + + def contains_device_static_pubkey(self, pubkey: bytes) -> bool: + return exists_remote_static_pubkey(pubkey) + + def add_device_static_pubkey(self, pubkey: bytes) -> None: + return set_remote_static_pubkey(pubkey) + + def get_app_static_privkey(self) -> Optional[bytes]: + return get_noise_privkey() + + def set_app_static_privkey(self, privkey: bytes) -> None: + return set_noise_privkey(privkey) + + if self.bitbox02_device is None: + hid_device = hid.device() + hid_device.open_path(self.bitbox_hid_info["path"]) + + self.bitbox02_device = bitbox02.BitBox02( + transport=u2fhid.U2FHid(hid_device), + device_info=self.bitbox_hid_info, + noise_config=NoiseConfig(), + ) + + if not self.bitbox02_device.device_info()["initialized"]: + raise Exception( + "Please initialize the BitBox02 using the BitBox app first before using the BitBox02 in electrum" + ) + + def check_device_firmware_version(self) -> bool: + if self.bitbox02_device is None: + raise Exception( + "Need to setup communication first before attempting any BitBox02 calls" + ) + return self.bitbox02_device.check_firmware_version() + + def coin_network_from_electrum_network(self) -> int: + if constants.net.TESTNET: + return bitbox02.btc.TBTC + return bitbox02.btc.BTC + + def get_password_for_storage_encryption(self) -> str: + derivation = get_derivation_used_for_hw_device_encryption() + derivation_list = bip32.convert_bip32_path_to_list_of_uint32(derivation) + xpub = self.bitbox02_device.electrum_encryption_key(derivation_list) + node = bip32.BIP32Node.from_xkey(xpub, net = constants.BitcoinMainnet()).subkey_at_public_derivation(()) + return node.eckey.get_public_key_bytes(compressed=True).hex() + + def get_xpub(self, bip32_path: str, xtype: str, display: bool = False) -> str: + if self.bitbox02_device is None: + self.pairing_dialog(wizard=False) + + if self.bitbox02_device is None: + raise Exception( + "Need to setup communication first before attempting any BitBox02 calls" + ) + + if not self.bitbox02_device.device_info()["initialized"]: + raise UserFacingException( + "Please initialize the BitBox02 using the BitBox app first before using the BitBox02 in electrum" + ) + + xpub_keypath = bip32.convert_bip32_path_to_list_of_uint32(bip32_path) + coin_network = self.coin_network_from_electrum_network() + + if xtype == "p2wpkh": + if coin_network == bitbox02.btc.BTC: + out_type = bitbox02.btc.BTCPubRequest.ZPUB + else: + out_type = bitbox02.btc.BTCPubRequest.VPUB + elif xtype == "p2wpkh-p2sh": + if coin_network == bitbox02.btc.BTC: + out_type = bitbox02.btc.BTCPubRequest.YPUB + else: + out_type = bitbox02.btc.BTCPubRequest.UPUB + elif xtype == "p2wsh": + if coin_network == bitbox02.btc.BTC: + out_type = bitbox02.btc.BTCPubRequest.CAPITAL_ZPUB + else: + out_type = bitbox02.btc.BTCPubRequest.CAPITAL_VPUB + # The other legacy types are not supported + else: + raise Exception("invalid xtype:{}".format(xtype)) + + return self.bitbox02_device.btc_xpub( + keypath=xpub_keypath, + xpub_type=out_type, + coin=coin_network, + display=display, + ) + + def request_root_fingerprint_from_device(self) -> str: + if self.bitbox02_device is None: + raise Exception( + "Need to setup communication first before attempting any BitBox02 calls" + ) + + return self.bitbox02_device.root_fingerprint().hex() + + def is_pairable(self) -> bool: + if self.bitbox_hid_info is None: + return False + return True + + def btc_multisig_config( + self, coin, bip32_path: List[int], wallet: Multisig_Wallet + ): + """ + Set and get a multisig config with the current device and some other arbitrary xpubs. + Registers it on the device if not already registered. + """ + + if self.bitbox02_device is None: + raise Exception( + "Need to setup communication first before attempting any BitBox02 calls" + ) + + account_keypath = bip32_path[:4] + xpubs = wallet.get_master_public_keys() + our_xpub = self.get_xpub( + bip32.convert_bip32_intpath_to_strpath(account_keypath), "p2wsh" + ) + + multisig_config = bitbox02.btc.BTCScriptConfig( + multisig=bitbox02.btc.BTCScriptConfig.Multisig( + threshold=wallet.m, + xpubs=[util.parse_xpub(xpub) for xpub in xpubs], + our_xpub_index=xpubs.index(our_xpub), + ) + ) + + is_registered = self.bitbox02_device.btc_is_script_config_registered( + coin, multisig_config, account_keypath + ) + if not is_registered: + name = self.handler.name_multisig_account() + try: + self.bitbox02_device.btc_register_script_config( + coin=coin, + script_config=multisig_config, + keypath=account_keypath, + name=name, + ) + except bitbox02.DuplicateEntryException: + raise + except: + raise UserFacingException("Failed to register multisig\naccount configuration on BitBox02") + return multisig_config + + def show_address( + self, bip32_path: str, address_type: str, wallet: Deterministic_Wallet + ) -> str: + + if self.bitbox02_device is None: + raise Exception( + "Need to setup communication first before attempting any BitBox02 calls" + ) + + address_keypath = bip32.convert_bip32_path_to_list_of_uint32(bip32_path) + coin_network = self.coin_network_from_electrum_network() + + if address_type == "p2wpkh": + script_config = bitbox02.btc.BTCScriptConfig( + simple_type=bitbox02.btc.BTCScriptConfig.P2WPKH + ) + elif address_type == "p2wpkh-p2sh": + script_config = bitbox02.btc.BTCScriptConfig( + simple_type=bitbox02.btc.BTCScriptConfig.P2WPKH_P2SH + ) + elif address_type == "p2wsh": + if type(wallet) is Multisig_Wallet: + script_config = self.btc_multisig_config( + coin_network, address_keypath, wallet + ) + else: + raise Exception("Can only use p2wsh with multisig wallets") + else: + raise Exception( + "invalid address xtype: {} is not supported by the BitBox02".format( + address_type + ) + ) + + return self.bitbox02_device.btc_address( + keypath=address_keypath, + coin=coin_network, + script_config=script_config, + display=True, + ) + + def sign_transaction( + self, + keystore: Hardware_KeyStore, + tx: PartialTransaction, + wallet: Deterministic_Wallet, + ): + if tx.is_complete(): + return + + if self.bitbox02_device is None: + raise Exception( + "Need to setup communication first before attempting any BitBox02 calls" + ) + + coin = bitbox02.btc.BTC + if constants.net.TESTNET: + coin = bitbox02.btc.TBTC + + tx_script_type = None + + # Build BTCInputType list + inputs = [] + for txin in tx.inputs(): + _, full_path = keystore.find_my_pubkey_in_txinout(txin) + + if full_path is None: + raise Exception( + "A wallet owned pubkey was not found in the transaction input to be signed" + ) + + inputs.append( + { + "prev_out_hash": txin.prevout.txid[::-1], + "prev_out_index": txin.prevout.out_idx, + "prev_out_value": txin.value_sats(), + "sequence": txin.nsequence, + "keypath": full_path, + } + ) + + if tx_script_type == None: + tx_script_type = txin.script_type + elif tx_script_type != txin.script_type: + raise Exception("Cannot mix different input script types") + + if tx_script_type == "p2wpkh": + tx_script_type = bitbox02.btc.BTCScriptConfig( + simple_type=bitbox02.btc.BTCScriptConfig.P2WPKH + ) + elif tx_script_type == "p2wpkh-p2sh": + tx_script_type = bitbox02.btc.BTCScriptConfig( + simple_type=bitbox02.btc.BTCScriptConfig.P2WPKH_P2SH + ) + elif tx_script_type == "p2wsh": + if type(wallet) is Multisig_Wallet: + tx_script_type = self.btc_multisig_config(coin, full_path, wallet) + else: + raise Exception("Can only use p2wsh with multisig wallets") + else: + raise UserFacingException( + "invalid input script type: {} is not supported by the BitBox02".format( + tx_script_type + ) + ) + + # Build BTCOutputType list + outputs = [] + for txout in tx.outputs(): + assert txout.address + # check for change + if txout.is_change: + _, change_pubkey_path = keystore.find_my_pubkey_in_txinout(txout) + outputs.append( + bitbox02.BTCOutputInternal( + keypath=change_pubkey_path, value=txout.value, + ) + ) + else: + addrtype, pubkey_hash = bitcoin.address_to_hash(txout.address) + if addrtype == bitcoin.WIF_SCRIPT_TYPES["p2pkh"]: + output_type = bitbox02.btc.P2PKH + elif addrtype == bitcoin.WIF_SCRIPT_TYPES["p2sh"]: + output_type = bitbox02.btc.P2SH + elif addrtype == bitcoin.WIF_SCRIPT_TYPES["p2wpkh"]: + output_type = bitbox02.btc.P2WPKH + elif addrtype == bitcoin.WIF_SCRIPT_TYPES["p2wsh"]: + output_type = bitbox02.btc.P2WSH + else: + raise UserFacingException( + "Received unsupported output type during transaction signing: {} is not supported by the BitBox02".format( + addrtype + ) + ) + outputs.append( + bitbox02.BTCOutputExternal( + output_type=output_type, + output_hash=pubkey_hash, + value=txout.value, + ) + ) + + if type(wallet) is Standard_Wallet: + keypath_account = full_path[:3] + elif type(wallet) is Multisig_Wallet: + keypath_account = full_path[:4] + else: + raise Exception( + "BitBox02 does not support this wallet type: {}".format(type(wallet)) + ) + + sigs = self.bitbox02_device.btc_sign( + coin, + tx_script_type, + keypath_account=keypath_account, + inputs=inputs, + outputs=outputs, + locktime=tx.locktime, + version=tx.version, + ) + + # Fill signatures + if len(sigs) != len(tx.inputs()): + raise Exception("Incorrect number of inputs signed.") # Should never occur + signatures = [bh2u(ecc.der_sig_from_sig_string(x[1])) + "01" for x in sigs] + tx.update_signatures(signatures) + + +class BitBox02_KeyStore(Hardware_KeyStore): + hw_type = "bitbox02" + device = "BitBox02" + plugin: "BitBox02Plugin" + + def __init__(self, d: StoredDict): + super().__init__(d) + self.force_watching_only = False + self.ux_busy = False + + def get_client(self): + return self.plugin.get_client(self) + + def give_error(self, message: Exception, clear_client: bool = False): + self.logger.info(message) + if not self.ux_busy: + self.handler.show_error(message) + else: + self.ux_busy = False + if clear_client: + self.client = None + raise UserFacingException(message) + + def decrypt_message(self, pubkey, message, password): + raise UserFacingException( + _( + "Message encryption, decryption and signing are currently not supported for {}" + ).format(self.device) + ) + + def sign_message(self, sequence, message, password): + raise UserFacingException( + _( + "Message encryption, decryption and signing are currently not supported for {}" + ).format(self.device) + ) + + def sign_transaction(self, tx: PartialTransaction, password: str): + if tx.is_complete(): + return + client = self.get_client() + + try: + try: + self.handler.show_message("Authorize Transaction...") + client.sign_transaction(self, tx, self.handler.win.wallet) + + finally: + self.handler.finished() + + except Exception as e: + self.logger.exception("") + self.give_error(e, True) + return + + def show_address( + self, sequence: Tuple[int, int], txin_type: str, wallet: Deterministic_Wallet + ): + client = self.get_client() + address_path = "{}/{}/{}".format( + self.get_derivation_prefix(), sequence[0], sequence[1] + ) + try: + try: + self.handler.show_message(_("Showing address ...")) + dev_addr = client.show_address(address_path, txin_type, wallet) + finally: + self.handler.finished() + except Exception as e: + self.logger.exception("") + self.handler.show_error(e) + +class BitBox02Plugin(HW_PluginBase): + keystore_class = BitBox02_KeyStore + + DEVICE_IDS = [(0x03EB, 0x2403)] + + SUPPORTED_XTYPES = ("p2wpkh-p2sh", "p2wpkh", "p2wsh") + + def __init__(self, parent: HW_PluginBase, config: SimpleConfig, name: str): + super().__init__(parent, config, name) + + self.libraries_available = self.check_libraries_available() + if not self.libraries_available: + return + self.device_manager().register_devices(self.DEVICE_IDS) + + def get_library_version(self): + try: + from bitbox02 import bitbox02 + version = bitbox02.__version__ + except: + version = "unknown" + if requirements_ok: + return version + else: + raise ImportError() + + + # handler is a BitBox02_Handler + def create_client(self, device: Device, handler: Any) -> BitBox02Client: + if not handler: + self.handler = handler + return BitBox02Client(handler, device, self.config) + + def setup_device( + self, device_info: DeviceInfo, wizard: BaseWizard, purpose: int + ): + devmgr = self.device_manager() + device_id = device_info.device.id_ + client = devmgr.client_by_id(device_id) + if client is None: + raise UserFacingException( + _("Failed to create a client for this device.") + + "\n" + + _("Make sure it is in the correct state.") + ) + client.handler = self.create_handler(wizard) + if client.bitbox02_device is None: + client.pairing_dialog() + + def get_xpub( + self, device_id: bytes, derivation: str, xtype: str, wizard: BaseWizard + ): + if xtype not in self.SUPPORTED_XTYPES: + raise ScriptTypeNotSupported( + _("This type of script is not supported with {}.").format(self.device) + ) + devmgr = self.device_manager() + client = devmgr.client_by_id(device_id) + if client.bitbox02_device is None: + client.handler = self.create_handler(wizard) + client.pairing_dialog() + return client.get_xpub(derivation, xtype) + + def get_client(self, keystore: BitBox02_KeyStore, force_pair: bool = True): + devmgr = self.device_manager() + handler = keystore.handler + with devmgr.hid_lock: + client = devmgr.client_for_keystore(self, handler, keystore, force_pair) + + return client + + def show_address( + self, + wallet: Deterministic_Wallet, + address: str, + keystore: BitBox02_KeyStore = None, + ): + if keystore is None: + keystore = wallet.get_keystore() + if not self.show_address_helper(wallet, address, keystore): + return + + txin_type = wallet.get_txin_type(address) + sequence = wallet.get_address_index(address) + keystore.show_address(sequence, txin_type, wallet) diff --git a/electrum/plugins/bitbox02/qt.py b/electrum/plugins/bitbox02/qt.py new file mode 100644 index 0000000000..74535b7997 --- /dev/null +++ b/electrum/plugins/bitbox02/qt.py @@ -0,0 +1,174 @@ +import time, os +from functools import partial +import copy + +from PyQt5.QtCore import Qt, pyqtSignal +from PyQt5.QtWidgets import ( + QPushButton, + QLabel, + QVBoxLayout, + QWidget, + QGridLayout, + QLineEdit, + QHBoxLayout, +) + +from PyQt5.QtCore import Qt, QMetaObject, Q_RETURN_ARG, pyqtSignal, pyqtSlot + +from electrum.gui.qt.util import ( + WindowModalDialog, + Buttons, + OkButton, + CancelButton, + get_parent_main_window, +) +from electrum.gui.qt.transaction_dialog import TxDialog + +from electrum.i18n import _ +from electrum.plugin import hook +from electrum.wallet import Multisig_Wallet +from electrum.transaction import PartialTransaction +from electrum import keystore + +from .bitbox02 import BitBox02Plugin +from ..hw_wallet.qt import QtHandlerBase, QtPluginBase +from ..hw_wallet.plugin import only_hook_if_libraries_available, LibraryFoundButUnusable + + +class Plugin(BitBox02Plugin, QtPluginBase): + icon_unpaired = "bitbox02_unpaired.png" + icon_paired = "bitbox02.png" + + def create_handler(self, window): + return BitBox02_Handler(window) + + @only_hook_if_libraries_available + @hook + def receive_menu(self, menu, addrs, wallet): + # Context menu on each address in the Addresses Tab, right click... + if len(addrs) != 1: + return + for keystore in wallet.get_keystores(): + if type(keystore) == self.keystore_class: + + def show_address(keystore=keystore): + keystore.thread.add( + partial(self.show_address, wallet, addrs[0], keystore=keystore) + ) + + device_name = "{} ({})".format(self.device, keystore.label) + menu.addAction(_("Show on {}").format(device_name), show_address) + + @only_hook_if_libraries_available + @hook + def show_xpub_button(self, main_window, dialog, labels_clayout): + # user is about to see the "Wallet Information" dialog + # - add a button to show the xpub on the BitBox02 device + wallet = main_window.wallet + if not any(type(ks) == self.keystore_class for ks in wallet.get_keystores()): + # doesn't involve a BitBox02 wallet, hide feature + return + + btn = QPushButton(_("Show on BitBox02")) + + def on_button_click(): + selected_keystore_index = 0 + if labels_clayout is not None: + selected_keystore_index = labels_clayout.selected_index() + keystores = wallet.get_keystores() + selected_keystore = keystores[selected_keystore_index] + derivation = selected_keystore.get_derivation_prefix() + if type(selected_keystore) != self.keystore_class: + main_window.show_error("Select a BitBox02 xpub") + selected_keystore.get_client().get_xpub( + derivation, keystore.xtype_from_derivation(derivation), True + ) + + btn.clicked.connect(lambda unused: on_button_click()) + return btn + + +class BitBox02_Handler(QtHandlerBase): + setup_signal = pyqtSignal() + + def __init__(self, win): + super(BitBox02_Handler, self).__init__(win, "BitBox02") + self.setup_signal.connect(self.setup_dialog) + + def message_dialog(self, msg): + self.clear_dialog() + self.dialog = dialog = WindowModalDialog( + self.top_level_window(), _("BitBox02 Status") + ) + l = QLabel(msg) + vbox = QVBoxLayout(dialog) + vbox.addWidget(l) + dialog.show() + + def attestation_failed_warning(self, msg): + self.clear_dialog() + self.dialog = dialog = WindowModalDialog(None, "BitBox02 Attestation Failed") + l = QLabel(msg) + vbox = QVBoxLayout(dialog) + vbox.addWidget(l) + okButton = OkButton(dialog) + vbox.addWidget(okButton) + dialog.setLayout(vbox) + dialog.exec_() + return + + def pairing_code_dialog(self, code): + self.clear_dialog() + self.dialog = dialog = WindowModalDialog(None, "BitBox02 Pairing Code") + l = QLabel(code) + vbox = QVBoxLayout(dialog) + vbox.addWidget(l) + vbox.addLayout(Buttons(CancelButton(dialog), OkButton(dialog))) + dialog.setLayout(vbox) + dialog.exec_() + return + + def get_setup(self): + self.done.clear() + self.setup_signal.emit() + self.done.wait() + return + + def name_multisig_account(self): + return QMetaObject.invokeMethod( + self, + "_name_multisig_account", + Qt.BlockingQueuedConnection, + Q_RETURN_ARG(str), + ) + + @pyqtSlot(result=str) + def _name_multisig_account(self): + dialog = WindowModalDialog(None, "Create Multisig Account") + vbox = QVBoxLayout() + label = QLabel( + _( + "Enter a descriptive name for your multisig account.\nYou should later be able to use the name to uniquely identify this multisig account" + ) + ) + hl = QHBoxLayout() + hl.addWidget(label) + name = QLineEdit() + name.setMaxLength(30) + name.resize(200, 40) + he = QHBoxLayout() + he.addWidget(name) + okButton = OkButton(dialog) + hlb = QHBoxLayout() + hlb.addWidget(okButton) + hlb.addStretch(2) + vbox.addLayout(hl) + vbox.addLayout(he) + vbox.addLayout(hlb) + dialog.setLayout(vbox) + dialog.exec_() + return name.text().strip() + + def setup_dialog(self): + self.show_error(_("Please initialize your BitBox02 while connected.")) + return diff --git a/electrum/plugins/coldcard/qt.py b/electrum/plugins/coldcard/qt.py index 358fa59b88..4c590c859c 100644 --- a/electrum/plugins/coldcard/qt.py +++ b/electrum/plugins/coldcard/qt.py @@ -57,7 +57,7 @@ def wallet_info_buttons(self, main_window, dialog): btn = QPushButton(_("Export for Coldcard")) btn.clicked.connect(lambda unused: self.export_multisig_setup(main_window, wallet)) - return Buttons(btn, CloseButton(dialog)) + return btn def export_multisig_setup(self, main_window, wallet): From c0c3627bd2e9d60bc3cd971ac87c18f8077d9b9d Mon Sep 17 00:00:00 2001 From: SomberNight Date: Mon, 6 Apr 2020 13:54:17 +0200 Subject: [PATCH 045/117] bitbox02: adapt to updated master --- electrum/plugins/bitbox02/bitbox02.py | 21 +++++---------------- electrum/plugins/hw_wallet/plugin.py | 2 +- 2 files changed, 6 insertions(+), 17 deletions(-) diff --git a/electrum/plugins/bitbox02/bitbox02.py b/electrum/plugins/bitbox02/bitbox02.py index c7098d6318..b56075aa23 100644 --- a/electrum/plugins/bitbox02/bitbox02.py +++ b/electrum/plugins/bitbox02/bitbox02.py @@ -569,39 +569,28 @@ def create_client(self, device: Device, handler: Any) -> BitBox02Client: def setup_device( self, device_info: DeviceInfo, wizard: BaseWizard, purpose: int ): - devmgr = self.device_manager() device_id = device_info.device.id_ - client = devmgr.client_by_id(device_id) - if client is None: - raise UserFacingException( - _("Failed to create a client for this device.") - + "\n" - + _("Make sure it is in the correct state.") - ) - client.handler = self.create_handler(wizard) + client = self.scan_and_create_client_for_device(device_id=device_id, wizard=wizard) if client.bitbox02_device is None: client.pairing_dialog() + return client def get_xpub( - self, device_id: bytes, derivation: str, xtype: str, wizard: BaseWizard + self, device_id: str, derivation: str, xtype: str, wizard: BaseWizard ): if xtype not in self.SUPPORTED_XTYPES: raise ScriptTypeNotSupported( _("This type of script is not supported with {}.").format(self.device) ) - devmgr = self.device_manager() - client = devmgr.client_by_id(device_id) + client = self.scan_and_create_client_for_device(device_id=device_id, wizard=wizard) if client.bitbox02_device is None: - client.handler = self.create_handler(wizard) client.pairing_dialog() return client.get_xpub(derivation, xtype) def get_client(self, keystore: BitBox02_KeyStore, force_pair: bool = True): devmgr = self.device_manager() handler = keystore.handler - with devmgr.hid_lock: - client = devmgr.client_for_keystore(self, handler, keystore, force_pair) - + client = devmgr.client_for_keystore(self, handler, keystore, force_pair) return client def show_address( diff --git a/electrum/plugins/hw_wallet/plugin.py b/electrum/plugins/hw_wallet/plugin.py index 68335f746d..884732dadb 100644 --- a/electrum/plugins/hw_wallet/plugin.py +++ b/electrum/plugins/hw_wallet/plugin.py @@ -165,7 +165,7 @@ def create_client(self, device: 'Device', handler: Optional['HardwareHandlerBase']) -> Optional['HardwareClientBase']: raise NotImplementedError() - def get_xpub(self, device_id, derivation: str, xtype, wizard: 'BaseWizard') -> str: + def get_xpub(self, device_id: str, derivation: str, xtype, wizard: 'BaseWizard') -> str: raise NotImplementedError() def create_handler(self, window) -> 'HardwareHandlerBase': From 5f5a1e96abb7a75d6fb2b8eecb151ddfec0744e0 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Mon, 6 Apr 2020 16:05:33 +0200 Subject: [PATCH 046/117] bitbox02: add udev rules --- contrib/udev/53-hid-bitbox02.rules | 1 + contrib/udev/54-hid-bitbox02.rules | 1 + contrib/udev/README.md | 3 ++- 3 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 contrib/udev/53-hid-bitbox02.rules create mode 100644 contrib/udev/54-hid-bitbox02.rules diff --git a/contrib/udev/53-hid-bitbox02.rules b/contrib/udev/53-hid-bitbox02.rules new file mode 100644 index 0000000000..2daffc03ba --- /dev/null +++ b/contrib/udev/53-hid-bitbox02.rules @@ -0,0 +1 @@ +SUBSYSTEM=="usb", TAG+="uaccess", TAG+="udev-acl", SYMLINK+="bitbox02_%n", ATTRS{idVendor}=="03eb", ATTRS{idProduct}=="2403" diff --git a/contrib/udev/54-hid-bitbox02.rules b/contrib/udev/54-hid-bitbox02.rules new file mode 100644 index 0000000000..1b74e47743 --- /dev/null +++ b/contrib/udev/54-hid-bitbox02.rules @@ -0,0 +1 @@ +KERNEL=="hidraw*", SUBSYSTEM=="hidraw", ATTRS{idVendor}=="03eb", ATTRS{idProduct}=="2403", TAG+="uaccess", TAG+="udev-acl", SYMLINK+="bitbox02-%n" diff --git a/contrib/udev/README.md b/contrib/udev/README.md index 6ff403a4fd..451ef2b2fc 100644 --- a/contrib/udev/README.md +++ b/contrib/udev/README.md @@ -6,7 +6,8 @@ These are necessary for the devices to be usable on Linux environments. - `20-hw1.rules` (Ledger): https://github.com/LedgerHQ/udev-rules/blob/master/20-hw1.rules - `51-coinkite.rules` (Coldcard): https://github.com/Coldcard/ckcc-protocol/blob/master/51-coinkite.rules - - `51-hid-digitalbitbox.rules`, `52-hid-digitalbitbox.rules` (Digital Bitbox): https://shiftcrypto.ch/start_linux + - `51-hid-digitalbitbox.rules`, `52-hid-digitalbitbox.rules` (Digital Bitbox): https://github.com/digitalbitbox/bitbox-wallet-app/blob/master/frontends/qt/resources/deb-afterinstall.sh + - `53-hid-bitbox02.rules`, `54-hid-bitbox02.rules` (BitBox02): https://github.com/digitalbitbox/bitbox-wallet-app/blob/master/frontends/qt/resources/deb-afterinstall.sh - `51-trezor.rules` (Trezor): https://github.com/trezor/trezor-common/blob/master/udev/51-trezor.rules - `51-usb-keepkey.rules` (Keepkey): https://github.com/keepkey/udev-rules/blob/master/51-usb-keepkey.rules - `51-safe-t.rules` (Archos): https://github.com/archos-safe-t/safe-t-common/blob/master/udev/51-safe-t.rules From 15102855c15eae90ca158935dc9e052b24bf88c9 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Mon, 6 Apr 2020 16:13:13 +0200 Subject: [PATCH 047/117] bitbox02: fix pairing_dialog --- electrum/plugins/bitbox02/bitbox02.py | 56 ++++++++++++++------------- electrum/plugins/bitbox02/qt.py | 11 ------ 2 files changed, 30 insertions(+), 37 deletions(-) diff --git a/electrum/plugins/bitbox02/bitbox02.py b/electrum/plugins/bitbox02/bitbox02.py index b56075aa23..bcc3ebac70 100644 --- a/electrum/plugins/bitbox02/bitbox02.py +++ b/electrum/plugins/bitbox02/bitbox02.py @@ -4,7 +4,7 @@ import hid import hashlib -from typing import TYPE_CHECKING, Dict, Tuple, Optional, List, Any +from typing import TYPE_CHECKING, Dict, Tuple, Optional, List, Any, Callable from electrum import bip32, constants from electrum.i18n import _ @@ -84,16 +84,22 @@ def close(self): def has_usable_connection_with_device(self) -> bool: if self.bitbox_hid_info is None: - return False + return False return True def pairing_dialog(self, wizard: bool = True): - def pairing_step(code): - msg = "Please compare and confirm the pairing code on your BitBox02:\n" - choice = [code] - if wizard == True: - return self.handler.win.query_choice(msg, choice) - self.handler.pairing_code_dialog(code) + def pairing_step(code: str, device_response: Callable[[], bool]) -> bool: + msg = "Please compare and confirm the pairing code on your BitBox02:\n" + code + self.handler.show_message(msg) + try: + res = device_response() + except: + # Close the hid device on exception + hid_device.close() + raise + finally: + self.handler.finished() + return res def exists_remote_static_pubkey(pubkey: bytes) -> bool: bitbox02_config = self.config.get("bitbox02") @@ -132,15 +138,8 @@ def attestation_warning() -> None: class NoiseConfig(bitbox_api_protocol.BitBoxNoiseConfig): """NoiseConfig extends BitBoxNoiseConfig""" - def show_pairing(self, code: str) -> bool: - choice = [code] - try: - reply = pairing_step(code) - except: - # Close the hid device on exception - hid_device.close() - raise - return True + def show_pairing(self, code: str, device_response: Callable[[], bool]) -> bool: + return pairing_step(code, device_response) def attestation_check(self, result: bool) -> None: if not result: @@ -168,6 +167,10 @@ def set_app_static_privkey(self, privkey: bytes) -> None: noise_config=NoiseConfig(), ) + self.fail_if_not_initialized() + + def fail_if_not_initialized(self) -> None: + assert self.bitbox02_device if not self.bitbox02_device.device_info()["initialized"]: raise Exception( "Please initialize the BitBox02 using the BitBox app first before using the BitBox02 in electrum" @@ -201,10 +204,7 @@ def get_xpub(self, bip32_path: str, xtype: str, display: bool = False) -> str: "Need to setup communication first before attempting any BitBox02 calls" ) - if not self.bitbox02_device.device_info()["initialized"]: - raise UserFacingException( - "Please initialize the BitBox02 using the BitBox app first before using the BitBox02 in electrum" - ) + self.fail_if_not_initialized() xpub_keypath = bip32.convert_bip32_path_to_list_of_uint32(bip32_path) coin_network = self.coin_network_from_electrum_network() @@ -502,11 +502,12 @@ def sign_transaction(self, tx: PartialTransaction, password: str): if tx.is_complete(): return client = self.get_client() + assert isinstance(client, BitBox02Client) try: try: self.handler.show_message("Authorize Transaction...") - client.sign_transaction(self, tx, self.handler.win.wallet) + client.sign_transaction(self, tx, self.handler.get_wallet()) finally: self.handler.finished() @@ -535,7 +536,7 @@ def show_address( class BitBox02Plugin(HW_PluginBase): keystore_class = BitBox02_KeyStore - + minimum_library = (2, 0, 2) DEVICE_IDS = [(0x03EB, 0x2403)] SUPPORTED_XTYPES = ("p2wpkh-p2sh", "p2wpkh", "p2wsh") @@ -571,8 +572,11 @@ def setup_device( ): device_id = device_info.device.id_ client = self.scan_and_create_client_for_device(device_id=device_id, wizard=wizard) + assert isinstance(client, BitBox02Client) if client.bitbox02_device is None: - client.pairing_dialog() + wizard.run_task_without_blocking_gui( + task=lambda client=client: client.pairing_dialog()) + client.fail_if_not_initialized() return client def get_xpub( @@ -583,8 +587,8 @@ def get_xpub( _("This type of script is not supported with {}.").format(self.device) ) client = self.scan_and_create_client_for_device(device_id=device_id, wizard=wizard) - if client.bitbox02_device is None: - client.pairing_dialog() + assert isinstance(client, BitBox02Client) + assert client.bitbox02_device is not None return client.get_xpub(derivation, xtype) def get_client(self, keystore: BitBox02_KeyStore, force_pair: bool = True): diff --git a/electrum/plugins/bitbox02/qt.py b/electrum/plugins/bitbox02/qt.py index 74535b7997..4404ffcdd7 100644 --- a/electrum/plugins/bitbox02/qt.py +++ b/electrum/plugins/bitbox02/qt.py @@ -117,17 +117,6 @@ def attestation_failed_warning(self, msg): dialog.exec_() return - def pairing_code_dialog(self, code): - self.clear_dialog() - self.dialog = dialog = WindowModalDialog(None, "BitBox02 Pairing Code") - l = QLabel(code) - vbox = QVBoxLayout(dialog) - vbox.addWidget(l) - vbox.addLayout(Buttons(CancelButton(dialog), OkButton(dialog))) - dialog.setLayout(vbox) - dialog.exec_() - return - def get_setup(self): self.done.clear() self.setup_signal.emit() From 0268b63fcb67b9b08e98f109031a81925fc0d901 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Mon, 6 Apr 2020 16:21:09 +0200 Subject: [PATCH 048/117] bitbox02: rm some dead code --- electrum/plugins/bitbox02/bitbox02.py | 5 +++-- electrum/plugins/bitbox02/qt.py | 24 ------------------------ electrum/plugins/coldcard/cmdline.py | 6 ------ electrum/plugins/coldcard/qt.py | 16 +--------------- 4 files changed, 4 insertions(+), 47 deletions(-) diff --git a/electrum/plugins/bitbox02/bitbox02.py b/electrum/plugins/bitbox02/bitbox02.py index bcc3ebac70..087546dc45 100644 --- a/electrum/plugins/bitbox02/bitbox02.py +++ b/electrum/plugins/bitbox02/bitbox02.py @@ -131,8 +131,9 @@ def set_noise_privkey(privkey: bytes) -> None: self.config.set_key("bitbox02", bitbox02_config) def attestation_warning() -> None: - self.handler.attestation_failed_warning( - "The BitBox02 attestation failed.\nTry reconnecting the BitBox02.\nWarning: The device might not be genuine, if the\n problem persists please contact Shift support." + self.handler.show_error( + "The BitBox02 attestation failed.\nTry reconnecting the BitBox02.\nWarning: The device might not be genuine, if the\n problem persists please contact Shift support.", + blocking=True ) class NoiseConfig(bitbox_api_protocol.BitBoxNoiseConfig): diff --git a/electrum/plugins/bitbox02/qt.py b/electrum/plugins/bitbox02/qt.py index 4404ffcdd7..2a63b681f1 100644 --- a/electrum/plugins/bitbox02/qt.py +++ b/electrum/plugins/bitbox02/qt.py @@ -89,11 +89,9 @@ def on_button_click(): class BitBox02_Handler(QtHandlerBase): - setup_signal = pyqtSignal() def __init__(self, win): super(BitBox02_Handler, self).__init__(win, "BitBox02") - self.setup_signal.connect(self.setup_dialog) def message_dialog(self, msg): self.clear_dialog() @@ -105,24 +103,6 @@ def message_dialog(self, msg): vbox.addWidget(l) dialog.show() - def attestation_failed_warning(self, msg): - self.clear_dialog() - self.dialog = dialog = WindowModalDialog(None, "BitBox02 Attestation Failed") - l = QLabel(msg) - vbox = QVBoxLayout(dialog) - vbox.addWidget(l) - okButton = OkButton(dialog) - vbox.addWidget(okButton) - dialog.setLayout(vbox) - dialog.exec_() - return - - def get_setup(self): - self.done.clear() - self.setup_signal.emit() - self.done.wait() - return - def name_multisig_account(self): return QMetaObject.invokeMethod( self, @@ -157,7 +137,3 @@ def _name_multisig_account(self): dialog.setLayout(vbox) dialog.exec_() return name.text().strip() - - def setup_dialog(self): - self.show_error(_("Please initialize your BitBox02 while connected.")) - return diff --git a/electrum/plugins/coldcard/cmdline.py b/electrum/plugins/coldcard/cmdline.py index ab86f463c6..4f246ec771 100644 --- a/electrum/plugins/coldcard/cmdline.py +++ b/electrum/plugins/coldcard/cmdline.py @@ -28,12 +28,6 @@ def yes_no_question(self, msg): def stop(self): pass - def show_message(self, msg, on_cancel=None): - print_stderr(msg) - - def show_error(self, msg, blocking=False): - print_stderr(msg) - def update_status(self, b): _logger.info(f'hw device status {b}') diff --git a/electrum/plugins/coldcard/qt.py b/electrum/plugins/coldcard/qt.py index 4c590c859c..a9b8ce51a3 100644 --- a/electrum/plugins/coldcard/qt.py +++ b/electrum/plugins/coldcard/qt.py @@ -77,15 +77,10 @@ def show_settings_dialog(self, window, keystore): class Coldcard_Handler(QtHandlerBase): - setup_signal = pyqtSignal() - #auth_signal = pyqtSignal(object) def __init__(self, win): super(Coldcard_Handler, self).__init__(win, 'Coldcard') - self.setup_signal.connect(self.setup_dialog) - #self.auth_signal.connect(self.auth_dialog) - def message_dialog(self, msg): self.clear_dialog() self.dialog = dialog = WindowModalDialog(self.top_level_window(), _("Coldcard Status")) @@ -93,16 +88,7 @@ def message_dialog(self, msg): vbox = QVBoxLayout(dialog) vbox.addWidget(l) dialog.show() - - def get_setup(self): - self.done.clear() - self.setup_signal.emit() - self.done.wait() - return - - def setup_dialog(self): - self.show_error(_('Please initialize your Coldcard while disconnected.')) - return + class CKCCSettingsDialog(WindowModalDialog): From ffe3cef21acd67a4e98e51b49fd13ba1123c7652 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Mon, 6 Apr 2020 16:49:11 +0200 Subject: [PATCH 049/117] bitbox02: don't run show_xpub on GUI thread --- electrum/plugins/bitbox02/bitbox02.py | 9 ++++++++- electrum/plugins/bitbox02/qt.py | 7 +++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/electrum/plugins/bitbox02/bitbox02.py b/electrum/plugins/bitbox02/bitbox02.py index 087546dc45..5c3f85d45b 100644 --- a/electrum/plugins/bitbox02/bitbox02.py +++ b/electrum/plugins/bitbox02/bitbox02.py @@ -196,7 +196,7 @@ def get_password_for_storage_encryption(self) -> str: node = bip32.BIP32Node.from_xkey(xpub, net = constants.BitcoinMainnet()).subkey_at_public_derivation(()) return node.eckey.get_public_key_bytes(compressed=True).hex() - def get_xpub(self, bip32_path: str, xtype: str, display: bool = False) -> str: + def get_xpub(self, bip32_path: str, xtype: str, *, display: bool = False) -> str: if self.bitbox02_device is None: self.pairing_dialog(wizard=False) @@ -612,3 +612,10 @@ def show_address( txin_type = wallet.get_txin_type(address) sequence = wallet.get_address_index(address) keystore.show_address(sequence, txin_type, wallet) + + def show_xpub(self, keystore: BitBox02_KeyStore): + client = keystore.get_client() + assert isinstance(client, BitBox02Client) + derivation = keystore.get_derivation_prefix() + xtype = keystore.get_bip32_node_for_xpub().xtype + client.get_xpub(derivation, xtype, display=True) diff --git a/electrum/plugins/bitbox02/qt.py b/electrum/plugins/bitbox02/qt.py index 2a63b681f1..9b1ccc261d 100644 --- a/electrum/plugins/bitbox02/qt.py +++ b/electrum/plugins/bitbox02/qt.py @@ -28,7 +28,6 @@ from electrum.plugin import hook from electrum.wallet import Multisig_Wallet from electrum.transaction import PartialTransaction -from electrum import keystore from .bitbox02 import BitBox02Plugin from ..hw_wallet.qt import QtHandlerBase, QtPluginBase @@ -77,11 +76,11 @@ def on_button_click(): selected_keystore_index = labels_clayout.selected_index() keystores = wallet.get_keystores() selected_keystore = keystores[selected_keystore_index] - derivation = selected_keystore.get_derivation_prefix() if type(selected_keystore) != self.keystore_class: main_window.show_error("Select a BitBox02 xpub") - selected_keystore.get_client().get_xpub( - derivation, keystore.xtype_from_derivation(derivation), True + return + selected_keystore.thread.add( + partial(self.show_xpub, keystore=selected_keystore) ) btn.clicked.connect(lambda unused: on_button_click()) From cc4aa1812dfcbe5ff8a569e0672ac60ceb764a9b Mon Sep 17 00:00:00 2001 From: SomberNight Date: Mon, 6 Apr 2020 16:51:38 +0200 Subject: [PATCH 050/117] rm some unused imports --- electrum/plugins/bitbox02/bitbox02.py | 5 +---- electrum/plugins/bitbox02/qt.py | 15 ++------------- 2 files changed, 3 insertions(+), 17 deletions(-) diff --git a/electrum/plugins/bitbox02/bitbox02.py b/electrum/plugins/bitbox02/bitbox02.py index 5c3f85d45b..ca99f816c6 100644 --- a/electrum/plugins/bitbox02/bitbox02.py +++ b/electrum/plugins/bitbox02/bitbox02.py @@ -3,18 +3,16 @@ # import hid -import hashlib from typing import TYPE_CHECKING, Dict, Tuple, Optional, List, Any, Callable from electrum import bip32, constants from electrum.i18n import _ -from electrum.keystore import Hardware_KeyStore, Xpub +from electrum.keystore import Hardware_KeyStore from electrum.transaction import PartialTransaction from electrum.wallet import Standard_Wallet, Multisig_Wallet, Deterministic_Wallet from electrum.util import bh2u, UserFacingException from electrum.base_wizard import ScriptTypeNotSupported, BaseWizard from electrum.logging import get_logger -from electrum.crypto import hmac_oneshot from electrum.plugin import Device, DeviceInfo from electrum.simple_config import SimpleConfig from electrum.json_db import StoredDict @@ -24,7 +22,6 @@ import electrum.ecc as ecc from ..hw_wallet import HW_PluginBase, HardwareClientBase -from ..hw_wallet.plugin import LibraryFoundButUnusable try: diff --git a/electrum/plugins/bitbox02/qt.py b/electrum/plugins/bitbox02/qt.py index 9b1ccc261d..96165f203c 100644 --- a/electrum/plugins/bitbox02/qt.py +++ b/electrum/plugins/bitbox02/qt.py @@ -1,37 +1,26 @@ -import time, os from functools import partial -import copy -from PyQt5.QtCore import Qt, pyqtSignal from PyQt5.QtWidgets import ( QPushButton, QLabel, QVBoxLayout, - QWidget, - QGridLayout, QLineEdit, QHBoxLayout, ) -from PyQt5.QtCore import Qt, QMetaObject, Q_RETURN_ARG, pyqtSignal, pyqtSlot +from PyQt5.QtCore import Qt, QMetaObject, Q_RETURN_ARG, pyqtSlot from electrum.gui.qt.util import ( WindowModalDialog, - Buttons, OkButton, - CancelButton, - get_parent_main_window, ) -from electrum.gui.qt.transaction_dialog import TxDialog from electrum.i18n import _ from electrum.plugin import hook -from electrum.wallet import Multisig_Wallet -from electrum.transaction import PartialTransaction from .bitbox02 import BitBox02Plugin from ..hw_wallet.qt import QtHandlerBase, QtPluginBase -from ..hw_wallet.plugin import only_hook_if_libraries_available, LibraryFoundButUnusable +from ..hw_wallet.plugin import only_hook_if_libraries_available class Plugin(BitBox02Plugin, QtPluginBase): From 66c264f6130c28f600a26c5d0dc28f0ee79f5298 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Mon, 6 Apr 2020 17:23:22 +0200 Subject: [PATCH 051/117] bitcoin.py: change API of address_to_hash --- electrum/bitcoin.py | 34 +++++++++++++++++++++------ electrum/plugins/bitbox02/bitbox02.py | 9 +++---- 2 files changed, 32 insertions(+), 11 deletions(-) diff --git a/electrum/bitcoin.py b/electrum/bitcoin.py index c1c2d73be3..794eb54cdd 100644 --- a/electrum/bitcoin.py +++ b/electrum/bitcoin.py @@ -25,7 +25,8 @@ import hashlib from typing import List, Tuple, TYPE_CHECKING, Optional, Union -from enum import IntEnum +import enum +from enum import IntEnum, Enum from .util import bfh, bh2u, BitcoinException, assert_bytes, to_bytes, inv_dict from . import version @@ -432,20 +433,39 @@ def address_to_script(addr: str, *, net=None) -> str: raise BitcoinException(f'unknown address type: {addrtype}') return script -def address_to_hash(addr: str, *, net=None) -> Tuple[int, bytes]: - """Return the pubkey hash / witness program of an address""" + +class OnchainOutputType(Enum): + """Opaque types of scriptPubKeys. + In case of p2sh, p2wsh and similar, no knowledge of redeem script, etc. + """ + P2PKH = enum.auto() + P2SH = enum.auto() + WITVER0_P2WPKH = enum.auto() + WITVER0_P2WSH = enum.auto() + + +def address_to_hash(addr: str, *, net=None) -> Tuple[OnchainOutputType, bytes]: + """Return (type, pubkey hash / witness program) for an address.""" if net is None: net = constants.net if not is_address(addr, net=net): raise BitcoinException(f"invalid bitcoin address: {addr}") witver, witprog = segwit_addr.decode(net.SEGWIT_HRP, addr) if witprog is not None: + if witver != 0: + raise BitcoinException(f"not implemented handling for witver={witver}") if len(witprog) == 20: - return WIF_SCRIPT_TYPES['p2wpkh'], bytes(witprog) - return WIF_SCRIPT_TYPES['p2wsh'], bytes(witprog) + return OnchainOutputType.WITVER0_P2WPKH, bytes(witprog) + elif len(witprog) == 32: + return OnchainOutputType.WITVER0_P2WSH, bytes(witprog) + else: + raise BitcoinException(f"unexpected length for segwit witver=0 witprog: len={len(witprog)}") addrtype, hash_160_ = b58_address_to_hash160(addr) if addrtype == net.ADDRTYPE_P2PKH: - return WIF_SCRIPT_TYPES['p2pkh'], hash_160_ - return WIF_SCRIPT_TYPES['p2sh'], hash_160_ + return OnchainOutputType.P2PKH, hash_160_ + elif addrtype == net.ADDRTYPE_P2SH: + return OnchainOutputType.P2SH, hash_160_ + raise BitcoinException(f"unknown address type: {addrtype}") + def address_to_scripthash(addr: str) -> str: script = address_to_script(addr) diff --git a/electrum/plugins/bitbox02/bitbox02.py b/electrum/plugins/bitbox02/bitbox02.py index ca99f816c6..9d6143583a 100644 --- a/electrum/plugins/bitbox02/bitbox02.py +++ b/electrum/plugins/bitbox02/bitbox02.py @@ -17,6 +17,7 @@ from electrum.simple_config import SimpleConfig from electrum.json_db import StoredDict from electrum.storage import get_derivation_used_for_hw_device_encryption +from electrum.bitcoin import OnchainOutputType import electrum.bitcoin as bitcoin import electrum.ecc as ecc @@ -411,13 +412,13 @@ def sign_transaction( ) else: addrtype, pubkey_hash = bitcoin.address_to_hash(txout.address) - if addrtype == bitcoin.WIF_SCRIPT_TYPES["p2pkh"]: + if addrtype == OnchainOutputType.P2PKH: output_type = bitbox02.btc.P2PKH - elif addrtype == bitcoin.WIF_SCRIPT_TYPES["p2sh"]: + elif addrtype == OnchainOutputType.P2SH: output_type = bitbox02.btc.P2SH - elif addrtype == bitcoin.WIF_SCRIPT_TYPES["p2wpkh"]: + elif addrtype == OnchainOutputType.WITVER0_P2WPKH: output_type = bitbox02.btc.P2WPKH - elif addrtype == bitcoin.WIF_SCRIPT_TYPES["p2wsh"]: + elif addrtype == OnchainOutputType.WITVER0_P2WSH: output_type = bitbox02.btc.P2WSH else: raise UserFacingException( From e830ef309f4b13d0e02587ad2d043eecd88aca74 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Mon, 6 Apr 2020 18:20:21 +0200 Subject: [PATCH 052/117] hww: factor out part of hid scan code to HW_PluginBase so that bitbox02 can override it --- electrum/plugin.py | 31 +++++-------------- electrum/plugins/bitbox02/bitbox02.py | 9 +++++- electrum/plugins/coldcard/coldcard.py | 2 +- .../plugins/digitalbitbox/digitalbitbox.py | 2 +- electrum/plugins/hw_wallet/plugin.py | 16 ++++++++++ electrum/plugins/keepkey/keepkey.py | 2 +- electrum/plugins/ledger/ledger.py | 2 +- 7 files changed, 36 insertions(+), 28 deletions(-) diff --git a/electrum/plugin.py b/electrum/plugin.py index 084063cd9b..d86e767069 100644 --- a/electrum/plugin.py +++ b/electrum/plugin.py @@ -360,9 +360,8 @@ def __init__(self, config: SimpleConfig): # A list of clients. The key is the client, the value is # a (path, id_) pair. Needs self.lock. self.clients = {} # type: Dict[HardwareClientBase, Tuple[Union[str, bytes], str]] - # What we recognise. Each entry is a (vendor_id, product_id) - # pair. - self.recognised_hardware = set() + # What we recognise. (vendor_id, product_id) -> Plugin + self._recognised_hardware = {} # type: Dict[Tuple[int, int], HW_PluginBase] # Custom enumerate functions for devices we don't know about. self._enumerate_func = set() # Needs self.lock. # locks: if you need to take multiple ones, acquire them in the order they are defined here! @@ -390,9 +389,9 @@ def run(self): for client in clients: client.timeout(cutoff) - def register_devices(self, device_pairs): + def register_devices(self, device_pairs, *, plugin: 'HW_PluginBase'): for pair in device_pairs: - self.recognised_hardware.add(pair) + self._recognised_hardware[pair] = plugin def register_enumerate_func(self, func): with self.lock: @@ -642,24 +641,10 @@ def _scan_devices_with_hid(self) -> List['Device']: devices = [] for d in hid_list: product_key = (d['vendor_id'], d['product_id']) - if product_key in self.recognised_hardware: - # Older versions of hid don't provide interface_number - interface_number = d.get('interface_number', -1) - usage_page = d['usage_page'] - id_ = d['serial_number'] - if len(id_) == 0: - id_ = str(d['path']) - id_ += str(interface_number) + str(usage_page) - # The BitBox02's product_id is not unique per device, thus use the path instead to - # distinguish devices. - if d["product_id"] == 0x2403: - id_ = d['path'] - devices.append(Device(path=d['path'], - interface_number=interface_number, - id_=id_, - product_key=product_key, - usage_page=usage_page, - transport_ui_string='hid')) + if product_key in self._recognised_hardware: + plugin = self._recognised_hardware[product_key] + device = plugin.create_device_from_hid_enumeration(d, product_key=product_key) + devices.append(device) return devices @with_scan_lock diff --git a/electrum/plugins/bitbox02/bitbox02.py b/electrum/plugins/bitbox02/bitbox02.py index 9d6143583a..6a62bee884 100644 --- a/electrum/plugins/bitbox02/bitbox02.py +++ b/electrum/plugins/bitbox02/bitbox02.py @@ -546,7 +546,7 @@ def __init__(self, parent: HW_PluginBase, config: SimpleConfig, name: str): self.libraries_available = self.check_libraries_available() if not self.libraries_available: return - self.device_manager().register_devices(self.DEVICE_IDS) + self.device_manager().register_devices(self.DEVICE_IDS, plugin=self) def get_library_version(self): try: @@ -617,3 +617,10 @@ def show_xpub(self, keystore: BitBox02_KeyStore): derivation = keystore.get_derivation_prefix() xtype = keystore.get_bip32_node_for_xpub().xtype client.get_xpub(derivation, xtype, display=True) + + def create_device_from_hid_enumeration(self, d: dict, *, product_key) -> 'Device': + device = super().create_device_from_hid_enumeration(d, product_key=product_key) + # The BitBox02's product_id is not unique per device, thus use the path instead to + # distinguish devices. + id_ = str(d['path']) + return device._replace(id_=id_) diff --git a/electrum/plugins/coldcard/coldcard.py b/electrum/plugins/coldcard/coldcard.py index 810fd91b37..b8cae820c3 100644 --- a/electrum/plugins/coldcard/coldcard.py +++ b/electrum/plugins/coldcard/coldcard.py @@ -477,7 +477,7 @@ def __init__(self, parent, config, name): if not self.libraries_available: return - self.device_manager().register_devices(self.DEVICE_IDS) + self.device_manager().register_devices(self.DEVICE_IDS, plugin=self) self.device_manager().register_enumerate_func(self.detect_simulator) def get_library_version(self): diff --git a/electrum/plugins/digitalbitbox/digitalbitbox.py b/electrum/plugins/digitalbitbox/digitalbitbox.py index b141955785..15b91de7ce 100644 --- a/electrum/plugins/digitalbitbox/digitalbitbox.py +++ b/electrum/plugins/digitalbitbox/digitalbitbox.py @@ -675,7 +675,7 @@ class DigitalBitboxPlugin(HW_PluginBase): def __init__(self, parent, config, name): HW_PluginBase.__init__(self, parent, config, name) if self.libraries_available: - self.device_manager().register_devices(self.DEVICE_IDS) + self.device_manager().register_devices(self.DEVICE_IDS, plugin=self) self.digitalbitbox_config = self.config.get('digitalbitbox', {}) diff --git a/electrum/plugins/hw_wallet/plugin.py b/electrum/plugins/hw_wallet/plugin.py index 884732dadb..c7c0c1eefc 100644 --- a/electrum/plugins/hw_wallet/plugin.py +++ b/electrum/plugins/hw_wallet/plugin.py @@ -60,6 +60,22 @@ def is_enabled(self): def device_manager(self) -> 'DeviceMgr': return self.parent.device_manager + def create_device_from_hid_enumeration(self, d: dict, *, product_key) -> 'Device': + # Older versions of hid don't provide interface_number + interface_number = d.get('interface_number', -1) + usage_page = d['usage_page'] + id_ = d['serial_number'] + if len(id_) == 0: + id_ = str(d['path']) + id_ += str(interface_number) + str(usage_page) + device = Device(path=d['path'], + interface_number=interface_number, + id_=id_, + product_key=product_key, + usage_page=usage_page, + transport_ui_string='hid') + return device + @hook def close_wallet(self, wallet: 'Abstract_Wallet'): for keystore in wallet.get_keystores(): diff --git a/electrum/plugins/keepkey/keepkey.py b/electrum/plugins/keepkey/keepkey.py index 1722eaea64..4b0f19a3fc 100644 --- a/electrum/plugins/keepkey/keepkey.py +++ b/electrum/plugins/keepkey/keepkey.py @@ -88,7 +88,7 @@ def __init__(self, parent, config, name): self.DEVICE_IDS = (keepkeylib.transport_hid.DEVICE_IDS + keepkeylib.transport_webusb.DEVICE_IDS) # only "register" hid device id: - self.device_manager().register_devices(keepkeylib.transport_hid.DEVICE_IDS) + self.device_manager().register_devices(keepkeylib.transport_hid.DEVICE_IDS, plugin=self) # for webusb transport, use custom enumerate function: self.device_manager().register_enumerate_func(self.enumerate) self.libraries_available = True diff --git a/electrum/plugins/ledger/ledger.py b/electrum/plugins/ledger/ledger.py index d5abc5dbef..08cea77a38 100644 --- a/electrum/plugins/ledger/ledger.py +++ b/electrum/plugins/ledger/ledger.py @@ -578,7 +578,7 @@ def __init__(self, parent, config, name): self.segwit = config.get("segwit") HW_PluginBase.__init__(self, parent, config, name) if self.libraries_available: - self.device_manager().register_devices(self.DEVICE_IDS) + self.device_manager().register_devices(self.DEVICE_IDS, plugin=self) def get_btchip_device(self, device): ledger = False From dda20583c24e086012885bad079b0e15354c1e27 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Wed, 8 Apr 2020 18:38:22 +0200 Subject: [PATCH 053/117] bitbox02: rm BitBox02Client.label override if placeholder anyway, just use base impl (alternatively we should list it in electrum.plugin.PLACEHOLDER_HW_CLIENT_LABELS) --- electrum/plugins/bitbox02/bitbox02.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/electrum/plugins/bitbox02/bitbox02.py b/electrum/plugins/bitbox02/bitbox02.py index 6a62bee884..948fc1e167 100644 --- a/electrum/plugins/bitbox02/bitbox02.py +++ b/electrum/plugins/bitbox02/bitbox02.py @@ -68,9 +68,6 @@ def __init__(self, handler: Any, device: Device, config: SimpleConfig): if self.bitbox_hid_info is None: raise Exception("No BitBox02 detected") - def label(self) -> str: - return "BitBox02" - def is_initialized(self) -> bool: return True From 10c358dd38520cc9b17727b94ecdbb72d7c20f88 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Wed, 8 Apr 2020 22:18:42 +0200 Subject: [PATCH 054/117] bitbox02: rm plugin.get_client method: just use default impl --- electrum/plugins/bitbox02/bitbox02.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/electrum/plugins/bitbox02/bitbox02.py b/electrum/plugins/bitbox02/bitbox02.py index 948fc1e167..3b28923277 100644 --- a/electrum/plugins/bitbox02/bitbox02.py +++ b/electrum/plugins/bitbox02/bitbox02.py @@ -587,12 +587,6 @@ def get_xpub( assert client.bitbox02_device is not None return client.get_xpub(derivation, xtype) - def get_client(self, keystore: BitBox02_KeyStore, force_pair: bool = True): - devmgr = self.device_manager() - handler = keystore.handler - client = devmgr.client_for_keystore(self, handler, keystore, force_pair) - return client - def show_address( self, wallet: Deterministic_Wallet, From 04dcfe6fd120912a914afff5012cde8b1f969733 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Thu, 9 Apr 2020 20:36:07 +0200 Subject: [PATCH 055/117] bitbox02: add to requirements-hw, and include in win/mac binaries --- contrib/build-wine/deterministic.spec | 2 + .../deterministic-build/requirements-hw.txt | 63 +++++++++++++++++++ contrib/osx/osx.spec | 2 + contrib/requirements/requirements-hw.txt | 1 + 4 files changed, 68 insertions(+) diff --git a/contrib/build-wine/deterministic.spec b/contrib/build-wine/deterministic.spec index 2ad11294b0..4509c956ea 100644 --- a/contrib/build-wine/deterministic.spec +++ b/contrib/build-wine/deterministic.spec @@ -23,6 +23,7 @@ hiddenimports += collect_submodules('btchip') hiddenimports += collect_submodules('keepkeylib') hiddenimports += collect_submodules('websocket') hiddenimports += collect_submodules('ckcc') +hiddenimports += collect_submodules('bitbox02') hiddenimports += ['PyQt5.QtPrintSupport'] # needed by Revealer @@ -48,6 +49,7 @@ datas += collect_data_files('safetlib') datas += collect_data_files('btchip') datas += collect_data_files('keepkeylib') datas += collect_data_files('ckcc') +datas += collect_data_files('bitbox02') datas += collect_data_files('jsonrpcserver') datas += collect_data_files('jsonrpcclient') diff --git a/contrib/deterministic-build/requirements-hw.txt b/contrib/deterministic-build/requirements-hw.txt index 17b38ce36b..5b4f853d0d 100644 --- a/contrib/deterministic-build/requirements-hw.txt +++ b/contrib/deterministic-build/requirements-hw.txt @@ -1,8 +1,43 @@ +base58==2.0.0 \ + --hash=sha256:4c7f5687da771b519cf86b3236250e7c3543368c576404c9fe2d992a287666e0 \ + --hash=sha256:c83584a8b917dc52dd634307137f2ad2721a9efb4f1de32fc7eaaaf87844177e +bitbox02==2.0.3 \ + --hash=sha256:1f0164fd9941d3c3a17fb7db3bceddd89458986ef3da6171845e6433c3f66889 \ + --hash=sha256:53d06baafc597a8d14f990e285cd608cdf00be41a6d42ae40c316abad7798bd5 btchip-python==0.1.28 \ --hash=sha256:da09d0d7a6180d428833795ea9a233c3b317ddfcccea8cc6f0eba59435e5dd83 certifi==2020.4.5.1 \ --hash=sha256:1d987a998c75633c40847cc966fcf5904906c920a7f17ef374f5aa4282abd304 \ --hash=sha256:51fcb31174be6e6664c5f69e3e1691a2d72a1a12e90f872cbdb1567eb47b6519 +cffi==1.14.0 \ + --hash=sha256:001bf3242a1bb04d985d63e138230802c6c8d4db3668fb545fb5005ddf5bb5ff \ + --hash=sha256:00789914be39dffba161cfc5be31b55775de5ba2235fe49aa28c148236c4e06b \ + --hash=sha256:028a579fc9aed3af38f4892bdcc7390508adabc30c6af4a6e4f611b0c680e6ac \ + --hash=sha256:14491a910663bf9f13ddf2bc8f60562d6bc5315c1f09c704937ef17293fb85b0 \ + --hash=sha256:1cae98a7054b5c9391eb3249b86e0e99ab1e02bb0cc0575da191aedadbdf4384 \ + --hash=sha256:2089ed025da3919d2e75a4d963d008330c96751127dd6f73c8dc0c65041b4c26 \ + --hash=sha256:2d384f4a127a15ba701207f7639d94106693b6cd64173d6c8988e2c25f3ac2b6 \ + --hash=sha256:337d448e5a725bba2d8293c48d9353fc68d0e9e4088d62a9571def317797522b \ + --hash=sha256:399aed636c7d3749bbed55bc907c3288cb43c65c4389964ad5ff849b6370603e \ + --hash=sha256:3b911c2dbd4f423b4c4fcca138cadde747abdb20d196c4a48708b8a2d32b16dd \ + --hash=sha256:3d311bcc4a41408cf5854f06ef2c5cab88f9fded37a3b95936c9879c1640d4c2 \ + --hash=sha256:62ae9af2d069ea2698bf536dcfe1e4eed9090211dbaafeeedf5cb6c41b352f66 \ + --hash=sha256:66e41db66b47d0d8672d8ed2708ba91b2f2524ece3dee48b5dfb36be8c2f21dc \ + --hash=sha256:675686925a9fb403edba0114db74e741d8181683dcf216be697d208857e04ca8 \ + --hash=sha256:7e63cbcf2429a8dbfe48dcc2322d5f2220b77b2e17b7ba023d6166d84655da55 \ + --hash=sha256:8a6c688fefb4e1cd56feb6c511984a6c4f7ec7d2a1ff31a10254f3c817054ae4 \ + --hash=sha256:8c0ffc886aea5df6a1762d0019e9cb05f825d0eec1f520c51be9d198701daee5 \ + --hash=sha256:95cd16d3dee553f882540c1ffe331d085c9e629499ceadfbda4d4fde635f4b7d \ + --hash=sha256:99f748a7e71ff382613b4e1acc0ac83bf7ad167fb3802e35e90d9763daba4d78 \ + --hash=sha256:b8c78301cefcf5fd914aad35d3c04c2b21ce8629b5e4f4e45ae6812e461910fa \ + --hash=sha256:c420917b188a5582a56d8b93bdd8e0f6eca08c84ff623a4c16e809152cd35793 \ + --hash=sha256:c43866529f2f06fe0edc6246eb4faa34f03fe88b64a0a9a942561c8e22f4b71f \ + --hash=sha256:cab50b8c2250b46fe738c77dbd25ce017d5e6fb35d3407606e7a4180656a5a6a \ + --hash=sha256:cef128cb4d5e0b3493f058f10ce32365972c554572ff821e175dbc6f8ff6924f \ + --hash=sha256:cf16e3cf6c0a5fdd9bc10c21687e19d29ad1fe863372b5543deaec1039581a30 \ + --hash=sha256:e56c744aa6ff427a607763346e4170629caf7e48ead6921745986db3692f987f \ + --hash=sha256:e577934fc5f8779c554639376beeaa5657d54349096ef24abe8c74c5d9c117c3 \ + --hash=sha256:f2b0fa0c01d8a0c7483afd9f31d7ecf2d71760ca24499c8697aeb5ca37dc090c chardet==3.0.4 \ --hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae \ --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 @@ -14,6 +49,26 @@ click==7.1.1 \ --hash=sha256:e345d143d80bf5ee7534056164e5e112ea5e22716bbb1ce727941f4c8b471b9a construct==2.10.56 \ --hash=sha256:97ba13edcd98546f10f7555af41c8ce7ae9d8221525ec4062c03f9adbf940661 +cryptography==2.9 \ + --hash=sha256:0cacd3ef5c604b8e5f59bf2582c076c98a37fe206b31430d0cd08138aff0986e \ + --hash=sha256:192ca04a36852a994ef21df13cca4d822adbbdc9d5009c0f96f1d2929e375d4f \ + --hash=sha256:19ae795137682a9778892fb4390c07811828b173741bce91e30f899424b3934d \ + --hash=sha256:1b9b535d6b55936a79dbe4990b64bb16048f48747c76c29713fea8c50eca2acf \ + --hash=sha256:2a2ad24d43398d89f92209289f15265107928f22a8d10385f70def7a698d6a02 \ + --hash=sha256:3be7a5722d5bfe69894d3f7bbed15547b17619f3a88a318aab2e37f457524164 \ + --hash=sha256:49870684da168b90110bbaf86140d4681032c5e6a2461adc7afdd93be5634216 \ + --hash=sha256:587f98ce27ac4547177a0c6fe0986b8736058daffe9160dcf5f1bd411b7fbaa1 \ + --hash=sha256:5aca6f00b2f42546b9bdf11a69f248d1881212ce5b9e2618b04935b87f6f82a1 \ + --hash=sha256:6b744039b55988519cc183149cceb573189b3e46e16ccf6f8c46798bb767c9dc \ + --hash=sha256:6b91cab3841b4c7cb70e4db1697c69f036c8bc0a253edc0baa6783154f1301e4 \ + --hash=sha256:7598974f6879a338c785c513e7c5a4329fbc58b9f6b9a6305035fca5b1076552 \ + --hash=sha256:7a279f33a081d436e90e91d1a7c338553c04e464de1c9302311a5e7e4b746088 \ + --hash=sha256:95e1296e0157361fe2f5f0ed307fd31f94b0ca13372e3673fa95095a627636a1 \ + --hash=sha256:9fc9da390e98cb6975eadf251b6e5fa088820141061bf041cd5c72deba1dc526 \ + --hash=sha256:cc20316e3f5a6b582fc3b029d8dc03aabeb645acfcb7fc1d9848841a33265748 \ + --hash=sha256:d1bf5a1a0d60c7f9a78e448adcb99aa101f3f9588b16708044638881be15d6bc \ + --hash=sha256:ed1d0760c7e46436ec90834d6f10477ff09475c692ed1695329d324b2c5cd547 \ + --hash=sha256:ef9a55013676907df6c9d7dd943eb1770d014f68beaa7e73250fb43c759f4585 Cython==0.29.16 \ --hash=sha256:0542a6c4ff1be839b6479deffdbdff1a330697d7953dd63b6de99c078e3acd5f \ --hash=sha256:0bcf7f87aa0ba8b62d4f3b6e0146e48779eaa4f39f92092d7ff90081ef6133e0 \ @@ -72,6 +127,8 @@ libusb1==1.7.1 \ mnemonic==0.19 \ --hash=sha256:4e37eb02b2cbd56a0079cabe58a6da93e60e3e4d6e757a586d9f23d96abea931 \ --hash=sha256:a8d78c5100acfa7df9bab6b9db7390831b0e54490934b718ff9efd68f0d731a6 +noiseprotocol==0.3.1 \ + --hash=sha256:2e1a603a38439636cf0ffd8b3e8b12cee27d368a28b41be7dbe568b2abb23111 pip==20.0.2 \ --hash=sha256:4ae14a42d8adba3205ebeb38aa68cfc0b6c346e1ae2e699a0b3bad4da19cef5c \ --hash=sha256:7db0c8ea4c7ea51c8049640e8e6e7fde949de672bfa4949920675563a5a6967f @@ -97,12 +154,18 @@ protobuf==3.11.3 \ --hash=sha256:fdfb6ad138dbbf92b5dbea3576d7c8ba7463173f7d2cb0ca1bd336ec88ddbd80 pyaes==1.6.1 \ --hash=sha256:02c1b1405c38d3c370b085fb952dd8bea3fadcee6411ad99f312cc129c536d8f +pycparser==2.20 \ + --hash=sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0 \ + --hash=sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705 requests==2.23.0 \ --hash=sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee \ --hash=sha256:b3f43d496c6daba4493e7c431722aeb7dbc6288f52a6e04e7b6023b0247817e6 safet==0.1.5 \ --hash=sha256:a7fd4b68bb1bc6185298af665c8e8e00e2bb2bcbddbb22844ead929b845c635e \ --hash=sha256:f966a23243312f64d14c7dfe02e8f13f6eeba4c3f51341f2c11ae57831f07de3 +semver==2.9.1 \ + --hash=sha256:095c3cba6d5433f21451101463b22cf831fe6996fcc8a603407fd8bea54f116b \ + --hash=sha256:723be40c74b6468861e0e3dbb80a41fc3b171a2a45bf956c245304773dc06055 setuptools==46.1.3 \ --hash=sha256:4fe404eec2738c20ab5841fa2d791902d2a645f32318a7850ef26f8d7215a8ee \ --hash=sha256:795e0475ba6cd7fa082b1ee6e90d552209995627a2a227a47c6ea93282f4bfb1 diff --git a/contrib/osx/osx.spec b/contrib/osx/osx.spec index f9ba4fec45..4a6a3a9472 100644 --- a/contrib/osx/osx.spec +++ b/contrib/osx/osx.spec @@ -66,6 +66,7 @@ hiddenimports += collect_submodules('btchip') hiddenimports += collect_submodules('keepkeylib') hiddenimports += collect_submodules('websocket') hiddenimports += collect_submodules('ckcc') +hiddenimports += collect_submodules('bitbox02') hiddenimports += ['PyQt5.QtPrintSupport'] # needed by Revealer datas = [ @@ -81,6 +82,7 @@ datas += collect_data_files('safetlib') datas += collect_data_files('btchip') datas += collect_data_files('keepkeylib') datas += collect_data_files('ckcc') +datas += collect_data_files('bitbox02') datas += collect_data_files('jsonrpcserver') datas += collect_data_files('jsonrpcclient') diff --git a/contrib/requirements/requirements-hw.txt b/contrib/requirements/requirements-hw.txt index a442d71475..bf9c11f6e7 100644 --- a/contrib/requirements/requirements-hw.txt +++ b/contrib/requirements/requirements-hw.txt @@ -13,4 +13,5 @@ safet>=0.1.5 keepkey>=6.3.1 btchip-python>=0.1.26 ckcc-protocol>=0.7.7 +bitbox02>=2.0.2 hidapi From fe86f911100aeec1f3d5afcfdb5da1babbe4d98f Mon Sep 17 00:00:00 2001 From: SomberNight Date: Mon, 13 Apr 2020 05:00:26 +0200 Subject: [PATCH 056/117] adapt to new aiohttp_socks: fix deprecation warnings ...\electrum\electrum\util.py:1096: DeprecationWarning: SocksConnector is deprecated. Use ProxyConnector instead. connector = SocksConnector( ...\Python38\site-packages\aiohttp_socks\proxy\socks5_proxy.py:37: DeprecationWarning: Parameter family is deprecated and will be ignored. super().__init__( --- electrum/util.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/electrum/util.py b/electrum/util.py index 5a302d3c3f..cd08c1e992 100644 --- a/electrum/util.py +++ b/electrum/util.py @@ -43,7 +43,7 @@ import ipaddress import aiohttp -from aiohttp_socks import SocksConnector, SocksVer +from aiohttp_socks import ProxyConnector, ProxyType from aiorpcx import TaskGroup import certifi import dns.resolver @@ -1093,8 +1093,8 @@ def make_aiohttp_session(proxy: Optional[dict], headers=None, timeout=None): ssl_context = ssl.create_default_context(purpose=ssl.Purpose.SERVER_AUTH, cafile=ca_path) if proxy: - connector = SocksConnector( - socks_ver=SocksVer.SOCKS5 if proxy['mode'] == 'socks5' else SocksVer.SOCKS4, + connector = ProxyConnector( + proxy_type=ProxyType.SOCKS5 if proxy['mode'] == 'socks5' else ProxyType.SOCKS4, host=proxy['host'], port=int(proxy['port']), username=proxy.get('user', None), From bddb0bfcdd1c89528d1e3938414ea0cde9e7bea7 Mon Sep 17 00:00:00 2001 From: ThomasV Date: Mon, 13 Apr 2020 11:28:42 +0200 Subject: [PATCH 057/117] Do not wait wallet sync to reestablish channel (revert e32807d29d). --- electrum/lnworker.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/electrum/lnworker.py b/electrum/lnworker.py index f1bf2a9c04..8fabed52cd 100644 --- a/electrum/lnworker.py +++ b/electrum/lnworker.py @@ -1288,9 +1288,6 @@ async def reestablish_peer_for_given_channel(self, chan: Channel) -> None: async def reestablish_peers_and_channels(self): while True: await asyncio.sleep(1) - # wait until on-chain state is synchronized - if not (self.wallet.is_up_to_date() and self.lnwatcher.is_up_to_date()): - continue with self.lock: channels = list(self.channels.values()) for chan in channels: From 821431a23913349ae1c698b7653f42fc91ccb241 Mon Sep 17 00:00:00 2001 From: ThomasV Date: Mon, 13 Apr 2020 11:34:58 +0200 Subject: [PATCH 058/117] lnpeer: move ping_if_required away from message_loop If our connection dies because we went to sleep, message_loop will stall and ping_if_required will never be called. --- electrum/lnpeer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/electrum/lnpeer.py b/electrum/lnpeer.py index 9eb5aab3dd..b3864eab76 100644 --- a/electrum/lnpeer.py +++ b/electrum/lnpeer.py @@ -463,7 +463,6 @@ async def _message_loop(self): async for msg in self.transport.read_messages(): self.process_message(msg) await asyncio.sleep(.01) - self.ping_if_required() def on_reply_short_channel_ids_end(self, payload): self.querying.set() @@ -1452,8 +1451,10 @@ def verify_signature(tx, sig): return closing_tx.txid() async def htlc_switch(self): + await self.initialized while True: await asyncio.sleep(0.1) + self.ping_if_required() for chan_id, chan in self.channels.items(): if not chan.can_send_ctx_updates(): continue From 8e8ab775ebf019eabd1640ab4a7d9c820b08a384 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Mon, 13 Apr 2020 15:57:53 +0200 Subject: [PATCH 059/117] lnchannel: make AbstractChannel inherit ABC and add some type annotations, clean up method signatures --- electrum/lnchannel.py | 148 ++++++++++++++++++++++++++++------------ electrum/lnsweep.py | 4 +- electrum/lntransport.py | 1 + electrum/lnutil.py | 8 +-- electrum/lnwatcher.py | 42 ++++++------ electrum/lnworker.py | 2 +- 6 files changed, 134 insertions(+), 71 deletions(-) diff --git a/electrum/lnchannel.py b/electrum/lnchannel.py index 3331546093..ecc2276270 100644 --- a/electrum/lnchannel.py +++ b/electrum/lnchannel.py @@ -27,13 +27,14 @@ Iterable, Sequence, TYPE_CHECKING, Iterator, Union) import time import threading +from abc import ABC, abstractmethod from aiorpcx import NetAddress import attr from . import ecc from . import constants -from .util import bfh, bh2u, chunks +from .util import bfh, bh2u, chunks, TxMinedInfo from .bitcoin import redeem_script_to_address from .crypto import sha256, sha256d from .transaction import Transaction, PartialTransaction @@ -113,7 +114,9 @@ class peer_states(IntEnum): # TODO rename to use CamelCase del cs # delete as name is ambiguous without context -RevokeAndAck = namedtuple("RevokeAndAck", ["per_commitment_secret", "next_per_commitment_point"]) +class RevokeAndAck(NamedTuple): + per_commitment_secret: bytes + next_per_commitment_point: bytes class RemoteCtnTooFarInFuture(Exception): pass @@ -123,7 +126,16 @@ def htlcsum(htlcs): return sum([x.amount_msat for x in htlcs]) -class AbstractChannel(Logger): +class AbstractChannel(Logger, ABC): + storage: Union['StoredDict', dict] + config: Dict[HTLCOwner, Union[LocalConfig, RemoteConfig]] + _sweep_info: Dict[str, Dict[str, 'SweepInfo']] + lnworker: Optional['LNWallet'] + sweep_address: str + channel_id: bytes + funding_outpoint: Outpoint + node_id: bytes + _state: channel_states def set_short_channel_id(self, short_id: ShortChannelID) -> None: self.short_channel_id = short_id @@ -168,7 +180,7 @@ def is_closed(self): def is_redeemed(self): return self.get_state() == channel_states.REDEEMED - def save_funding_height(self, txid, height, timestamp): + def save_funding_height(self, *, txid: str, height: int, timestamp: Optional[int]) -> None: self.storage['funding_height'] = txid, height, timestamp def get_funding_height(self): @@ -177,7 +189,7 @@ def get_funding_height(self): def delete_funding_height(self): self.storage.pop('funding_height', None) - def save_closing_height(self, txid, height, timestamp): + def save_closing_height(self, *, txid: str, height: int, timestamp: Optional[int]) -> None: self.storage['closing_height'] = txid, height, timestamp def get_closing_height(self): @@ -197,30 +209,34 @@ def is_backup(self): def sweep_ctx(self, ctx: Transaction) -> Dict[str, SweepInfo]: txid = ctx.txid() - if self.sweep_info.get(txid) is None: + if self._sweep_info.get(txid) is None: our_sweep_info = self.create_sweeptxs_for_our_ctx(ctx) their_sweep_info = self.create_sweeptxs_for_their_ctx(ctx) if our_sweep_info is not None: - self.sweep_info[txid] = our_sweep_info + self._sweep_info[txid] = our_sweep_info self.logger.info(f'we force closed') elif their_sweep_info is not None: - self.sweep_info[txid] = their_sweep_info + self._sweep_info[txid] = their_sweep_info self.logger.info(f'they force closed.') else: - self.sweep_info[txid] = {} + self._sweep_info[txid] = {} self.logger.info(f'not sure who closed.') - return self.sweep_info[txid] + return self._sweep_info[txid] - # ancestor for Channel and ChannelBackup - def update_onchain_state(self, funding_txid, funding_height, closing_txid, closing_height, keep_watching): + def update_onchain_state(self, *, funding_txid: str, funding_height: TxMinedInfo, + closing_txid: str, closing_height: TxMinedInfo, keep_watching: bool) -> None: # note: state transitions are irreversible, but # save_funding_height, save_closing_height are reversible if funding_height.height == TX_HEIGHT_LOCAL: self.update_unfunded_state() elif closing_height.height == TX_HEIGHT_LOCAL: - self.update_funded_state(funding_txid, funding_height) + self.update_funded_state(funding_txid=funding_txid, funding_height=funding_height) else: - self.update_closed_state(funding_txid, funding_height, closing_txid, closing_height, keep_watching) + self.update_closed_state(funding_txid=funding_txid, + funding_height=funding_height, + closing_txid=closing_txid, + closing_height=closing_height, + keep_watching=keep_watching) def update_unfunded_state(self): self.delete_funding_height() @@ -249,8 +265,8 @@ def update_unfunded_state(self): if now - self.storage.get('init_timestamp', 0) > CHANNEL_OPENING_TIMEOUT: self.lnworker.remove_channel(self.channel_id) - def update_funded_state(self, funding_txid, funding_height): - self.save_funding_height(funding_txid, funding_height.height, funding_height.timestamp) + def update_funded_state(self, *, funding_txid: str, funding_height: TxMinedInfo) -> None: + self.save_funding_height(txid=funding_txid, height=funding_height.height, timestamp=funding_height.timestamp) self.delete_closing_height() if funding_height.conf>0: self.set_short_channel_id(ShortChannelID.from_components( @@ -259,9 +275,10 @@ def update_funded_state(self, funding_txid, funding_height): if self.is_funding_tx_mined(funding_height): self.set_state(channel_states.FUNDED) - def update_closed_state(self, funding_txid, funding_height, closing_txid, closing_height, keep_watching): - self.save_funding_height(funding_txid, funding_height.height, funding_height.timestamp) - self.save_closing_height(closing_txid, closing_height.height, closing_height.timestamp) + def update_closed_state(self, *, funding_txid: str, funding_height: TxMinedInfo, + closing_txid: str, closing_height: TxMinedInfo, keep_watching: bool) -> None: + self.save_funding_height(txid=funding_txid, height=funding_height.height, timestamp=funding_height.timestamp) + self.save_closing_height(txid=closing_txid, height=closing_height.height, timestamp=closing_height.timestamp) if self.get_state() < channel_states.CLOSED: conf = closing_height.conf if conf > 0: @@ -273,6 +290,66 @@ def update_closed_state(self, funding_txid, funding_height, closing_txid, closin if self.get_state() == channel_states.CLOSED and not keep_watching: self.set_state(channel_states.REDEEMED) + @abstractmethod + def is_initiator(self) -> bool: + pass + + @abstractmethod + def is_funding_tx_mined(self, funding_height: TxMinedInfo) -> bool: + pass + + @abstractmethod + def get_funding_address(self) -> str: + pass + + @abstractmethod + def get_state_for_GUI(self) -> str: + pass + + @abstractmethod + def get_oldest_unrevoked_ctn(self, subject: HTLCOwner) -> int: + pass + + @abstractmethod + def included_htlcs(self, subject: HTLCOwner, direction: Direction, ctn: int = None) -> Sequence[UpdateAddHtlc]: + pass + + @abstractmethod + def funding_txn_minimum_depth(self) -> int: + pass + + @abstractmethod + def balance(self, whose: HTLCOwner, *, ctx_owner=HTLCOwner.LOCAL, ctn: int = None) -> int: + """This balance (in msat) only considers HTLCs that have been settled by ctn. + It disregards reserve, fees, and pending HTLCs (in both directions). + """ + pass + + @abstractmethod + def balance_minus_outgoing_htlcs(self, whose: HTLCOwner, *, + ctx_owner: HTLCOwner = HTLCOwner.LOCAL, + ctn: int = None) -> int: + """This balance (in msat), which includes the value of + pending outgoing HTLCs, is used in the UI. + """ + pass + + @abstractmethod + def is_frozen_for_sending(self) -> bool: + """Whether the user has marked this channel as frozen for sending. + Frozen channels are not supposed to be used for new outgoing payments. + (note that payment-forwarding ignores this option) + """ + pass + + @abstractmethod + def is_frozen_for_receiving(self) -> bool: + """Whether the user has marked this channel as frozen for receiving. + Frozen channels are not supposed to be used for new incoming payments. + (note that payment-forwarding ignores this option) + """ + pass + class ChannelBackup(AbstractChannel): """ @@ -288,7 +365,7 @@ def __init__(self, cb: ChannelBackupStorage, *, sweep_address=None, lnworker=Non self.name = None Logger.__init__(self) self.cb = cb - self.sweep_info = {} # type: Dict[str, Dict[str, SweepInfo]] + self._sweep_info = {} self.sweep_address = sweep_address self.storage = {} # dummy storage self._state = channel_states.OPENING @@ -351,7 +428,7 @@ def get_state_for_GUI(self): def get_oldest_unrevoked_ctn(self, who): return -1 - def included_htlcs(self, subject, direction, ctn): + def included_htlcs(self, subject, direction, ctn=None): return [] def funding_txn_minimum_depth(self): @@ -381,16 +458,16 @@ class Channel(AbstractChannel): def __init__(self, state: 'StoredDict', *, sweep_address=None, name=None, lnworker=None, initial_feerate=None): self.name = name Logger.__init__(self) - self.lnworker = lnworker # type: Optional[LNWallet] + self.lnworker = lnworker self.sweep_address = sweep_address self.storage = state self.db_lock = self.storage.db.lock if self.storage.db else threading.RLock() - self.config = {} # type: Dict[HTLCOwner, Union[LocalConfig, RemoteConfig]] + self.config = {} self.config[LOCAL] = state["local_config"] self.config[REMOTE] = state["remote_config"] self.channel_id = bfh(state["channel_id"]) self.constraints = state["constraints"] # type: ChannelConstraints - self.funding_outpoint = state["funding_outpoint"] # type: Outpoint + self.funding_outpoint = state["funding_outpoint"] self.node_id = bfh(state["node_id"]) self.short_channel_id = ShortChannelID.normalize(state["short_channel_id"]) self.onion_keys = state['onion_keys'] # type: Dict[int, bytes] @@ -398,7 +475,7 @@ def __init__(self, state: 'StoredDict', *, sweep_address=None, name=None, lnwork self.hm = HTLCManager(log=state['log'], initial_feerate=initial_feerate) self._state = channel_states[state['state']] self.peer_state = peer_states.DISCONNECTED - self.sweep_info = {} # type: Dict[str, Dict[str, SweepInfo]] + self._sweep_info = {} self._outgoing_channel_update = None # type: Optional[bytes] self._chan_ann_without_sigs = None # type: Optional[bytes] self.revocation_store = RevocationStore(state["revocation_store"]) @@ -596,10 +673,6 @@ def can_send_update_add_htlc(self) -> bool: return self.can_send_ctx_updates() and not self.is_closing() def is_frozen_for_sending(self) -> bool: - """Whether the user has marked this channel as frozen for sending. - Frozen channels are not supposed to be used for new outgoing payments. - (note that payment-forwarding ignores this option) - """ return self.storage.get('frozen_for_sending', False) def set_frozen_for_sending(self, b: bool) -> None: @@ -608,10 +681,6 @@ def set_frozen_for_sending(self, b: bool) -> None: self.lnworker.network.trigger_callback('channel', self) def is_frozen_for_receiving(self) -> bool: - """Whether the user has marked this channel as frozen for receiving. - Frozen channels are not supposed to be used for new incoming payments. - (note that payment-forwarding ignores this option) - """ return self.storage.get('frozen_for_receiving', False) def set_frozen_for_receiving(self, b: bool) -> None: @@ -880,9 +949,6 @@ def receive_revocation(self, revocation: RevokeAndAck): self.lnworker.payment_failed(self, htlc.payment_hash, payment_attempt) def balance(self, whose: HTLCOwner, *, ctx_owner=HTLCOwner.LOCAL, ctn: int = None) -> int: - """This balance (in msat) only considers HTLCs that have been settled by ctn. - It disregards reserve, fees, and pending HTLCs (in both directions). - """ assert type(whose) is HTLCOwner initial = self.config[whose].initial_msat return self.hm.get_balance_msat(whose=whose, @@ -891,10 +957,7 @@ def balance(self, whose: HTLCOwner, *, ctx_owner=HTLCOwner.LOCAL, ctn: int = Non initial_balance_msat=initial) def balance_minus_outgoing_htlcs(self, whose: HTLCOwner, *, ctx_owner: HTLCOwner = HTLCOwner.LOCAL, - ctn: int = None): - """This balance (in msat), which includes the value of - pending outgoing HTLCs, is used in the UI. - """ + ctn: int = None) -> int: assert type(whose) is HTLCOwner if ctn is None: ctn = self.get_next_ctn(ctx_owner) @@ -1282,11 +1345,6 @@ def should_be_closed_due_to_expiring_htlcs(self, local_height) -> bool: return total_value_sat > min_value_worth_closing_channel_over_sat def is_funding_tx_mined(self, funding_height): - """ - Checks if Funding TX has been mined. If it has, save the short channel ID in chan; - if it's also deep enough, also save to disk. - Returns tuple (mined_deep_enough, num_confirmations). - """ funding_txid = self.funding_outpoint.txid funding_idx = self.funding_outpoint.output_index conf = funding_height.conf diff --git a/electrum/lnsweep.py b/electrum/lnsweep.py index 6f95444084..c2b3db27c5 100644 --- a/electrum/lnsweep.py +++ b/electrum/lnsweep.py @@ -21,7 +21,7 @@ from .logging import get_logger, Logger if TYPE_CHECKING: - from .lnchannel import Channel + from .lnchannel import Channel, AbstractChannel _logger = get_logger(__name__) @@ -169,7 +169,7 @@ def create_sweeptx_for_their_revoked_htlc(chan: 'Channel', ctx: Transaction, htl -def create_sweeptxs_for_our_ctx(*, chan: 'Channel', ctx: Transaction, +def create_sweeptxs_for_our_ctx(*, chan: 'AbstractChannel', ctx: Transaction, sweep_address: str) -> Optional[Dict[str, SweepInfo]]: """Handle the case where we force close unilaterally with our latest ctx. Construct sweep txns for 'to_local', and for all HTLCs (2 txns each). diff --git a/electrum/lntransport.py b/electrum/lntransport.py index 11ba5b1553..c006f36ec2 100644 --- a/electrum/lntransport.py +++ b/electrum/lntransport.py @@ -89,6 +89,7 @@ def create_ephemeral_key() -> (bytes, bytes): class LNTransportBase: reader: StreamReader writer: StreamWriter + privkey: bytes def name(self) -> str: raise NotImplementedError() diff --git a/electrum/lnutil.py b/electrum/lnutil.py index 0d6acb55e8..487bac613f 100644 --- a/electrum/lnutil.py +++ b/electrum/lnutil.py @@ -27,7 +27,7 @@ from .transaction import BCDataStream if TYPE_CHECKING: - from .lnchannel import Channel + from .lnchannel import Channel, AbstractChannel from .lnrouter import LNPaymentRoute from .lnonion import OnionRoutingFailureMessage @@ -504,8 +504,8 @@ def make_htlc_output_witness_script(is_received_htlc: bool, remote_revocation_pu payment_hash=payment_hash) -def get_ordered_channel_configs(chan: 'Channel', for_us: bool) -> Tuple[Union[LocalConfig, RemoteConfig], - Union[LocalConfig, RemoteConfig]]: +def get_ordered_channel_configs(chan: 'AbstractChannel', for_us: bool) -> Tuple[Union[LocalConfig, RemoteConfig], + Union[LocalConfig, RemoteConfig]]: conf = chan.config[LOCAL] if for_us else chan.config[REMOTE] other_conf = chan.config[LOCAL] if not for_us else chan.config[REMOTE] return conf, other_conf @@ -781,7 +781,7 @@ def extract_ctn_from_tx(tx: Transaction, txin_index: int, funder_payment_basepoi obs = ((sequence & 0xffffff) << 24) + (locktime & 0xffffff) return get_obscured_ctn(obs, funder_payment_basepoint, fundee_payment_basepoint) -def extract_ctn_from_tx_and_chan(tx: Transaction, chan: 'Channel') -> int: +def extract_ctn_from_tx_and_chan(tx: Transaction, chan: 'AbstractChannel') -> int: funder_conf = chan.config[LOCAL] if chan.is_initiator() else chan.config[REMOTE] fundee_conf = chan.config[LOCAL] if not chan.is_initiator() else chan.config[REMOTE] return extract_ctn_from_tx(tx, txin_index=0, diff --git a/electrum/lnwatcher.py b/electrum/lnwatcher.py index 624546ca5a..b255322237 100644 --- a/electrum/lnwatcher.py +++ b/electrum/lnwatcher.py @@ -4,20 +4,13 @@ from typing import NamedTuple, Iterable, TYPE_CHECKING import os -import queue -import threading -import concurrent -from collections import defaultdict import asyncio from enum import IntEnum, auto from typing import NamedTuple, Dict from .sql_db import SqlDB, sql from .wallet_db import WalletDB -from .util import bh2u, bfh, log_exceptions, ignore_exceptions -from .lnutil import Outpoint -from . import wallet -from .storage import WalletStorage +from .util import bh2u, bfh, log_exceptions, ignore_exceptions, TxMinedInfo from .address_synchronizer import AddressSynchronizer, TX_HEIGHT_LOCAL, TX_HEIGHT_UNCONF_PARENT, TX_HEIGHT_UNCONFIRMED from .transaction import Transaction @@ -199,17 +192,22 @@ async def check_onchain_situation(self, address, funding_outpoint): else: keep_watching = True await self.update_channel_state( - funding_outpoint, funding_txid, - funding_height, closing_txid, - closing_height, keep_watching) + funding_outpoint=funding_outpoint, + funding_txid=funding_txid, + funding_height=funding_height, + closing_txid=closing_txid, + closing_height=closing_height, + keep_watching=keep_watching) if not keep_watching: await self.unwatch_channel(address, funding_outpoint) - async def do_breach_remedy(self, funding_outpoint, closing_tx, spenders): - raise NotImplementedError() # implemented by subclasses + async def do_breach_remedy(self, funding_outpoint, closing_tx, spenders) -> bool: + raise NotImplementedError() # implemented by subclasses - async def update_channel_state(self, *args): - raise NotImplementedError() # implemented by subclasses + async def update_channel_state(self, *, funding_outpoint: str, funding_txid: str, + funding_height: TxMinedInfo, closing_txid: str, + closing_height: TxMinedInfo, keep_watching: bool) -> None: + raise NotImplementedError() # implemented by subclasses def inspect_tx_candidate(self, outpoint, n): prev_txid, index = outpoint.split(':') @@ -325,7 +323,7 @@ async def unwatch_channel(self, address, funding_outpoint): if funding_outpoint in self.tx_progress: self.tx_progress[funding_outpoint].all_done.set() - async def update_channel_state(self, *args): + async def update_channel_state(self, *args, **kwargs): pass @@ -340,17 +338,23 @@ def __init__(self, lnworker: 'LNWallet', network: 'Network'): @ignore_exceptions @log_exceptions - async def update_channel_state(self, funding_outpoint, funding_txid, funding_height, closing_txid, closing_height, keep_watching): + async def update_channel_state(self, *, funding_outpoint: str, funding_txid: str, + funding_height: TxMinedInfo, closing_txid: str, + closing_height: TxMinedInfo, keep_watching: bool) -> None: chan = self.lnworker.channel_by_txo(funding_outpoint) if not chan: return - chan.update_onchain_state(funding_txid, funding_height, closing_txid, closing_height, keep_watching) + chan.update_onchain_state(funding_txid=funding_txid, + funding_height=funding_height, + closing_txid=closing_txid, + closing_height=closing_height, + keep_watching=keep_watching) await self.lnworker.on_channel_update(chan) async def do_breach_remedy(self, funding_outpoint, closing_tx, spenders): chan = self.lnworker.channel_by_txo(funding_outpoint) if not chan: - return + return False # detect who closed and set sweep_info sweep_info_dict = chan.sweep_ctx(closing_tx) keep_watching = False if sweep_info_dict else not self.is_deeply_mined(closing_tx.txid()) diff --git a/electrum/lnworker.py b/electrum/lnworker.py index 8fabed52cd..3806ae405c 100644 --- a/electrum/lnworker.py +++ b/electrum/lnworker.py @@ -432,7 +432,7 @@ def __init__(self, wallet: 'Abstract_Wallet', xprv): self.preimages = self.db.get_dict('lightning_preimages') # RHASH -> preimage self.sweep_address = wallet.get_receiving_address() self.lock = threading.RLock() - self.logs = defaultdict(list) # (not persisted) type: Dict[str, List[PaymentAttemptLog]] # key is RHASH + self.logs = defaultdict(list) # type: Dict[str, List[PaymentAttemptLog]] # key is RHASH # (not persisted) self.is_routing = set() # (not persisted) keys of invoices that are in PR_ROUTING state # used in tests self.enable_htlc_settle = asyncio.Event() From 12283d625b49c3a7d70f5fa7e9246098b6caf6bc Mon Sep 17 00:00:00 2001 From: SomberNight Date: Mon, 13 Apr 2020 16:02:05 +0200 Subject: [PATCH 060/117] (trivial) rename lnchannel.channel_states to ChannelState --- electrum/gui/qt/channels_list.py | 4 +- electrum/lnchannel.py | 88 ++++++++++++++++---------------- electrum/lnpeer.py | 36 ++++++------- electrum/lnworker.py | 18 +++---- electrum/tests/test_lnchannel.py | 6 +-- electrum/tests/test_lnpeer.py | 18 +++---- 6 files changed, 86 insertions(+), 84 deletions(-) diff --git a/electrum/gui/qt/channels_list.py b/electrum/gui/qt/channels_list.py index 543840dbc7..95178199df 100644 --- a/electrum/gui/qt/channels_list.py +++ b/electrum/gui/qt/channels_list.py @@ -11,7 +11,7 @@ from electrum.util import bh2u, NotEnoughFunds, NoDynamicFeeEstimates from electrum.i18n import _ -from electrum.lnchannel import Channel, peer_states +from electrum.lnchannel import Channel, PeerState from electrum.wallet import Abstract_Wallet from electrum.lnutil import LOCAL, REMOTE, format_short_channel_id, LN_MAX_FUNDING_SAT from electrum.lnworker import LNWallet @@ -179,7 +179,7 @@ def create_menu(self, position): menu.addAction(_("View funding transaction"), lambda: self.parent.show_transaction(funding_tx)) if not chan.is_closed(): menu.addSeparator() - if chan.peer_state == peer_states.GOOD: + if chan.peer_state == PeerState.GOOD: menu.addAction(_("Close channel"), lambda: self.close_channel(channel_id)) menu.addAction(_("Force-close channel"), lambda: self.force_close(channel_id)) else: diff --git a/electrum/lnchannel.py b/electrum/lnchannel.py index ecc2276270..7eb76cc99e 100644 --- a/electrum/lnchannel.py +++ b/electrum/lnchannel.py @@ -69,25 +69,27 @@ # lightning channel states # Note: these states are persisted by name (for a given channel) in the wallet file, # so consider doing a wallet db upgrade when changing them. -class channel_states(IntEnum): # TODO rename to use CamelCase - PREOPENING = 0 # Initial negotiation. Channel will not be reestablished - OPENING = 1 # Channel will be reestablished. (per BOLT2) - # - Funding node: has received funding_signed (can broadcast the funding tx) - # - Non-funding node: has sent the funding_signed message. - FUNDED = 2 # Funding tx was mined (requires min_depth and tx verification) - OPEN = 3 # both parties have sent funding_locked - CLOSING = 4 # shutdown has been sent, and closing tx is unconfirmed. - FORCE_CLOSING = 5 # we force-closed, and closing tx is unconfirmed. (otherwise we remain OPEN) - CLOSED = 6 # closing tx has been mined - REDEEMED = 7 # we can stop watching - -class peer_states(IntEnum): # TODO rename to use CamelCase +class ChannelState(IntEnum): + PREOPENING = 0 # Initial negotiation. Channel will not be reestablished + OPENING = 1 # Channel will be reestablished. (per BOLT2) + # - Funding node: has received funding_signed (can broadcast the funding tx) + # - Non-funding node: has sent the funding_signed message. + FUNDED = 2 # Funding tx was mined (requires min_depth and tx verification) + OPEN = 3 # both parties have sent funding_locked + CLOSING = 4 # shutdown has been sent, and closing tx is unconfirmed. + FORCE_CLOSING = 5 # we force-closed, and closing tx is unconfirmed. (otherwise we remain OPEN) + CLOSED = 6 # closing tx has been mined + REDEEMED = 7 # we can stop watching + + +class PeerState(IntEnum): DISCONNECTED = 0 REESTABLISHING = 1 GOOD = 2 BAD = 3 -cs = channel_states + +cs = ChannelState state_transitions = [ (cs.PREOPENING, cs.OPENING), (cs.OPENING, cs.FUNDED), @@ -102,14 +104,14 @@ class peer_states(IntEnum): # TODO rename to use CamelCase (cs.OPENING, cs.CLOSED), (cs.FUNDED, cs.CLOSED), (cs.OPEN, cs.CLOSED), - (cs.CLOSING, cs.CLOSING), # if we reestablish + (cs.CLOSING, cs.CLOSING), # if we reestablish (cs.CLOSING, cs.CLOSED), - (cs.FORCE_CLOSING, cs.FORCE_CLOSING), # allow multiple attempts + (cs.FORCE_CLOSING, cs.FORCE_CLOSING), # allow multiple attempts (cs.FORCE_CLOSING, cs.CLOSED), (cs.FORCE_CLOSING, cs.REDEEMED), (cs.CLOSED, cs.REDEEMED), - (cs.OPENING, cs.REDEEMED), # channel never funded (dropped from mempool) - (cs.PREOPENING, cs.REDEEMED), # channel never funded + (cs.OPENING, cs.REDEEMED), # channel never funded (dropped from mempool) + (cs.PREOPENING, cs.REDEEMED), # channel never funded ] del cs # delete as name is ambiguous without context @@ -135,7 +137,7 @@ class AbstractChannel(Logger, ABC): channel_id: bytes funding_outpoint: Outpoint node_id: bytes - _state: channel_states + _state: ChannelState def set_short_channel_id(self, short_id: ShortChannelID) -> None: self.short_channel_id = short_id @@ -150,7 +152,7 @@ def get_id_for_log(self) -> str: def short_id_for_GUI(self) -> str: return format_short_channel_id(self.short_channel_id) - def set_state(self, state: channel_states) -> None: + def set_state(self, state: ChannelState) -> None: """ set on-chain state """ old_state = self._state if (old_state, state) not in state_transitions: @@ -161,24 +163,24 @@ def set_state(self, state: channel_states) -> None: if self.lnworker: self.lnworker.channel_state_changed(self) - def get_state(self) -> channel_states: + def get_state(self) -> ChannelState: return self._state def is_funded(self): - return self.get_state() >= channel_states.FUNDED + return self.get_state() >= ChannelState.FUNDED def is_open(self): - return self.get_state() == channel_states.OPEN + return self.get_state() == ChannelState.OPEN def is_closing(self): - return self.get_state() in [channel_states.CLOSING, channel_states.FORCE_CLOSING] + return self.get_state() in [ChannelState.CLOSING, ChannelState.FORCE_CLOSING] def is_closed(self): # the closing txid has been saved - return self.get_state() >= channel_states.CLOSED + return self.get_state() >= ChannelState.CLOSED def is_redeemed(self): - return self.get_state() == channel_states.REDEEMED + return self.get_state() == ChannelState.REDEEMED def save_funding_height(self, *, txid: str, height: int, timestamp: Optional[int]) -> None: self.storage['funding_height'] = txid, height, timestamp @@ -241,7 +243,7 @@ def update_onchain_state(self, *, funding_txid: str, funding_height: TxMinedInfo def update_unfunded_state(self): self.delete_funding_height() self.delete_closing_height() - if self.get_state() in [channel_states.PREOPENING, channel_states.OPENING, channel_states.FORCE_CLOSING] and self.lnworker: + if self.get_state() in [ChannelState.PREOPENING, ChannelState.OPENING, ChannelState.FORCE_CLOSING] and self.lnworker: if self.is_initiator(): # set channel state to REDEEMED so that it can be removed manually # to protect ourselves against a server lying by omission, @@ -249,7 +251,7 @@ def update_unfunded_state(self): inputs = self.storage.get('funding_inputs', []) if not inputs: self.logger.info(f'channel funding inputs are not provided') - self.set_state(channel_states.REDEEMED) + self.set_state(ChannelState.REDEEMED) for i in inputs: spender_txid = self.lnworker.wallet.db.get_spent_outpoint(*i) if spender_txid is None: @@ -258,7 +260,7 @@ def update_unfunded_state(self): tx_mined_height = self.lnworker.wallet.get_tx_height(spender_txid) if tx_mined_height.conf > lnutil.REDEEM_AFTER_DOUBLE_SPENT_DELAY: self.logger.info(f'channel is double spent {inputs}') - self.set_state(channel_states.REDEEMED) + self.set_state(ChannelState.REDEEMED) break else: now = int(time.time()) @@ -271,24 +273,24 @@ def update_funded_state(self, *, funding_txid: str, funding_height: TxMinedInfo) if funding_height.conf>0: self.set_short_channel_id(ShortChannelID.from_components( funding_height.height, funding_height.txpos, self.funding_outpoint.output_index)) - if self.get_state() == channel_states.OPENING: + if self.get_state() == ChannelState.OPENING: if self.is_funding_tx_mined(funding_height): - self.set_state(channel_states.FUNDED) + self.set_state(ChannelState.FUNDED) def update_closed_state(self, *, funding_txid: str, funding_height: TxMinedInfo, closing_txid: str, closing_height: TxMinedInfo, keep_watching: bool) -> None: self.save_funding_height(txid=funding_txid, height=funding_height.height, timestamp=funding_height.timestamp) self.save_closing_height(txid=closing_txid, height=closing_height.height, timestamp=closing_height.timestamp) - if self.get_state() < channel_states.CLOSED: + if self.get_state() < ChannelState.CLOSED: conf = closing_height.conf if conf > 0: - self.set_state(channel_states.CLOSED) + self.set_state(ChannelState.CLOSED) else: # we must not trust the server with unconfirmed transactions # if the remote force closed, we remain OPEN until the closing tx is confirmed pass - if self.get_state() == channel_states.CLOSED and not keep_watching: - self.set_state(channel_states.REDEEMED) + if self.get_state() == ChannelState.CLOSED and not keep_watching: + self.set_state(ChannelState.REDEEMED) @abstractmethod def is_initiator(self) -> bool: @@ -368,7 +370,7 @@ def __init__(self, cb: ChannelBackupStorage, *, sweep_address=None, lnworker=Non self._sweep_info = {} self.sweep_address = sweep_address self.storage = {} # dummy storage - self._state = channel_states.OPENING + self._state = ChannelState.OPENING self.config = {} self.config[LOCAL] = LocalConfig.from_seed( channel_seed=cb.channel_seed, @@ -473,8 +475,8 @@ def __init__(self, state: 'StoredDict', *, sweep_address=None, name=None, lnwork self.onion_keys = state['onion_keys'] # type: Dict[int, bytes] self.data_loss_protect_remote_pcp = state['data_loss_protect_remote_pcp'] self.hm = HTLCManager(log=state['log'], initial_feerate=initial_feerate) - self._state = channel_states[state['state']] - self.peer_state = peer_states.DISCONNECTED + self._state = ChannelState[state['state']] + self.peer_state = PeerState.DISCONNECTED self._sweep_info = {} self._outgoing_channel_update = None # type: Optional[bytes] self._chan_ann_without_sigs = None # type: Optional[bytes] @@ -644,7 +646,7 @@ def open_with_first_pcp(self, remote_pcp: bytes, remote_sig: bytes) -> None: self.config[REMOTE].next_per_commitment_point = None self.config[LOCAL].current_commitment_signature = remote_sig self.hm.channel_open_finished() - self.peer_state = peer_states.GOOD + self.peer_state = PeerState.GOOD def get_state_for_GUI(self): # status displayed in the GUI @@ -652,7 +654,7 @@ def get_state_for_GUI(self): if self.is_closed(): return cs.name ps = self.peer_state - if ps != peer_states.GOOD: + if ps != PeerState.GOOD: return ps.name return cs.name @@ -663,7 +665,7 @@ def can_send_ctx_updates(self) -> bool: """Whether we can send update_fee, update_*_htlc changes to the remote.""" if not (self.is_open() or self.is_closing()): return False - if self.peer_state != peer_states.GOOD: + if self.peer_state != PeerState.GOOD: return False if not self._can_send_ctx_updates: return False @@ -699,7 +701,7 @@ def _assert_can_add_htlc(self, *, htlc_proposer: HTLCOwner, amount_msat: int) -> chan_config = self.config[htlc_receiver] if self.is_closed(): raise PaymentFailure('Channel closed') - if self.get_state() != channel_states.OPEN: + if self.get_state() != ChannelState.OPEN: raise PaymentFailure('Channel not open', self.get_state()) if htlc_proposer == LOCAL: if not self.can_send_ctx_updates(): @@ -763,7 +765,7 @@ def can_receive(self, amount_msat: int, *, check_frozen=False) -> bool: return True def should_try_to_reestablish_peer(self) -> bool: - return channel_states.PREOPENING < self._state < channel_states.FORCE_CLOSING and self.peer_state == peer_states.DISCONNECTED + return ChannelState.PREOPENING < self._state < ChannelState.FORCE_CLOSING and self.peer_state == PeerState.DISCONNECTED def get_funding_address(self): script = funding_output_script(self.config[LOCAL], self.config[REMOTE]) diff --git a/electrum/lnpeer.py b/electrum/lnpeer.py index b3864eab76..fb0365c5d3 100644 --- a/electrum/lnpeer.py +++ b/electrum/lnpeer.py @@ -31,7 +31,7 @@ process_onion_packet, OnionPacket, construct_onion_error, OnionRoutingFailureMessage, ProcessedOnionPacket, UnsupportedOnionPacketVersion, InvalidOnionMac, InvalidOnionPubkey, OnionFailureCodeMetaFlag) -from .lnchannel import Channel, RevokeAndAck, htlcsum, RemoteCtnTooFarInFuture, channel_states, peer_states +from .lnchannel import Channel, RevokeAndAck, htlcsum, RemoteCtnTooFarInFuture, ChannelState, PeerState from . import lnutil from .lnutil import (Outpoint, LocalConfig, RECEIVED, UpdateAddHtlc, RemoteConfig, OnlyPubkeyKeypair, ChannelConstraints, RevocationStore, @@ -619,7 +619,7 @@ async def channel_establishment_flow(self, password: Optional[str], funding_tx: remote_sig = payload['signature'] chan.receive_new_commitment(remote_sig, []) chan.open_with_first_pcp(remote_per_commitment_point, remote_sig) - chan.set_state(channel_states.OPENING) + chan.set_state(ChannelState.OPENING) self.lnworker.add_new_channel(chan) return chan, funding_tx @@ -633,7 +633,7 @@ def create_channel_storage(self, channel_id, outpoint, local_config, remote_conf "local_config": local_config, "constraints": constraints, "remote_update": None, - "state": channel_states.PREOPENING.name, + "state": ChannelState.PREOPENING.name, 'onion_keys': {}, 'data_loss_protect_remote_pcp': {}, "log": {}, @@ -714,7 +714,7 @@ async def on_open_channel(self, payload): ) self.funding_signed_sent.add(chan.channel_id) chan.open_with_first_pcp(payload['first_per_commitment_point'], remote_sig) - chan.set_state(channel_states.OPENING) + chan.set_state(ChannelState.OPENING) self.lnworker.add_new_channel(chan) def validate_remote_reserve(self, remote_reserve_sat: int, dust_limit: int, funding_sat: int) -> int: @@ -738,12 +738,12 @@ async def trigger_force_close(self, channel_id): async def reestablish_channel(self, chan: Channel): await self.initialized chan_id = chan.channel_id - assert channel_states.PREOPENING < chan.get_state() < channel_states.FORCE_CLOSING - if chan.peer_state != peer_states.DISCONNECTED: + assert ChannelState.PREOPENING < chan.get_state() < ChannelState.FORCE_CLOSING + if chan.peer_state != PeerState.DISCONNECTED: self.logger.info(f'reestablish_channel was called but channel {chan.get_id_for_log()} ' f'already in peer_state {chan.peer_state}') return - chan.peer_state = peer_states.REESTABLISHING + chan.peer_state = PeerState.REESTABLISHING self.network.trigger_callback('channel', chan) # BOLT-02: "A node [...] upon disconnection [...] MUST reverse any uncommitted updates sent by the other side" chan.hm.discard_unsigned_remote_updates() @@ -878,21 +878,21 @@ def are_datalossprotect_fields_valid() -> bool: # data_loss_protect_remote_pcp is used in lnsweep chan.set_data_loss_protect_remote_pcp(their_next_local_ctn - 1, their_local_pcp) self.lnworker.save_channel(chan) - chan.peer_state = peer_states.BAD + chan.peer_state = PeerState.BAD return elif we_are_ahead: self.logger.warning(f"channel_reestablish ({chan.get_id_for_log()}): we are ahead of remote! trying to force-close.") await self.lnworker.try_force_closing(chan_id) return - chan.peer_state = peer_states.GOOD + chan.peer_state = PeerState.GOOD if chan.is_funded() and their_next_local_ctn == next_local_ctn == 1: self.send_funding_locked(chan) # checks done if chan.is_funded() and chan.config[LOCAL].funding_locked_received: self.mark_open(chan) self.network.trigger_callback('channel', chan) - if chan.get_state() == channel_states.CLOSING: + if chan.get_state() == ChannelState.CLOSING: await self.send_shutdown(chan) def send_funding_locked(self, chan: Channel): @@ -972,13 +972,13 @@ def mark_open(self, chan: Channel): assert chan.is_funded() # only allow state transition from "FUNDED" to "OPEN" old_state = chan.get_state() - if old_state == channel_states.OPEN: + if old_state == ChannelState.OPEN: return - if old_state != channel_states.FUNDED: + if old_state != ChannelState.FUNDED: self.logger.info(f"cannot mark open ({chan.get_id_for_log()}), current state: {repr(old_state)}") return assert chan.config[LOCAL].funding_locked_received - chan.set_state(channel_states.OPEN) + chan.set_state(ChannelState.OPEN) self.network.trigger_callback('channel', chan) # peer may have sent us a channel update for the incoming direction previously pending_channel_update = self.orphan_channel_updates.get(chan.short_channel_id) @@ -1071,7 +1071,7 @@ def send_revoke_and_ack(self, chan: Channel): self.maybe_send_commitment(chan) def on_commitment_signed(self, chan: Channel, payload): - if chan.peer_state == peer_states.BAD: + if chan.peer_state == PeerState.BAD: return self.logger.info(f'on_commitment_signed. chan {chan.short_channel_id}. ctn: {chan.get_next_ctn(LOCAL)}.') # make sure there were changes to the ctx, otherwise the remote peer is misbehaving @@ -1116,7 +1116,7 @@ def on_update_add_htlc(self, chan: Channel, payload): cltv_expiry = payload["cltv_expiry"] amount_msat_htlc = payload["amount_msat"] onion_packet = payload["onion_routing_packet"] - if chan.get_state() != channel_states.OPEN: + if chan.get_state() != ChannelState.OPEN: raise RemoteMisbehaving(f"received update_add_htlc while chan.get_state() != OPEN. state was {chan.get_state()}") if cltv_expiry > bitcoin.NLOCKTIME_BLOCKHEIGHT_MAX: asyncio.ensure_future(self.lnworker.try_force_closing(chan.channel_id)) @@ -1285,7 +1285,7 @@ def fail_htlc(self, *, chan: Channel, htlc_id: int, onion_packet: Optional[Onion failure_code=reason.code) def on_revoke_and_ack(self, chan: Channel, payload): - if chan.peer_state == peer_states.BAD: + if chan.peer_state == PeerState.BAD: return self.logger.info(f'on_revoke_and_ack. chan {chan.short_channel_id}. ctn: {chan.get_oldest_unrevoked_ctn(REMOTE)}') rev = RevokeAndAck(payload["per_commitment_secret"], payload["next_per_commitment_point"]) @@ -1360,7 +1360,7 @@ async def on_shutdown(self, chan: Channel, payload): self.logger.info(f'({chan.get_id_for_log()}) Channel closed by remote peer {txid}') def can_send_shutdown(self, chan): - if chan.get_state() >= channel_states.OPENING: + if chan.get_state() >= ChannelState.OPENING: return True if chan.constraints.is_initiator and chan.channel_id in self.funding_created_sent: return True @@ -1377,7 +1377,7 @@ async def send_shutdown(self, chan: Channel): while chan.has_pending_changes(REMOTE): await asyncio.sleep(0.1) self.send_message('shutdown', channel_id=chan.channel_id, len=len(scriptpubkey), scriptpubkey=scriptpubkey) - chan.set_state(channel_states.CLOSING) + chan.set_state(ChannelState.CLOSING) # can fullfill or fail htlcs. cannot add htlcs, because of CLOSING state chan.set_can_send_ctx_updates(True) diff --git a/electrum/lnworker.py b/electrum/lnworker.py index 3806ae405c..4147add5c8 100644 --- a/electrum/lnworker.py +++ b/electrum/lnworker.py @@ -42,7 +42,7 @@ from .lnaddr import lnencode, LnAddr, lndecode from .ecc import der_sig_from_sig_string from .lnchannel import Channel -from .lnchannel import channel_states, peer_states +from .lnchannel import ChannelState, PeerState from . import lnutil from .lnutil import funding_output_script from .bitcoin import redeem_script_to_address @@ -514,7 +514,7 @@ def start_network(self, network: 'Network'): def peer_closed(self, peer): for chan in self.channels_for_peer(peer.pubkey).values(): - chan.peer_state = peer_states.DISCONNECTED + chan.peer_state = PeerState.DISCONNECTED self.network.trigger_callback('channel', chan) super().peer_closed(peer) @@ -664,23 +664,23 @@ def channel_by_txo(self, txo): async def on_channel_update(self, chan): - if chan.get_state() == channel_states.OPEN and chan.should_be_closed_due_to_expiring_htlcs(self.network.get_local_height()): + if chan.get_state() == ChannelState.OPEN and chan.should_be_closed_due_to_expiring_htlcs(self.network.get_local_height()): self.logger.info(f"force-closing due to expiring htlcs") await self.try_force_closing(chan.channel_id) - elif chan.get_state() == channel_states.FUNDED: + elif chan.get_state() == ChannelState.FUNDED: peer = self.peers.get(chan.node_id) if peer and peer.is_initialized(): peer.send_funding_locked(chan) - elif chan.get_state() == channel_states.OPEN: + elif chan.get_state() == ChannelState.OPEN: peer = self.peers.get(chan.node_id) if peer: await peer.maybe_update_fee(chan) conf = self.lnwatcher.get_tx_height(chan.funding_outpoint.txid).conf peer.on_network_update(chan, conf) - elif chan.get_state() == channel_states.FORCE_CLOSING: + elif chan.get_state() == ChannelState.FORCE_CLOSING: force_close_tx = chan.force_close_tx() txid = force_close_tx.txid() height = self.lnwatcher.get_tx_height(txid).height @@ -1235,19 +1235,19 @@ async def force_close_channel(self, chan_id): chan = self.channels[chan_id] tx = chan.force_close_tx() await self.network.broadcast_transaction(tx) - chan.set_state(channel_states.FORCE_CLOSING) + chan.set_state(ChannelState.FORCE_CLOSING) return tx.txid() async def try_force_closing(self, chan_id): # fails silently but sets the state, so that we will retry later chan = self.channels[chan_id] tx = chan.force_close_tx() - chan.set_state(channel_states.FORCE_CLOSING) + chan.set_state(ChannelState.FORCE_CLOSING) await self.network.try_broadcasting(tx, 'force-close') def remove_channel(self, chan_id): chan = self.channels[chan_id] - assert chan.get_state() == channel_states.REDEEMED + assert chan.get_state() == ChannelState.REDEEMED with self.lock: self.channels.pop(chan_id) self.db.get('channels').pop(chan_id.hex()) diff --git a/electrum/tests/test_lnchannel.py b/electrum/tests/test_lnchannel.py index fe1163c29e..c67a529a22 100644 --- a/electrum/tests/test_lnchannel.py +++ b/electrum/tests/test_lnchannel.py @@ -37,7 +37,7 @@ from electrum.lnutil import FeeUpdate from electrum.ecc import sig_string_from_der_sig from electrum.logging import console_stderr_handler -from electrum.lnchannel import channel_states +from electrum.lnchannel import ChannelState from electrum.json_db import StoredDict from . import ElectrumTestCase @@ -143,8 +143,8 @@ def create_test_channels(*, feerate=6000, local_msat=None, remote_msat=None): alice.hm.log[LOCAL]['ctn'] = 0 bob.hm.log[LOCAL]['ctn'] = 0 - alice._state = channel_states.OPEN - bob._state = channel_states.OPEN + alice._state = ChannelState.OPEN + bob._state = ChannelState.OPEN a_out = alice.get_latest_commitment(LOCAL).outputs() b_out = bob.get_next_commitment(REMOTE).outputs() diff --git a/electrum/tests/test_lnpeer.py b/electrum/tests/test_lnpeer.py index 9500ec0574..f38f9ad495 100644 --- a/electrum/tests/test_lnpeer.py +++ b/electrum/tests/test_lnpeer.py @@ -22,7 +22,7 @@ from electrum.lnutil import LNPeerAddr, Keypair, privkey_to_pubkey from electrum.lnutil import LightningPeerConnectionClosed, RemoteMisbehaving from electrum.lnutil import PaymentFailure, LnFeatures, HTLCOwner -from electrum.lnchannel import channel_states, peer_states, Channel +from electrum.lnchannel import ChannelState, PeerState, Channel from electrum.lnrouter import LNPathFinder from electrum.channel_db import ChannelDB from electrum.lnworker import LNWallet, NoPathFound @@ -219,8 +219,8 @@ def prepare_peers(self, alice_channel, bob_channel): w2.peer = p2 # mark_open won't work if state is already OPEN. # so set it to FUNDED - alice_channel._state = channel_states.FUNDED - bob_channel._state = channel_states.FUNDED + alice_channel._state = ChannelState.FUNDED + bob_channel._state = ChannelState.FUNDED # this populates the channel graph: p1.mark_open(alice_channel) p2.mark_open(bob_channel) @@ -250,13 +250,13 @@ def test_reestablish(self): alice_channel, bob_channel = create_test_channels() p1, p2, w1, w2, _q1, _q2 = self.prepare_peers(alice_channel, bob_channel) for chan in (alice_channel, bob_channel): - chan.peer_state = peer_states.DISCONNECTED + chan.peer_state = PeerState.DISCONNECTED async def reestablish(): await asyncio.gather( p1.reestablish_channel(alice_channel), p2.reestablish_channel(bob_channel)) - self.assertEqual(alice_channel.peer_state, peer_states.GOOD) - self.assertEqual(bob_channel.peer_state, peer_states.GOOD) + self.assertEqual(alice_channel.peer_state, PeerState.GOOD) + self.assertEqual(bob_channel.peer_state, PeerState.GOOD) gath.cancel() gath = asyncio.gather(reestablish(), p1._message_loop(), p2._message_loop(), p1.htlc_switch(), p1.htlc_switch()) async def f(): @@ -282,13 +282,13 @@ async def f(): p1, p2, w1, w2, _q1, _q2 = self.prepare_peers(alice_channel_0, bob_channel) for chan in (alice_channel_0, bob_channel): - chan.peer_state = peer_states.DISCONNECTED + chan.peer_state = PeerState.DISCONNECTED async def reestablish(): await asyncio.gather( p1.reestablish_channel(alice_channel_0), p2.reestablish_channel(bob_channel)) - self.assertEqual(alice_channel_0.peer_state, peer_states.BAD) - self.assertEqual(bob_channel._state, channel_states.FORCE_CLOSING) + self.assertEqual(alice_channel_0.peer_state, PeerState.BAD) + self.assertEqual(bob_channel._state, ChannelState.FORCE_CLOSING) # wait so that pending messages are processed #await asyncio.sleep(1) gath.cancel() From 54e1520ee4cce041d46a011cdef3ba9d2d4ec043 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Mon, 13 Apr 2020 17:04:27 +0200 Subject: [PATCH 061/117] ln: check if chain tip is stale when receiving HTLC if so, don't release preimage / don't forward HTLC --- electrum/blockchain.py | 15 +++++++++++++ electrum/lnpeer.py | 40 +++++++++++++++++++++++------------ electrum/lnutil.py | 3 ++- electrum/tests/test_lnpeer.py | 14 ++++++++++++ electrum/wallet.py | 6 +----- 5 files changed, 58 insertions(+), 20 deletions(-) diff --git a/electrum/blockchain.py b/electrum/blockchain.py index 7a0bb983d8..15884c5119 100644 --- a/electrum/blockchain.py +++ b/electrum/blockchain.py @@ -22,6 +22,7 @@ # SOFTWARE. import os import threading +import time from typing import Optional, Dict, Mapping, Sequence from . import util @@ -484,6 +485,20 @@ def header_at_tip(self) -> Optional[dict]: height = self.height() return self.read_header(height) + def is_tip_stale(self) -> bool: + STALE_DELAY = 8 * 60 * 60 # in seconds + header = self.header_at_tip() + if not header: + return True + # note: We check the timestamp only in the latest header. + # The Bitcoin consensus has a lot of leeway here: + # - needs to be greater than the median of the timestamps of the past 11 blocks, and + # - up to at most 2 hours into the future compared to local clock + # so there is ~2 hours of leeway in either direction + if header['timestamp'] + STALE_DELAY < time.time(): + return True + return False + def get_hash(self, height: int) -> str: def is_height_checkpoint(): within_cp_range = height <= constants.net.max_checkpoint() diff --git a/electrum/lnpeer.py b/electrum/lnpeer.py index fb0365c5d3..6bd02a2d44 100644 --- a/electrum/lnpeer.py +++ b/electrum/lnpeer.py @@ -1131,19 +1131,23 @@ def on_update_add_htlc(self, chan: Channel, payload): chan.receive_htlc(htlc, onion_packet) def maybe_forward_htlc(self, chan: Channel, htlc: UpdateAddHtlc, *, - onion_packet: OnionPacket, processed_onion: ProcessedOnionPacket): + onion_packet: OnionPacket, processed_onion: ProcessedOnionPacket + ) -> Optional[OnionRoutingFailureMessage]: # Forward HTLC # FIXME: there are critical safety checks MISSING here forwarding_enabled = self.network.config.get('lightning_forward_payments', False) if not forwarding_enabled: self.logger.info(f"forwarding is disabled. failing htlc.") return OnionRoutingFailureMessage(code=OnionFailureCode.PERMANENT_CHANNEL_FAILURE, data=b'') + chain = self.network.blockchain() + if chain.is_tip_stale(): + return OnionRoutingFailureMessage(code=OnionFailureCode.TEMPORARY_NODE_FAILURE, data=b'') try: next_chan_scid = processed_onion.hop_data.payload["short_channel_id"]["short_channel_id"] except: return OnionRoutingFailureMessage(code=OnionFailureCode.INVALID_ONION_PAYLOAD, data=b'\x00\x00\x00') next_chan = self.lnworker.get_channel_by_short_id(next_chan_scid) - local_height = self.network.get_local_height() + local_height = chain.height() if next_chan is None: self.logger.info(f"cannot forward htlc. cannot find next_chan {next_chan_scid}") return OnionRoutingFailureMessage(code=OnionFailureCode.UNKNOWN_NEXT_PEER, data=b'') @@ -1161,7 +1165,7 @@ def maybe_forward_htlc(self, chan: Channel, htlc: UpdateAddHtlc, *, if htlc.cltv_expiry - next_cltv_expiry < NBLOCK_OUR_CLTV_EXPIRY_DELTA: data = htlc.cltv_expiry.to_bytes(4, byteorder="big") + outgoing_chan_upd_len + outgoing_chan_upd return OnionRoutingFailureMessage(code=OnionFailureCode.INCORRECT_CLTV_EXPIRY, data=data) - if htlc.cltv_expiry - lnutil.NBLOCK_DEADLINE_BEFORE_EXPIRY_FOR_RECEIVED_HTLCS <= local_height \ + if htlc.cltv_expiry - lnutil.MIN_FINAL_CLTV_EXPIRY_ACCEPTED <= local_height \ or next_cltv_expiry <= local_height: data = outgoing_chan_upd_len + outgoing_chan_upd return OnionRoutingFailureMessage(code=OnionFailureCode.EXPIRY_TOO_SOON, data=data) @@ -1202,14 +1206,15 @@ def maybe_forward_htlc(self, chan: Channel, htlc: UpdateAddHtlc, *, return OnionRoutingFailureMessage(code=OnionFailureCode.TEMPORARY_CHANNEL_FAILURE, data=data) return None - def maybe_fulfill_htlc(self, chan: Channel, htlc: UpdateAddHtlc, *, - onion_packet: OnionPacket, processed_onion: ProcessedOnionPacket): + def maybe_fulfill_htlc(self, *, chan: Channel, htlc: UpdateAddHtlc, + onion_packet: OnionPacket, processed_onion: ProcessedOnionPacket, + ) -> Tuple[Optional[bytes], Optional[OnionRoutingFailureMessage]]: try: info = self.lnworker.get_payment_info(htlc.payment_hash) preimage = self.lnworker.get_preimage(htlc.payment_hash) except UnknownPaymentHash: reason = OnionRoutingFailureMessage(code=OnionFailureCode.INCORRECT_OR_UNKNOWN_PAYMENT_DETAILS, data=b'') - return False, reason + return None, reason try: payment_secret_from_onion = processed_onion.hop_data.payload["payment_data"]["payment_secret"] except: @@ -1217,30 +1222,37 @@ def maybe_fulfill_htlc(self, chan: Channel, htlc: UpdateAddHtlc, *, else: if payment_secret_from_onion != derive_payment_secret_from_payment_preimage(preimage): reason = OnionRoutingFailureMessage(code=OnionFailureCode.INCORRECT_OR_UNKNOWN_PAYMENT_DETAILS, data=b'') - return False, reason + return None, reason expected_received_msat = int(info.amount * 1000) if info.amount is not None else None if expected_received_msat is not None and \ not (expected_received_msat <= htlc.amount_msat <= 2 * expected_received_msat): reason = OnionRoutingFailureMessage(code=OnionFailureCode.INCORRECT_OR_UNKNOWN_PAYMENT_DETAILS, data=b'') - return False, reason - local_height = self.network.get_local_height() + return None, reason + # Check that our blockchain tip is sufficiently recent so that we have an approx idea of the height. + # We should not release the preimage for an HTLC that its sender could already time out as + # then they might try to force-close and it becomes a race. + chain = self.network.blockchain() + if chain.is_tip_stale(): + reason = OnionRoutingFailureMessage(code=OnionFailureCode.TEMPORARY_NODE_FAILURE, data=b'') + return None, reason + local_height = chain.height() if local_height + MIN_FINAL_CLTV_EXPIRY_ACCEPTED > htlc.cltv_expiry: reason = OnionRoutingFailureMessage(code=OnionFailureCode.FINAL_EXPIRY_TOO_SOON, data=b'') - return False, reason + return None, reason try: cltv_from_onion = processed_onion.hop_data.payload["outgoing_cltv_value"]["outgoing_cltv_value"] except: reason = OnionRoutingFailureMessage(code=OnionFailureCode.INVALID_ONION_PAYLOAD, data=b'\x00\x00\x00') - return False, reason + return None, reason if cltv_from_onion != htlc.cltv_expiry: reason = OnionRoutingFailureMessage(code=OnionFailureCode.FINAL_INCORRECT_CLTV_EXPIRY, data=htlc.cltv_expiry.to_bytes(4, byteorder="big")) - return False, reason + return None, reason try: amount_from_onion = processed_onion.hop_data.payload["amt_to_forward"]["amt_to_forward"] except: reason = OnionRoutingFailureMessage(code=OnionFailureCode.INVALID_ONION_PAYLOAD, data=b'\x00\x00\x00') - return False, reason + return None, reason try: amount_from_onion = processed_onion.hop_data.payload["payment_data"]["total_msat"] except: @@ -1248,7 +1260,7 @@ def maybe_fulfill_htlc(self, chan: Channel, htlc: UpdateAddHtlc, *, if amount_from_onion > htlc.amount_msat: reason = OnionRoutingFailureMessage(code=OnionFailureCode.FINAL_INCORRECT_HTLC_AMOUNT, data=htlc.amount_msat.to_bytes(8, byteorder="big")) - return False, reason + return None, reason # all good return preimage, None diff --git a/electrum/lnutil.py b/electrum/lnutil.py index 487bac613f..3064a4e845 100644 --- a/electrum/lnutil.py +++ b/electrum/lnutil.py @@ -262,7 +262,8 @@ class PaymentFailure(UserFacingException): pass ##### CLTV-expiry-delta-related values # see https://github.com/lightningnetwork/lightning-rfc/blob/master/02-peer-protocol.md#cltv_expiry_delta-selection -# the minimum cltv_expiry accepted for terminal payments +# the minimum cltv_expiry accepted for newly received HTLCs +# note: when changing, consider Blockchain.is_tip_stale() MIN_FINAL_CLTV_EXPIRY_ACCEPTED = 144 # set it a tiny bit higher for invoices as blocks could get mined # during forward path of payment diff --git a/electrum/tests/test_lnpeer.py b/electrum/tests/test_lnpeer.py index f38f9ad495..320bd6fa87 100644 --- a/electrum/tests/test_lnpeer.py +++ b/electrum/tests/test_lnpeer.py @@ -58,6 +58,7 @@ def __init__(self, tx_queue): self.channel_db.data_loaded.set() self.path_finder = LNPathFinder(self.channel_db) self.tx_queue = tx_queue + self._blockchain = MockBlockchain() @property def callback_lock(self): @@ -70,6 +71,9 @@ def callback_lock(self): def get_local_height(self): return 0 + def blockchain(self): + return self._blockchain + async def broadcast_transaction(self, tx): if self.tx_queue: await self.tx_queue.put(tx) @@ -77,6 +81,16 @@ async def broadcast_transaction(self, tx): async def try_broadcasting(self, tx, name): await self.broadcast_transaction(tx) + +class MockBlockchain: + + def height(self): + return 0 + + def is_tip_stale(self): + return False + + class MockWallet: def set_label(self, x, y): pass diff --git a/electrum/wallet.py b/electrum/wallet.py index af17309d8f..68ff677d55 100644 --- a/electrum/wallet.py +++ b/electrum/wallet.py @@ -174,11 +174,7 @@ def get_locktime_for_new_transaction(network: 'Network') -> int: if not network: return 0 chain = network.blockchain() - header = chain.header_at_tip() - if not header: - return 0 - STALE_DELAY = 8 * 60 * 60 # in seconds - if header['timestamp'] + STALE_DELAY < time.time(): + if chain.is_tip_stale(): return 0 # discourage "fee sniping" locktime = chain.height() From 40dc54e8b8f5d02f490524cd5d821460b5dc5a2c Mon Sep 17 00:00:00 2001 From: SomberNight Date: Mon, 13 Apr 2020 19:53:52 +0200 Subject: [PATCH 062/117] macOS: duplicate Qt "Preferences" menu item There is a standardised location along with reserved hotkey for "Preferences" in applications on macOS. Let's put *another* preferences menu item there. The duplicate items ensure that - an electrum user coming from a different OS, - a macOS user used to the standardised preferences location, will both find "Preferences" easily. --- electrum/gui/qt/main_window.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/electrum/gui/qt/main_window.py b/electrum/gui/qt/main_window.py index 3125e20c38..a37f25cead 100644 --- a/electrum/gui/qt/main_window.py +++ b/electrum/gui/qt/main_window.py @@ -45,7 +45,8 @@ QVBoxLayout, QGridLayout, QLineEdit, QHBoxLayout, QPushButton, QScrollArea, QTextEdit, QShortcut, QMainWindow, QCompleter, QInputDialog, - QWidget, QSizePolicy, QStatusBar, QToolTip, QDialog) + QWidget, QSizePolicy, QStatusBar, QToolTip, QDialog, + QMenu, QAction) import electrum from electrum import (keystore, ecc, constants, util, bitcoin, commands, @@ -692,10 +693,17 @@ def add_toggle_action(view_menu, tab): add_toggle_action(view_menu, self.contacts_tab) add_toggle_action(view_menu, self.console_tab) - tools_menu = menubar.addMenu(_("&Tools")) + tools_menu = menubar.addMenu(_("&Tools")) # type: QMenu + preferences_action = tools_menu.addAction(_("Preferences"), self.settings_dialog) # type: QAction + if sys.platform == 'darwin': + # "Settings"/"Preferences" are all reserved keywords in macOS. + # preferences_action will get picked up based on name (and put into a standardized location, + # and given a standard reserved hotkey) + # Hence, this menu item will be at a "uniform location re macOS processes" + preferences_action.setMenuRole(QAction.PreferencesRole) # make sure OS recognizes it as preferences + # Add another preferences item, to also have a "uniform location for Electrum between different OSes" + tools_menu.addAction(_("Electrum preferences"), self.settings_dialog) - # Settings / Preferences are all reserved keywords in macOS using this as work around - tools_menu.addAction(_("Electrum preferences") if sys.platform == 'darwin' else _("Preferences"), self.settings_dialog) tools_menu.addAction(_("&Network"), self.gui_object.show_network_dialog).setEnabled(bool(self.network)) tools_menu.addAction(_("&Lightning Network"), self.gui_object.show_lightning_dialog).setEnabled(bool(self.wallet.has_lightning() and self.network)) tools_menu.addAction(_("Local &Watchtower"), self.gui_object.show_watchtower_dialog).setEnabled(bool(self.network and self.network.local_watchtower)) From 70f70d0f801c27e2762752857936f5985319a69e Mon Sep 17 00:00:00 2001 From: SomberNight Date: Tue, 14 Apr 2020 02:15:22 +0200 Subject: [PATCH 063/117] README: mention script location (for "electrum", after pip install) related: #6082 --- README.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index a6ff0596b0..5454860129 100644 --- a/README.rst +++ b/README.rst @@ -86,6 +86,8 @@ You can also install Electrum on your system, by running this command:: This will download and install the Python dependencies used by Electrum instead of using the 'packages' directory. +It will also place an executable named :code:`electrum` in :code:`~/.local/bin`, +so make sure that is on your :code:`PATH` variable. Development version (git clone) @@ -99,7 +101,7 @@ Check out the code from GitHub:: Run install (this should install dependencies):: - python3 -m pip install --user . + python3 -m pip install --user -e . Create translations (optional):: @@ -107,6 +109,9 @@ Create translations (optional):: sudo apt-get install python-requests gettext ./contrib/pull_locale +Finally, to start Electrum:: + + ./run_electrum From da8b24d61af00109d480fe4ee5b73d130a7c1f25 Mon Sep 17 00:00:00 2001 From: ThomasV Date: Tue, 14 Apr 2020 09:48:18 +0200 Subject: [PATCH 064/117] require aiohttp_socks>=0.3 --- contrib/requirements/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/requirements/requirements.txt b/contrib/requirements/requirements.txt index e67808732b..7132ea8ef4 100644 --- a/contrib/requirements/requirements.txt +++ b/contrib/requirements/requirements.txt @@ -6,7 +6,7 @@ dnspython qdarkstyle<2.7 aiorpcx>=0.18,<0.19 aiohttp>=3.3.0,<4.0.0 -aiohttp_socks +aiohttp_socks>=0.3 certifi bitstring jsonrpcserver From 1d667fe9327d17a752084ca9c69091ec71617d33 Mon Sep 17 00:00:00 2001 From: Luke Childs Date: Tue, 14 Apr 2020 19:15:28 +0700 Subject: [PATCH 065/117] Hard fail on bad server-string (#6086) * If server-string can't be parsed, fall back to localhost. Co-Authored-By: Luke Childs Co-authored-by: ghost43 --- electrum/network.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/electrum/network.py b/electrum/network.py index 956c4f361c..e3d9ed5531 100644 --- a/electrum/network.py +++ b/electrum/network.py @@ -268,8 +268,8 @@ def __init__(self, config: SimpleConfig, *, daemon: 'Daemon' = None): try: deserialize_server(self.default_server) except: - self.logger.warning('failed to parse server-string; falling back to random.') - self.default_server = None + self.logger.warning('failed to parse server-string; falling back to localhost.') + self.default_server = "localhost:50002:s" if not self.default_server: self.default_server = pick_random_server() From 73325831b7075f461e106f36099865a3fedadb72 Mon Sep 17 00:00:00 2001 From: ThomasV Date: Tue, 14 Apr 2020 18:28:23 +0200 Subject: [PATCH 066/117] run lnworker.main_loop directly on the event loop --- electrum/lnworker.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/electrum/lnworker.py b/electrum/lnworker.py index 4147add5c8..401e1067cd 100644 --- a/electrum/lnworker.py +++ b/electrum/lnworker.py @@ -232,11 +232,10 @@ def start_network(self, network: 'Network'): assert network self.network = network self.config = network.config - daemon = network.daemon self.channel_db = self.network.channel_db self._last_tried_peer = {} # type: Dict[LNPeerAddr, float] # LNPeerAddr -> unix timestamp self._add_peers_from_config() - asyncio.run_coroutine_threadsafe(daemon.taskgroup.spawn(self.main_loop()), self.network.asyncio_loop) + asyncio.run_coroutine_threadsafe(self.main_loop(), self.network.asyncio_loop) def _add_peers_from_config(self): peer_list = self.config.get('lightning_peers', []) From 92244041081db96b92925c9e76b117035e241011 Mon Sep 17 00:00:00 2001 From: ThomasV Date: Tue, 14 Apr 2020 16:12:47 +0200 Subject: [PATCH 067/117] Move callback manager out of Network class --- electrum/address_synchronizer.py | 8 ++-- electrum/channel_db.py | 6 +-- electrum/daemon.py | 3 +- electrum/exchange_rate.py | 10 ++--- electrum/gui/kivy/main_window.py | 30 +++++++-------- electrum/gui/qt/channel_details.py | 9 +++-- electrum/gui/qt/lightning_dialog.py | 7 ++-- electrum/gui/qt/main_window.py | 9 ++--- electrum/gui/qt/network_dialog.py | 4 +- electrum/gui/stdio.py | 3 +- electrum/gui/text.py | 4 +- electrum/interface.py | 6 +-- electrum/lnchannel.py | 8 ++-- electrum/lnpeer.py | 8 ++-- electrum/lnwatcher.py | 8 ++-- electrum/lnworker.py | 60 ++++++++++++++--------------- electrum/network.py | 47 ++++++---------------- electrum/synchronizer.py | 5 ++- electrum/util.py | 40 ++++++++++++++++++- 19 files changed, 146 insertions(+), 129 deletions(-) diff --git a/electrum/address_synchronizer.py b/electrum/address_synchronizer.py index 8e586cf1aa..8e1a36270d 100644 --- a/electrum/address_synchronizer.py +++ b/electrum/address_synchronizer.py @@ -28,7 +28,7 @@ from collections import defaultdict from typing import TYPE_CHECKING, Dict, Optional, Set, Tuple, NamedTuple, Sequence, List -from . import bitcoin +from . import bitcoin, util from .bitcoin import COINBASE_MATURITY from .util import profiler, bfh, TxMinedInfo from .transaction import Transaction, TxOutput, TxInput, PartialTxInput, TxOutpoint, PartialTransaction @@ -161,7 +161,7 @@ def start_network(self, network): if self.network is not None: self.synchronizer = Synchronizer(self) self.verifier = SPV(self.network, self) - self.network.register_callback(self.on_blockchain_updated, ['blockchain_updated']) + util.register_callback(self.on_blockchain_updated, ['blockchain_updated']) def on_blockchain_updated(self, event, *args): self._get_addr_balance_cache = {} # invalidate cache @@ -174,7 +174,7 @@ def stop_threads(self): if self.verifier: asyncio.run_coroutine_threadsafe(self.verifier.stop(), self.network.asyncio_loop) self.verifier = None - self.network.unregister_callback(self.on_blockchain_updated) + util.unregister_callback(self.on_blockchain_updated) self.db.put('stored_height', self.get_local_height()) def add_address(self, address): @@ -546,7 +546,7 @@ def add_verified_tx(self, tx_hash: str, info: TxMinedInfo): self.unverified_tx.pop(tx_hash, None) self.db.add_verified_tx(tx_hash, info) tx_mined_status = self.get_tx_height(tx_hash) - self.network.trigger_callback('verified', self, tx_hash, tx_mined_status) + util.trigger_callback('verified', self, tx_hash, tx_mined_status) def get_unverified_txs(self): '''Returns a map from tx hash to transaction height''' diff --git a/electrum/channel_db.py b/electrum/channel_db.py index f0da99897d..e374519d5e 100644 --- a/electrum/channel_db.py +++ b/electrum/channel_db.py @@ -35,7 +35,7 @@ from .sql_db import SqlDB, sql -from . import constants +from . import constants, util from .util import bh2u, profiler, get_headers_dir, bfh, is_ip_address, list_enabled_bits from .logging import Logger from .lnutil import (LNPeerAddr, format_short_channel_id, ShortChannelID, @@ -269,8 +269,8 @@ def update_counts(self): self.num_nodes = len(self._nodes) self.num_channels = len(self._channels) self.num_policies = len(self._policies) - self.network.trigger_callback('channel_db', self.num_nodes, self.num_channels, self.num_policies) - self.network.trigger_callback('ln_gossip_sync_progress') + util.trigger_callback('channel_db', self.num_nodes, self.num_channels, self.num_policies) + util.trigger_callback('ln_gossip_sync_progress') def get_channel_ids(self): with self.lock: diff --git a/electrum/daemon.py b/electrum/daemon.py index 77d190b9c7..41e9251606 100644 --- a/electrum/daemon.py +++ b/electrum/daemon.py @@ -41,6 +41,7 @@ from jsonrpcclient.clients.aiohttp_client import AiohttpClient from aiorpcx import TaskGroup +from . import util from .network import Network from .util import (json_decode, to_bytes, to_string, profiler, standardize_path, constant_time_compare) from .util import PR_PAID, PR_EXPIRED, get_request_status @@ -181,7 +182,7 @@ def __init__(self, daemon: 'Daemon'): self.daemon = daemon self.config = daemon.config self.pending = defaultdict(asyncio.Event) - self.daemon.network.register_callback(self.on_payment, ['payment_received']) + util.register_callback(self.on_payment, ['payment_received']) async def on_payment(self, evt, wallet, key, status): if status == PR_PAID: diff --git a/electrum/exchange_rate.py b/electrum/exchange_rate.py index 5cee7d33ad..49bad1c8b0 100644 --- a/electrum/exchange_rate.py +++ b/electrum/exchange_rate.py @@ -12,6 +12,7 @@ from aiorpcx.curio import timeout_after, TaskTimeout, TaskGroup +from . import util from .bitcoin import COIN from .i18n import _ from .util import (ThreadJob, make_dir, log_exceptions, @@ -456,8 +457,7 @@ def __init__(self, config: SimpleConfig, network: Network): ThreadJob.__init__(self) self.config = config self.network = network - if self.network: - self.network.register_callback(self.set_proxy, ['proxy_set']) + util.register_callback(self.set_proxy, ['proxy_set']) self.ccy = self.get_currency() self.history_used_spot = False self.ccy_combo = None @@ -567,12 +567,10 @@ def set_exchange(self, name): self.exchange.read_historical_rates(self.ccy, self.cache_dir) def on_quotes(self): - if self.network: - self.network.trigger_callback('on_quotes') + util.trigger_callback('on_quotes') def on_history(self): - if self.network: - self.network.trigger_callback('on_history') + util.trigger_callback('on_history') def exchange_rate(self) -> Decimal: """Returns the exchange rate as a Decimal""" diff --git a/electrum/gui/kivy/main_window.py b/electrum/gui/kivy/main_window.py index 373ae3804e..120a5f66ae 100644 --- a/electrum/gui/kivy/main_window.py +++ b/electrum/gui/kivy/main_window.py @@ -13,6 +13,7 @@ from electrum.wallet_db import WalletDB from electrum.wallet import Wallet, InternalAddressCorruption, Abstract_Wallet from electrum.plugin import run_hook +from electrum import util from electrum.util import (profiler, InvalidPassword, send_exception_to_crash_reporter, format_satoshis, format_satoshis_plain, format_fee_satoshis, PR_PAID, PR_FAILED, maybe_extract_bolt11_invoice) @@ -50,7 +51,6 @@ # delayed imports: for startup speed on android notification = app = ref = None -util = False # register widget cache for keeping memory down timeout to forever to cache # the data @@ -565,20 +565,20 @@ def on_start(self): if self.network: interests = ['wallet_updated', 'network_updated', 'blockchain_updated', 'status', 'new_transaction', 'verified'] - self.network.register_callback(self.on_network_event, interests) - self.network.register_callback(self.on_fee, ['fee']) - self.network.register_callback(self.on_fee_histogram, ['fee_histogram']) - self.network.register_callback(self.on_quotes, ['on_quotes']) - self.network.register_callback(self.on_history, ['on_history']) - self.network.register_callback(self.on_channels, ['channels_updated']) - self.network.register_callback(self.on_channel, ['channel']) - self.network.register_callback(self.on_invoice_status, ['invoice_status']) - self.network.register_callback(self.on_request_status, ['request_status']) - self.network.register_callback(self.on_payment_failed, ['payment_failed']) - self.network.register_callback(self.on_payment_succeeded, ['payment_succeeded']) - self.network.register_callback(self.on_channel_db, ['channel_db']) - self.network.register_callback(self.set_num_peers, ['gossip_peers']) - self.network.register_callback(self.set_unknown_channels, ['unknown_channels']) + util.register_callback(self.on_network_event, interests) + util.register_callback(self.on_fee, ['fee']) + util.register_callback(self.on_fee_histogram, ['fee_histogram']) + util.register_callback(self.on_quotes, ['on_quotes']) + util.register_callback(self.on_history, ['on_history']) + util.register_callback(self.on_channels, ['channels_updated']) + util.register_callback(self.on_channel, ['channel']) + util.register_callback(self.on_invoice_status, ['invoice_status']) + util.register_callback(self.on_request_status, ['request_status']) + util.register_callback(self.on_payment_failed, ['payment_failed']) + util.register_callback(self.on_payment_succeeded, ['payment_succeeded']) + util.register_callback(self.on_channel_db, ['channel_db']) + util.register_callback(self.set_num_peers, ['gossip_peers']) + util.register_callback(self.set_unknown_channels, ['unknown_channels']) # load wallet self.load_wallet_by_name(self.electrum_config.get_wallet_path(use_gui_last_wallet=True)) # URI passed in config diff --git a/electrum/gui/qt/channel_details.py b/electrum/gui/qt/channel_details.py index fa75d43d34..1beac21c9d 100644 --- a/electrum/gui/qt/channel_details.py +++ b/electrum/gui/qt/channel_details.py @@ -5,6 +5,7 @@ import PyQt5.QtCore as QtCore from PyQt5.QtWidgets import QLabel, QLineEdit +from electrum import util from electrum.i18n import _ from electrum.util import bh2u, format_time from electrum.lnutil import format_short_channel_id, LOCAL, REMOTE, UpdateAddHtlc, Direction @@ -132,10 +133,10 @@ def __init__(self, window: 'ElectrumWindow', chan_id: bytes): self.htlc_added.connect(self.do_htlc_added) # register callbacks for updating - window.network.register_callback(self.ln_payment_completed.emit, ['ln_payment_completed']) - window.network.register_callback(self.ln_payment_failed.emit, ['ln_payment_failed']) - window.network.register_callback(self.htlc_added.emit, ['htlc_added']) - window.network.register_callback(self.state_changed.emit, ['channel']) + util.register_callback(self.ln_payment_completed.emit, ['ln_payment_completed']) + util.register_callback(self.ln_payment_failed.emit, ['ln_payment_failed']) + util.register_callback(self.htlc_added.emit, ['htlc_added']) + util.register_callback(self.state_changed.emit, ['channel']) # set attributes of QDialog self.setWindowTitle(_('Channel Details')) diff --git a/electrum/gui/qt/lightning_dialog.py b/electrum/gui/qt/lightning_dialog.py index cf467e7fe2..f30c33505e 100644 --- a/electrum/gui/qt/lightning_dialog.py +++ b/electrum/gui/qt/lightning_dialog.py @@ -27,6 +27,7 @@ from PyQt5.QtWidgets import (QDialog, QLabel, QVBoxLayout, QPushButton) +from electrum import util from electrum.i18n import _ from .util import Buttons @@ -58,9 +59,9 @@ def __init__(self, gui_object: 'ElectrumGui'): b = QPushButton(_('Close')) b.clicked.connect(self.close) vbox.addLayout(Buttons(b)) - self.network.register_callback(self.on_channel_db, ['channel_db']) - self.network.register_callback(self.set_num_peers, ['gossip_peers']) - self.network.register_callback(self.set_unknown_channels, ['unknown_channels']) + util.register_callback(self.on_channel_db, ['channel_db']) + util.register_callback(self.set_num_peers, ['gossip_peers']) + util.register_callback(self.set_unknown_channels, ['unknown_channels']) self.network.channel_db.update_counts() # trigger callback self.set_num_peers('', self.network.lngossip.num_peers()) self.set_unknown_channels('', len(self.network.lngossip.unknown_ids)) diff --git a/electrum/gui/qt/main_window.py b/electrum/gui/qt/main_window.py index a37f25cead..062f153bd3 100644 --- a/electrum/gui/qt/main_window.py +++ b/electrum/gui/qt/main_window.py @@ -272,7 +272,7 @@ def add_optional_tab(tabs, tab, icon, description, name): # window from being GC-ed when closed, callbacks should be # methods of this class only, and specifically not be # partials, lambdas or methods of subobjects. Hence... - self.network.register_callback(self.on_network, interests) + util.register_callback(self.on_network, interests) # set initial message self.console.showMessage(self.network.banner) @@ -466,8 +466,8 @@ def close_wallet(self): def load_wallet(self, wallet): wallet.thread = TaskThread(self, self.on_error) self.update_recently_visited(wallet.storage.path) - if wallet.lnworker and wallet.network: - wallet.network.trigger_callback('channels_updated', wallet) + if wallet.lnworker: + util.trigger_callback('channels_updated', wallet) self.need_update.set() # Once GUI has been initialized check if we want to announce something since the callback has been called before the GUI was initialized # update menus @@ -2889,8 +2889,7 @@ def closeEvent(self, event): def clean_up(self): self.wallet.thread.stop() - if self.network: - self.network.unregister_callback(self.on_network) + util.unregister_callback(self.on_network) self.config.set_key("is_maximized", self.isMaximized()) if not self.isMaximized(): g = self.geometry() diff --git a/electrum/gui/qt/network_dialog.py b/electrum/gui/qt/network_dialog.py index 428df1b516..b452295e39 100644 --- a/electrum/gui/qt/network_dialog.py +++ b/electrum/gui/qt/network_dialog.py @@ -35,7 +35,7 @@ from PyQt5.QtGui import QFontMetrics from electrum.i18n import _ -from electrum import constants, blockchain +from electrum import constants, blockchain, util from electrum.interface import serialize_server, deserialize_server from electrum.network import Network from electrum.logging import get_logger @@ -61,7 +61,7 @@ def __init__(self, network, config, network_updated_signal_obj): vbox.addLayout(Buttons(CloseButton(self))) self.network_updated_signal_obj.network_updated_signal.connect( self.on_update) - network.register_callback(self.on_network, ['network_updated']) + util.register_callback(self.on_network, ['network_updated']) def on_network(self, event, *args): self.network_updated_signal_obj.network_updated_signal.emit(event, args) diff --git a/electrum/gui/stdio.py b/electrum/gui/stdio.py index dc03b084d9..d618a66443 100644 --- a/electrum/gui/stdio.py +++ b/electrum/gui/stdio.py @@ -3,6 +3,7 @@ import datetime import logging +from electrum import util from electrum import WalletStorage, Wallet from electrum.util import format_satoshis from electrum.bitcoin import is_address, COIN @@ -43,7 +44,7 @@ def __init__(self, config, daemon, plugins): self.wallet.start_network(self.network) self.contacts = self.wallet.contacts - self.network.register_callback(self.on_network, ['wallet_updated', 'network_updated', 'banner']) + util.register_callback(self.on_network, ['wallet_updated', 'network_updated', 'banner']) self.commands = [_("[h] - displays this help text"), \ _("[i] - display transaction history"), \ _("[o] - enter payment order"), \ diff --git a/electrum/gui/text.py b/electrum/gui/text.py index eec61f8f10..8e075b068c 100644 --- a/electrum/gui/text.py +++ b/electrum/gui/text.py @@ -8,6 +8,7 @@ import logging import electrum +from electrum import util from electrum.util import format_satoshis from electrum.bitcoin import is_address, COIN from electrum.transaction import PartialTxOutput @@ -65,8 +66,7 @@ def __init__(self, config, daemon, plugins): self.str_fee = "" self.history = None - if self.network: - self.network.register_callback(self.update, ['wallet_updated', 'network_updated']) + util.register_callback(self.update, ['wallet_updated', 'network_updated']) self.tab_names = [_("History"), _("Send"), _("Receive"), _("Addresses"), _("Contacts"), _("Banner")] self.num_tabs = len(self.tab_names) diff --git a/electrum/interface.py b/electrum/interface.py index f6f01d064e..bafba8ceb9 100644 --- a/electrum/interface.py +++ b/electrum/interface.py @@ -548,7 +548,7 @@ async def run_fetch_blocks(self): raise GracefulDisconnect('server tip below max checkpoint') self._mark_ready() await self._process_header_at_tip() - self.network.trigger_callback('network_updated') + util.trigger_callback('network_updated') await self.network.switch_unwanted_fork_interface() await self.network.switch_lagging_interface() @@ -563,7 +563,7 @@ async def _process_header_at_tip(self): # in the simple case, height == self.tip+1 if height <= self.tip: await self.sync_until(height) - self.network.trigger_callback('blockchain_updated') + util.trigger_callback('blockchain_updated') async def sync_until(self, height, next_height=None): if next_height is None: @@ -578,7 +578,7 @@ async def sync_until(self, height, next_height=None): raise GracefulDisconnect('server chain conflicts with checkpoints or genesis') last, height = await self.step(height) continue - self.network.trigger_callback('network_updated') + util.trigger_callback('network_updated') height = (height // 2016 * 2016) + num_headers assert height <= next_height+1, (height, self.tip) last = 'catchup' diff --git a/electrum/lnchannel.py b/electrum/lnchannel.py index 7eb76cc99e..0f7405e4a4 100644 --- a/electrum/lnchannel.py +++ b/electrum/lnchannel.py @@ -33,7 +33,7 @@ import attr from . import ecc -from . import constants +from . import constants, util from .util import bfh, bh2u, chunks, TxMinedInfo from .bitcoin import redeem_script_to_address from .crypto import sha256, sha256d @@ -679,16 +679,14 @@ def is_frozen_for_sending(self) -> bool: def set_frozen_for_sending(self, b: bool) -> None: self.storage['frozen_for_sending'] = bool(b) - if self.lnworker: - self.lnworker.network.trigger_callback('channel', self) + util.trigger_callback('channel', self) def is_frozen_for_receiving(self) -> bool: return self.storage.get('frozen_for_receiving', False) def set_frozen_for_receiving(self, b: bool) -> None: self.storage['frozen_for_receiving'] = bool(b) - if self.lnworker: - self.lnworker.network.trigger_callback('channel', self) + util.trigger_callback('channel', self) def _assert_can_add_htlc(self, *, htlc_proposer: HTLCOwner, amount_msat: int) -> None: """Raises PaymentFailure if the htlc_proposer cannot add this new HTLC. diff --git a/electrum/lnpeer.py b/electrum/lnpeer.py index 6bd02a2d44..7157761a20 100644 --- a/electrum/lnpeer.py +++ b/electrum/lnpeer.py @@ -19,7 +19,7 @@ import aiorpcx from .crypto import sha256, sha256d -from . import bitcoin +from . import bitcoin, util from . import ecc from .ecc import sig_string_from_r_and_s, get_r_and_s_from_sig_string, der_sig_from_sig_string from . import constants @@ -744,7 +744,7 @@ async def reestablish_channel(self, chan: Channel): f'already in peer_state {chan.peer_state}') return chan.peer_state = PeerState.REESTABLISHING - self.network.trigger_callback('channel', chan) + util.trigger_callback('channel', chan) # BOLT-02: "A node [...] upon disconnection [...] MUST reverse any uncommitted updates sent by the other side" chan.hm.discard_unsigned_remote_updates() # ctns @@ -891,7 +891,7 @@ def are_datalossprotect_fields_valid() -> bool: # checks done if chan.is_funded() and chan.config[LOCAL].funding_locked_received: self.mark_open(chan) - self.network.trigger_callback('channel', chan) + util.trigger_callback('channel', chan) if chan.get_state() == ChannelState.CLOSING: await self.send_shutdown(chan) @@ -979,7 +979,7 @@ def mark_open(self, chan: Channel): return assert chan.config[LOCAL].funding_locked_received chan.set_state(ChannelState.OPEN) - self.network.trigger_callback('channel', chan) + util.trigger_callback('channel', chan) # peer may have sent us a channel update for the incoming direction previously pending_channel_update = self.orphan_channel_updates.get(chan.short_channel_id) if pending_channel_update: diff --git a/electrum/lnwatcher.py b/electrum/lnwatcher.py index b255322237..4787253561 100644 --- a/electrum/lnwatcher.py +++ b/electrum/lnwatcher.py @@ -8,6 +8,7 @@ from enum import IntEnum, auto from typing import NamedTuple, Dict +from . import util from .sql_db import SqlDB, sql from .wallet_db import WalletDB from .util import bh2u, bfh, log_exceptions, ignore_exceptions, TxMinedInfo @@ -139,8 +140,9 @@ def __init__(self, network: 'Network'): self.config = network.config self.channels = {} self.network = network - self.network.register_callback(self.on_network_update, - ['network_updated', 'blockchain_updated', 'verified', 'wallet_updated', 'fee']) + util.register_callback( + self.on_network_update, + ['network_updated', 'blockchain_updated', 'verified', 'wallet_updated', 'fee']) # status gets populated when we run self.channel_status = {} @@ -420,4 +422,4 @@ async def try_redeem(self, prevout: str, sweep_info: 'SweepInfo', name: str) -> tx_was_added = False if tx_was_added: self.logger.info(f'added future tx: {name}. prevout: {prevout}') - self.network.trigger_callback('wallet_updated', self.lnworker.wallet) + util.trigger_callback('wallet_updated', self.lnworker.wallet) diff --git a/electrum/lnworker.py b/electrum/lnworker.py index 401e1067cd..cc8e55bd9e 100644 --- a/electrum/lnworker.py +++ b/electrum/lnworker.py @@ -21,7 +21,7 @@ import dns.exception from aiorpcx import run_in_thread -from . import constants +from . import constants, util from . import keystore from .util import profiler from .util import PR_UNPAID, PR_EXPIRED, PR_PAID, PR_INFLIGHT, PR_FAILED, PR_ROUTING @@ -367,7 +367,7 @@ class LNGossip(LNWorker): max_age = 14*24*3600 LOGGING_SHORTCUT = 'g' - def __init__(self, network): + def __init__(self): seed = os.urandom(32) node = BIP32Node.from_rootseed(seed, xtype='standard') xprv = node.to_xprv() @@ -393,16 +393,16 @@ async def add_new_ids(self, ids): known = self.channel_db.get_channel_ids() new = set(ids) - set(known) self.unknown_ids.update(new) - self.network.trigger_callback('unknown_channels', len(self.unknown_ids)) - self.network.trigger_callback('gossip_peers', self.num_peers()) - self.network.trigger_callback('ln_gossip_sync_progress') + util.trigger_callback('unknown_channels', len(self.unknown_ids)) + util.trigger_callback('gossip_peers', self.num_peers()) + util.trigger_callback('ln_gossip_sync_progress') def get_ids_to_query(self): N = 500 l = list(self.unknown_ids) self.unknown_ids = set(l[N:]) - self.network.trigger_callback('unknown_channels', len(self.unknown_ids)) - self.network.trigger_callback('ln_gossip_sync_progress') + util.trigger_callback('unknown_channels', len(self.unknown_ids)) + util.trigger_callback('ln_gossip_sync_progress') return l[0:N] def get_sync_progress_estimate(self) -> Tuple[Optional[int], Optional[int]]: @@ -514,7 +514,7 @@ def start_network(self, network: 'Network'): def peer_closed(self, peer): for chan in self.channels_for_peer(peer.pubkey).values(): chan.peer_state = PeerState.DISCONNECTED - self.network.trigger_callback('channel', chan) + util.trigger_callback('channel', chan) super().peer_closed(peer) def get_settled_payments(self): @@ -645,14 +645,14 @@ def channels_for_peer(self, node_id): def channel_state_changed(self, chan): self.save_channel(chan) - self.network.trigger_callback('channel', chan) + util.trigger_callback('channel', chan) def save_channel(self, chan): assert type(chan) is Channel if chan.config[REMOTE].next_per_commitment_point == chan.config[REMOTE].current_per_commitment_point: raise Exception("Tried to save channel with next_point == current_point, this should not happen") self.wallet.save_db() - self.network.trigger_callback('channel', chan) + util.trigger_callback('channel', chan) def channel_by_txo(self, txo): with self.lock: @@ -703,7 +703,7 @@ async def _open_channel_coroutine(self, *, connect_str: str, funding_tx: Partial funding_sat=funding_sat, push_msat=push_sat * 1000, temp_channel_id=os.urandom(32)) - self.network.trigger_callback('channels_updated', self.wallet) + util.trigger_callback('channels_updated', self.wallet) self.wallet.add_transaction(funding_tx) # save tx as local into the wallet self.wallet.set_label(funding_tx.txid(), _('Open channel')) if funding_tx.is_complete(): @@ -804,10 +804,10 @@ async def _pay(self, invoice, amount_sat=None, attempts=1) -> bool: # note: path-finding runs in a separate thread so that we don't block the asyncio loop # graph updates might occur during the computation self.set_invoice_status(key, PR_ROUTING) - self.network.trigger_callback('invoice_status', key) + util.trigger_callback('invoice_status', key) route = await run_in_thread(self._create_route_from_invoice, lnaddr) self.set_invoice_status(key, PR_INFLIGHT) - self.network.trigger_callback('invoice_status', key) + util.trigger_callback('invoice_status', key) payment_attempt_log = await self._pay_to_route(route, lnaddr) except Exception as e: log.append(PaymentAttemptLog(success=False, exception=e)) @@ -820,11 +820,11 @@ async def _pay(self, invoice, amount_sat=None, attempts=1) -> bool: break else: reason = _('Failed after {} attempts').format(attempts) - self.network.trigger_callback('invoice_status', key) + util.trigger_callback('invoice_status', key) if success: - self.network.trigger_callback('payment_succeeded', key) + util.trigger_callback('payment_succeeded', key) else: - self.network.trigger_callback('payment_failed', key, reason) + util.trigger_callback('payment_failed', key, reason) return success async def _pay_to_route(self, route: LNPaymentRoute, lnaddr: LnAddr) -> PaymentAttemptLog: @@ -840,7 +840,7 @@ async def _pay_to_route(self, route: LNPaymentRoute, lnaddr: LnAddr) -> PaymentA payment_hash=lnaddr.paymenthash, min_final_cltv_expiry=lnaddr.get_min_final_cltv_expiry(), payment_secret=lnaddr.payment_secret) - self.network.trigger_callback('htlc_added', htlc, lnaddr, SENT) + util.trigger_callback('htlc_added', htlc, lnaddr, SENT) payment_attempt = await self.await_payment(lnaddr.paymenthash) if payment_attempt.success: failure_log = None @@ -1139,9 +1139,9 @@ def payment_failed(self, chan, payment_hash: bytes, payment_attempt: BarePayment f.set_result(payment_attempt) else: chan.logger.info('received unexpected payment_failed, probably from previous session') - self.network.trigger_callback('invoice_status', key) - self.network.trigger_callback('payment_failed', key, '') - self.network.trigger_callback('ln_payment_failed', payment_hash, chan.channel_id) + util.trigger_callback('invoice_status', key) + util.trigger_callback('payment_failed', key, '') + util.trigger_callback('ln_payment_failed', payment_hash, chan.channel_id) def payment_sent(self, chan, payment_hash: bytes): self.set_payment_status(payment_hash, PR_PAID) @@ -1155,14 +1155,14 @@ def payment_sent(self, chan, payment_hash: bytes): f.set_result(payment_attempt) else: chan.logger.info('received unexpected payment_sent, probably from previous session') - self.network.trigger_callback('invoice_status', key) - self.network.trigger_callback('payment_succeeded', key) - self.network.trigger_callback('ln_payment_completed', payment_hash, chan.channel_id) + util.trigger_callback('invoice_status', key) + util.trigger_callback('payment_succeeded', key) + util.trigger_callback('ln_payment_completed', payment_hash, chan.channel_id) def payment_received(self, chan, payment_hash: bytes): self.set_payment_status(payment_hash, PR_PAID) - self.network.trigger_callback('request_status', payment_hash.hex(), PR_PAID) - self.network.trigger_callback('ln_payment_completed', payment_hash, chan.channel_id) + util.trigger_callback('request_status', payment_hash.hex(), PR_PAID) + util.trigger_callback('ln_payment_completed', payment_hash, chan.channel_id) async def _calc_routing_hints_for_invoice(self, amount_sat): """calculate routing hints (BOLT-11 'r' field)""" @@ -1251,8 +1251,8 @@ def remove_channel(self, chan_id): self.channels.pop(chan_id) self.db.get('channels').pop(chan_id.hex()) - self.network.trigger_callback('channels_updated', self.wallet) - self.network.trigger_callback('wallet_updated', self.wallet) + util.trigger_callback('channels_updated', self.wallet) + util.trigger_callback('wallet_updated', self.wallet) @ignore_exceptions @log_exceptions @@ -1355,7 +1355,7 @@ def __init__(self, wallet: 'Abstract_Wallet'): self.channel_backups[bfh(channel_id)] = ChannelBackup(cb, sweep_address=self.sweep_address, lnworker=self) def channel_state_changed(self, chan): - self.network.trigger_callback('channel', chan) + util.trigger_callback('channel', chan) def peer_closed(self, chan): pass @@ -1389,7 +1389,7 @@ def import_channel_backup(self, encrypted): d[channel_id] = cb_storage self.channel_backups[bfh(channel_id)] = cb = ChannelBackup(cb_storage, sweep_address=self.sweep_address, lnworker=self) self.wallet.save_db() - self.network.trigger_callback('channels_updated', self.wallet) + util.trigger_callback('channels_updated', self.wallet) self.lnwatcher.add_channel(cb.funding_outpoint.to_str(), cb.get_funding_address()) def remove_channel_backup(self, channel_id): @@ -1399,7 +1399,7 @@ def remove_channel_backup(self, channel_id): d.pop(channel_id.hex()) self.channel_backups.pop(channel_id) self.wallet.save_db() - self.network.trigger_callback('channels_updated', self.wallet) + util.trigger_callback('channels_updated', self.wallet) @log_exceptions async def request_force_close(self, channel_id): diff --git a/electrum/network.py b/electrum/network.py index e3d9ed5531..804c91a0c7 100644 --- a/electrum/network.py +++ b/electrum/network.py @@ -278,7 +278,6 @@ def __init__(self, config: SimpleConfig, *, daemon: 'Daemon' = None): # locks self.restart_lock = asyncio.Lock() self.bhi_lock = asyncio.Lock() - self.callback_lock = threading.Lock() self.recent_servers_lock = threading.RLock() # <- re-entrant self.interfaces_lock = threading.Lock() # for mutating/iterating self.interfaces @@ -288,8 +287,6 @@ def __init__(self, config: SimpleConfig, *, daemon: 'Daemon' = None): self.banner = '' self.donation_address = '' self.relay_fee = None # type: Optional[int] - # callbacks set by the GUI - self.callbacks = defaultdict(list) # note: needs self.callback_lock dir_path = os.path.join(self.config.path, 'certs') util.make_dir(dir_path) @@ -332,7 +329,7 @@ def maybe_init_lightning(self): from . import channel_db self.channel_db = channel_db.ChannelDB(self) self.path_finder = lnrouter.LNPathFinder(self.channel_db) - self.lngossip = lnworker.LNGossip(self) + self.lngossip = lnworker.LNGossip() self.lngossip.start_network(self) def run_from_another_thread(self, coro, *, timeout=None): @@ -350,27 +347,6 @@ def func_wrapper(self, *args, **kwargs): return func(self, *args, **kwargs) return func_wrapper - def register_callback(self, callback, events): - with self.callback_lock: - for event in events: - self.callbacks[event].append(callback) - - def unregister_callback(self, callback): - with self.callback_lock: - for callbacks in self.callbacks.values(): - if callback in callbacks: - callbacks.remove(callback) - - def trigger_callback(self, event, *args): - with self.callback_lock: - callbacks = self.callbacks[event][:] - for callback in callbacks: - # FIXME: if callback throws, we will lose the traceback - if asyncio.iscoroutinefunction(callback): - asyncio.run_coroutine_threadsafe(callback(event, *args), self.asyncio_loop) - else: - self.asyncio_loop.call_soon_threadsafe(callback, event, *args) - def _read_recent_servers(self): if not self.config.path: return [] @@ -481,9 +457,9 @@ def get_status_value(self, key): def notify(self, key): if key in ['status', 'updated']: - self.trigger_callback(key) + util.trigger_callback(key) else: - self.trigger_callback(key, self.get_status_value(key)) + util.trigger_callback(key, self.get_status_value(key)) def get_parameters(self) -> NetworkParameters: host, port, protocol = deserialize_server(self.default_server) @@ -574,7 +550,7 @@ def _set_proxy(self, proxy: Optional[dict]): self.proxy = proxy dns_hacks.configure_dns_depending_on_proxy(bool(proxy)) self.logger.info(f'setting proxy {proxy}') - self.trigger_callback('proxy_set', self.proxy) + util.trigger_callback('proxy_set', self.proxy) @log_exceptions async def set_parameters(self, net_params: NetworkParameters): @@ -700,12 +676,13 @@ async def switch_to_interface(self, server: str): blockchain_updated = i.blockchain != self.blockchain() self.interface = i await i.taskgroup.spawn(self._request_server_info(i)) - self.trigger_callback('default_server_changed') + util.trigger_callback('default_server_changed') self.default_server_changed_event.set() self.default_server_changed_event.clear() self._set_status('connected') - self.trigger_callback('network_updated') - if blockchain_updated: self.trigger_callback('blockchain_updated') + util.trigger_callback('network_updated') + if blockchain_updated: + util.trigger_callback('blockchain_updated') async def _close_interface(self, interface: Interface): if interface: @@ -734,7 +711,7 @@ async def connection_down(self, interface: Interface): if server == self.default_server: self._set_status('disconnected') await self._close_interface(interface) - self.trigger_callback('network_updated') + util.trigger_callback('network_updated') def get_network_timeout_seconds(self, request_type=NetworkTimeout.Generic) -> int: if self.oneserver and not self.auto_connect: @@ -767,7 +744,7 @@ async def _run_new_interface(self, server): await self.switch_to_interface(server) self._add_recent_server(server) - self.trigger_callback('network_updated') + util.trigger_callback('network_updated') def check_interface_against_healthy_spread_of_connected_servers(self, iface_to_check) -> bool: # main interface is exempt. this makes switching servers easier @@ -1152,7 +1129,7 @@ async def main(): self.logger.info("taskgroup stopped.") asyncio.run_coroutine_threadsafe(main(), self.asyncio_loop) - self.trigger_callback('network_updated') + util.trigger_callback('network_updated') def start(self, jobs: Iterable = None): """Schedule starting the network, along with the given job co-routines. @@ -1176,7 +1153,7 @@ async def _stop(self, full_shutdown=False): self.connecting.clear() self.server_queue = None if not full_shutdown: - self.trigger_callback('network_updated') + util.trigger_callback('network_updated') def stop(self): assert self._loop_thread != threading.current_thread(), 'must not be called from network thread' diff --git a/electrum/synchronizer.py b/electrum/synchronizer.py index fe51b29008..6c47a9baf0 100644 --- a/electrum/synchronizer.py +++ b/electrum/synchronizer.py @@ -30,6 +30,7 @@ from aiorpcx import TaskGroup, run_in_thread, RPCError +from . import util from .transaction import Transaction, PartialTransaction from .util import bh2u, make_aiohttp_session, NetworkJobOnDefaultServer from .bitcoin import address_to_scripthash, is_address @@ -227,7 +228,7 @@ async def _get_transaction(self, tx_hash, *, allow_server_not_finding_tx=False): self.wallet.receive_tx_callback(tx_hash, tx, tx_height) self.logger.info(f"received tx {tx_hash} height: {tx_height} bytes: {len(raw_tx)}") # callbacks - self.wallet.network.trigger_callback('new_transaction', self.wallet, tx) + util.trigger_callback('new_transaction', self.wallet, tx) async def main(self): self.wallet.set_up_to_date(False) @@ -252,7 +253,7 @@ async def main(self): if up_to_date: self._reset_request_counters() self.wallet.set_up_to_date(up_to_date) - self.wallet.network.trigger_callback('wallet_updated', self.wallet) + util.trigger_callback('wallet_updated', self.wallet) class Notifier(SynchronizerBase): diff --git a/electrum/util.py b/electrum/util.py index cd08c1e992..1c47ec8c16 100644 --- a/electrum/util.py +++ b/electrum/util.py @@ -1130,7 +1130,7 @@ def __init__(self, network: 'Network'): self._restart_lock = asyncio.Lock() self._reset() asyncio.run_coroutine_threadsafe(self._restart(), network.asyncio_loop) - network.register_callback(self._restart, ['default_server_changed']) + register_callback(self._restart, ['default_server_changed']) def _reset(self): """Initialise fields. Called every time the underlying @@ -1304,3 +1304,41 @@ def randrange(bound: int) -> int: """Return a random integer k such that 1 <= k < bound, uniformly distributed across that range.""" return ecdsa.util.randrange(bound) + + +class CallbackManager: + # callbacks set by the GUI + def __init__(self): + self.callback_lock = threading.Lock() + self.callbacks = defaultdict(list) # note: needs self.callback_lock + self.asyncio_loop = None + + def register_callback(self, callback, events): + with self.callback_lock: + for event in events: + self.callbacks[event].append(callback) + + def unregister_callback(self, callback): + with self.callback_lock: + for callbacks in self.callbacks.values(): + if callback in callbacks: + callbacks.remove(callback) + + def trigger_callback(self, event, *args): + if self.asyncio_loop is None: + self.asyncio_loop = asyncio.get_event_loop() + assert self.asyncio_loop.is_running(), "event loop not running" + with self.callback_lock: + callbacks = self.callbacks[event][:] + for callback in callbacks: + # FIXME: if callback throws, we will lose the traceback + if asyncio.iscoroutinefunction(callback): + asyncio.run_coroutine_threadsafe(callback(event, *args), self.asyncio_loop) + else: + self.asyncio_loop.call_soon_threadsafe(callback, event, *args) + + +callback_mgr = CallbackManager() +trigger_callback = callback_mgr.trigger_callback +register_callback = callback_mgr.register_callback +unregister_callback = callback_mgr.unregister_callback From ef2ff11926343016004160215348c54ebe4ffd1e Mon Sep 17 00:00:00 2001 From: ThomasV Date: Tue, 14 Apr 2020 18:35:50 +0200 Subject: [PATCH 068/117] fix tests (follow-up prev commit) --- electrum/tests/test_lnpeer.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/electrum/tests/test_lnpeer.py b/electrum/tests/test_lnpeer.py index 320bd6fa87..beb4f0e402 100644 --- a/electrum/tests/test_lnpeer.py +++ b/electrum/tests/test_lnpeer.py @@ -64,10 +64,6 @@ def __init__(self, tx_queue): def callback_lock(self): return noop_lock() - register_callback = Network.register_callback - unregister_callback = Network.unregister_callback - trigger_callback = Network.trigger_callback - def get_local_height(self): return 0 From cf1f2ba4dca51f15f485211a530022165a89c4c4 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Tue, 14 Apr 2020 16:56:17 +0200 Subject: [PATCH 069/117] network: replace "server" strings with ServerAddr objects --- electrum/daemon.py | 2 + electrum/exchange_rate.py | 2 +- electrum/gui/qt/network_dialog.py | 40 +++++----- electrum/gui/text.py | 24 ++++-- electrum/interface.py | 83 +++++++++++++++----- electrum/network.py | 124 ++++++++++++++++-------------- 6 files changed, 172 insertions(+), 103 deletions(-) diff --git a/electrum/daemon.py b/electrum/daemon.py index 41e9251606..93b4baeb92 100644 --- a/electrum/daemon.py +++ b/electrum/daemon.py @@ -270,6 +270,8 @@ class AuthenticationCredentialsInvalid(AuthenticationError): class Daemon(Logger): + network: Optional[Network] + @profiler def __init__(self, config: SimpleConfig, fd=None, *, listen_jsonrpc=True): Logger.__init__(self) diff --git a/electrum/exchange_rate.py b/electrum/exchange_rate.py index 49bad1c8b0..168c69dc4d 100644 --- a/electrum/exchange_rate.py +++ b/electrum/exchange_rate.py @@ -453,7 +453,7 @@ def get_exchanges_by_ccy(history=True): class FxThread(ThreadJob): - def __init__(self, config: SimpleConfig, network: Network): + def __init__(self, config: SimpleConfig, network: Optional[Network]): ThreadJob.__init__(self) self.config = config self.network = network diff --git a/electrum/gui/qt/network_dialog.py b/electrum/gui/qt/network_dialog.py index b452295e39..bbf2df91f8 100644 --- a/electrum/gui/qt/network_dialog.py +++ b/electrum/gui/qt/network_dialog.py @@ -36,7 +36,7 @@ from electrum.i18n import _ from electrum import constants, blockchain, util -from electrum.interface import serialize_server, deserialize_server +from electrum.interface import ServerAddr from electrum.network import Network from electrum.logging import get_logger @@ -72,10 +72,13 @@ def on_update(self): class NodesListWidget(QTreeWidget): + SERVER_ADDR_ROLE = Qt.UserRole + 100 + CHAIN_ID_ROLE = Qt.UserRole + 101 + IS_SERVER_ROLE = Qt.UserRole + 102 def __init__(self, parent): QTreeWidget.__init__(self) - self.parent = parent + self.parent = parent # type: NetworkChoiceLayout self.setHeaderLabels([_('Connected node'), _('Height')]) self.setContextMenuPolicy(Qt.CustomContextMenu) self.customContextMenuRequested.connect(self.create_menu) @@ -84,13 +87,13 @@ def create_menu(self, position): item = self.currentItem() if not item: return - is_server = not bool(item.data(0, Qt.UserRole)) + is_server = bool(item.data(0, self.IS_SERVER_ROLE)) menu = QMenu() if is_server: - server = item.data(1, Qt.UserRole) + server = item.data(0, self.SERVER_ADDR_ROLE) # type: ServerAddr menu.addAction(_("Use as server"), lambda: self.parent.follow_server(server)) else: - chain_id = item.data(1, Qt.UserRole) + chain_id = item.data(0, self.CHAIN_ID_ROLE) menu.addAction(_("Follow this branch"), lambda: self.parent.follow_branch(chain_id)) menu.exec_(self.viewport().mapToGlobal(position)) @@ -117,15 +120,15 @@ def update(self, network: Network): name = b.get_name() if n_chains > 1: x = QTreeWidgetItem([name + '@%d'%b.get_max_forkpoint(), '%d'%b.height()]) - x.setData(0, Qt.UserRole, 1) - x.setData(1, Qt.UserRole, b.get_id()) + x.setData(0, self.IS_SERVER_ROLE, 0) + x.setData(0, self.CHAIN_ID_ROLE, b.get_id()) else: x = self for i in interfaces: star = ' *' if i == network.interface else '' item = QTreeWidgetItem([i.host + star, '%d'%i.tip]) - item.setData(0, Qt.UserRole, 0) - item.setData(1, Qt.UserRole, i.server) + item.setData(0, self.IS_SERVER_ROLE, 1) + item.setData(0, self.SERVER_ADDR_ROLE, i.server) x.addChild(item) if n_chains > 1: self.addTopLevelItem(x) @@ -144,11 +147,11 @@ class Columns(IntEnum): HOST = 0 PORT = 1 - SERVER_STR_ROLE = Qt.UserRole + 100 + SERVER_ADDR_ROLE = Qt.UserRole + 100 def __init__(self, parent): QTreeWidget.__init__(self) - self.parent = parent + self.parent = parent # type: NetworkChoiceLayout self.setHeaderLabels([_('Host'), _('Port')]) self.setContextMenuPolicy(Qt.CustomContextMenu) self.customContextMenuRequested.connect(self.create_menu) @@ -158,14 +161,13 @@ def create_menu(self, position): if not item: return menu = QMenu() - server = item.data(self.Columns.HOST, self.SERVER_STR_ROLE) + server = item.data(self.Columns.HOST, self.SERVER_ADDR_ROLE) menu.addAction(_("Use as server"), lambda: self.set_server(server)) menu.exec_(self.viewport().mapToGlobal(position)) - def set_server(self, s): - host, port, protocol = deserialize_server(s) - self.parent.server_host.setText(host) - self.parent.server_port.setText(port) + def set_server(self, server: ServerAddr): + self.parent.server_host.setText(server.host) + self.parent.server_port.setText(str(server.port)) self.parent.set_server() def keyPressEvent(self, event): @@ -188,8 +190,8 @@ def update(self, servers, protocol, use_tor): port = d.get(protocol) if port: x = QTreeWidgetItem([_host, port]) - server = serialize_server(_host, port, protocol) - x.setData(self.Columns.HOST, self.SERVER_STR_ROLE, server) + server = ServerAddr(_host, port, protocol=protocol) + x.setData(self.Columns.HOST, self.SERVER_ADDR_ROLE, server) self.addTopLevelItem(x) h = self.header() @@ -431,7 +433,7 @@ def follow_branch(self, chain_id): self.network.run_from_another_thread(self.network.follow_chain_given_id(chain_id)) self.update() - def follow_server(self, server): + def follow_server(self, server: ServerAddr): self.network.run_from_another_thread(self.network.follow_chain_given_server(server)) self.update() diff --git a/electrum/gui/text.py b/electrum/gui/text.py index 8e075b068c..71c6a49dd6 100644 --- a/electrum/gui/text.py +++ b/electrum/gui/text.py @@ -6,6 +6,7 @@ from decimal import Decimal import getpass import logging +from typing import TYPE_CHECKING import electrum from electrum import util @@ -15,15 +16,21 @@ from electrum.wallet import Wallet from electrum.storage import WalletStorage from electrum.network import NetworkParameters, TxBroadcastError, BestEffortRequestFailed -from electrum.interface import deserialize_server +from electrum.interface import ServerAddr from electrum.logging import console_stderr_handler +if TYPE_CHECKING: + from electrum.daemon import Daemon + from electrum.simple_config import SimpleConfig + from electrum.plugin import Plugins + + _ = lambda x:x # i18n class ElectrumGui: - def __init__(self, config, daemon, plugins): + def __init__(self, config: 'SimpleConfig', daemon: 'Daemon', plugins: 'Plugins'): self.config = config self.network = daemon.network @@ -404,21 +411,24 @@ def network_dialog(self): net_params = self.network.get_parameters() host, port, protocol = net_params.host, net_params.port, net_params.protocol proxy_config, auto_connect = net_params.proxy, net_params.auto_connect - srv = 'auto-connect' if auto_connect else self.network.default_server + srv = 'auto-connect' if auto_connect else str(self.network.default_server) out = self.run_dialog('Network', [ {'label':'server', 'type':'str', 'value':srv}, {'label':'proxy', 'type':'str', 'value':self.config.get('proxy', '')}, ], buttons = 1) if out: if out.get('server'): - server = out.get('server') - auto_connect = server == 'auto-connect' + server_str = out.get('server') + auto_connect = server_str == 'auto-connect' if not auto_connect: try: - host, port, protocol = deserialize_server(server) + server_addr = ServerAddr.from_str(server_str) except Exception: - self.show_message("Error:" + server + "\nIn doubt, type \"auto-connect\"") + self.show_message("Error:" + server_str + "\nIn doubt, type \"auto-connect\"") return False + host = server_addr.host + port = str(server_addr.port) + protocol = server_addr.protocol if out.get('server') or out.get('proxy'): proxy = electrum.network.deserialize_proxy(out.get('proxy')) if out.get('proxy') else proxy_config net_params = NetworkParameters(host, port, protocol, proxy, auto_connect) diff --git a/electrum/interface.py b/electrum/interface.py index bafba8ceb9..86fa5e0c8d 100644 --- a/electrum/interface.py +++ b/electrum/interface.py @@ -29,7 +29,7 @@ import traceback import asyncio import socket -from typing import Tuple, Union, List, TYPE_CHECKING, Optional, Set +from typing import Tuple, Union, List, TYPE_CHECKING, Optional, Set, NamedTuple from collections import defaultdict from ipaddress import IPv4Network, IPv6Network, ip_address, IPv6Address import itertools @@ -198,22 +198,57 @@ async def create_connection(self): raise ConnectError(e) from e -def deserialize_server(server_str: str) -> Tuple[str, str, str]: - # host might be IPv6 address, hence do rsplit: - host, port, protocol = str(server_str).rsplit(':', 2) - if not host: - raise ValueError('host must not be empty') - if host[0] == '[' and host[-1] == ']': # IPv6 - host = host[1:-1] - if protocol not in ('s', 't'): - raise ValueError('invalid network protocol: {}'.format(protocol)) - net_addr = NetAddress(host, port) # this validates host and port - host = str(net_addr.host) # canonical form (if e.g. IPv6 address) - return host, port, protocol +class ServerAddr: + def __init__(self, host: str, port: Union[int, str], *, protocol: str = None): + assert isinstance(host, str), repr(host) + if protocol is None: + protocol = 's' + if not host: + raise ValueError('host must not be empty') + if host[0] == '[' and host[-1] == ']': # IPv6 + host = host[1:-1] + try: + net_addr = NetAddress(host, port) # this validates host and port + except Exception as e: + raise ValueError(f"cannot construct ServerAddr: invalid host or port (host={host}, port={port})") from e + if protocol not in ('s', 't'): + raise ValueError(f"invalid network protocol: {protocol}") + self.host = str(net_addr.host) # canonical form (if e.g. IPv6 address) + self.port = int(net_addr.port) + self.protocol = protocol + self._net_addr_str = str(net_addr) + + @classmethod + def from_str(cls, s: str) -> 'ServerAddr': + # host might be IPv6 address, hence do rsplit: + host, port, protocol = str(s).rsplit(':', 2) + return ServerAddr(host=host, port=port, protocol=protocol) -def serialize_server(host: str, port: Union[str, int], protocol: str) -> str: - return str(':'.join([host, str(port), protocol])) + def __str__(self): + return '{}:{}'.format(self.net_addr_str(), self.protocol) + + def to_json(self) -> str: + return str(self) + + def __repr__(self): + return f'' + + def net_addr_str(self) -> str: + return self._net_addr_str + + def __eq__(self, other): + if not isinstance(other, ServerAddr): + return False + return (self.host == other.host + and self.port == other.port + and self.protocol == other.protocol) + + def __ne__(self, other): + return not (self == other) + + def __hash__(self): + return hash((self.host, self.port, self.protocol)) def _get_cert_path_for_host(*, config: 'SimpleConfig', host: str) -> str: @@ -232,12 +267,10 @@ class Interface(Logger): LOGGING_SHORTCUT = 'i' - def __init__(self, network: 'Network', server: str, proxy: Optional[dict]): + def __init__(self, *, network: 'Network', server: ServerAddr, proxy: Optional[dict]): self.ready = asyncio.Future() self.got_disconnected = asyncio.Future() self.server = server - self.host, self.port, self.protocol = deserialize_server(self.server) - self.port = int(self.port) Logger.__init__(self) assert network.config.path self.cert_path = _get_cert_path_for_host(config=network.config, host=self.host) @@ -259,8 +292,20 @@ def __init__(self, network: 'Network', server: str, proxy: Optional[dict]): self.network.taskgroup.spawn(self.run()), self.network.asyncio_loop) self.taskgroup = SilentTaskGroup() + @property + def host(self): + return self.server.host + + @property + def port(self): + return self.server.port + + @property + def protocol(self): + return self.server.protocol + def diagnostic_name(self): - return str(NetAddress(self.host, self.port)) + return self.server.net_addr_str() def __str__(self): return f"" diff --git a/electrum/network.py b/electrum/network.py index 804c91a0c7..4ea4a88d87 100644 --- a/electrum/network.py +++ b/electrum/network.py @@ -32,7 +32,7 @@ import json import sys import asyncio -from typing import NamedTuple, Optional, Sequence, List, Dict, Tuple, TYPE_CHECKING, Iterable +from typing import NamedTuple, Optional, Sequence, List, Dict, Tuple, TYPE_CHECKING, Iterable, Set import traceback import concurrent from concurrent import futures @@ -44,7 +44,7 @@ from . import util from .util import (log_exceptions, ignore_exceptions, bfh, SilentTaskGroup, make_aiohttp_session, send_exception_to_crash_reporter, - is_hash256_str, is_non_negative_integer) + is_hash256_str, is_non_negative_integer, MyEncoder) from .bitcoin import COIN from . import constants @@ -53,9 +53,9 @@ from . import dns_hacks from .transaction import Transaction from .blockchain import Blockchain, HEADER_SIZE -from .interface import (Interface, serialize_server, deserialize_server, +from .interface import (Interface, RequestTimedOut, NetworkTimeout, BUCKET_NAME_OF_ONION_SERVERS, - NetworkException, RequestCorrupted) + NetworkException, RequestCorrupted, ServerAddr) from .version import PROTOCOL_VERSION from .simple_config import SimpleConfig from .i18n import _ @@ -117,18 +117,18 @@ def filter_noonion(servers): return {k: v for k, v in servers.items() if not k.endswith('.onion')} -def filter_protocol(hostmap, protocol='s'): - '''Filters the hostmap for those implementing protocol. - The result is a list in serialized form.''' +def filter_protocol(hostmap, protocol='s') -> Sequence[ServerAddr]: + """Filters the hostmap for those implementing protocol.""" eligible = [] for host, portmap in hostmap.items(): port = portmap.get(protocol) if port: - eligible.append(serialize_server(host, port, protocol)) + eligible.append(ServerAddr(host, port, protocol=protocol)) return eligible -def pick_random_server(hostmap=None, protocol='s', exclude_set=None): +def pick_random_server(hostmap=None, *, protocol='s', + exclude_set: Set[ServerAddr] = None) -> Optional[ServerAddr]: if hostmap is None: hostmap = constants.net.DEFAULT_SERVERS if exclude_set is None: @@ -240,6 +240,14 @@ class Network(Logger): LOGGING_SHORTCUT = 'n' + taskgroup: Optional[TaskGroup] + interface: Optional[Interface] + interfaces: Dict[ServerAddr, Interface] + connecting: Set[ServerAddr] + server_queue: 'Optional[queue.Queue[ServerAddr]]' + disconnected_servers: Set[ServerAddr] + default_server: ServerAddr + def __init__(self, config: SimpleConfig, *, daemon: 'Daemon' = None): global _INSTANCE assert _INSTANCE is None, "Network is a singleton!" @@ -266,14 +274,15 @@ def __init__(self, config: SimpleConfig, *, daemon: 'Daemon' = None): # Sanitize default server if self.default_server: try: - deserialize_server(self.default_server) + self.default_server = ServerAddr.from_str(self.default_server) except: self.logger.warning('failed to parse server-string; falling back to localhost.') - self.default_server = "localhost:50002:s" - if not self.default_server: + self.default_server = ServerAddr.from_str("localhost:50002:s") + else: self.default_server = pick_random_server() + assert isinstance(self.default_server, ServerAddr), f"invalid type for default_server: {self.default_server!r}" - self.taskgroup = None # type: TaskGroup + self.taskgroup = None # locks self.restart_lock = asyncio.Lock() @@ -295,10 +304,10 @@ def __init__(self, config: SimpleConfig, *, daemon: 'Daemon' = None): self.server_retry_time = time.time() self.nodes_retry_time = time.time() # the main server we are currently communicating with - self.interface = None # type: Optional[Interface] + self.interface = None self.default_server_changed_event = asyncio.Event() # set of servers we have an ongoing connection with - self.interfaces = {} # type: Dict[str, Interface] + self.interfaces = {} self.auto_connect = self.config.get('auto_connect', True) self.connecting = set() self.server_queue = None @@ -347,14 +356,15 @@ def func_wrapper(self, *args, **kwargs): return func(self, *args, **kwargs) return func_wrapper - def _read_recent_servers(self): + def _read_recent_servers(self) -> List[ServerAddr]: if not self.config.path: return [] path = os.path.join(self.config.path, "recent_servers") try: with open(path, "r", encoding='utf-8') as f: data = f.read() - return json.loads(data) + servers_list = json.loads(data) + return [ServerAddr.from_str(s) for s in servers_list] except: return [] @@ -363,7 +373,7 @@ def _save_recent_servers(self): if not self.config.path: return path = os.path.join(self.config.path, "recent_servers") - s = json.dumps(self.recent_servers, indent=4, sort_keys=True) + s = json.dumps(self.recent_servers, indent=4, sort_keys=True, cls=MyEncoder) try: with open(path, "w", encoding='utf-8') as f: f.write(s) @@ -462,10 +472,10 @@ def notify(self, key): util.trigger_callback(key, self.get_status_value(key)) def get_parameters(self) -> NetworkParameters: - host, port, protocol = deserialize_server(self.default_server) - return NetworkParameters(host=host, - port=port, - protocol=protocol, + server = self.default_server + return NetworkParameters(host=server.host, + port=str(server.port), + protocol=server.protocol, proxy=self.proxy, auto_connect=self.auto_connect, oneserver=self.oneserver) @@ -474,7 +484,7 @@ def get_donation_address(self): if self.is_connected(): return self.donation_address - def get_interfaces(self) -> List[str]: + def get_interfaces(self) -> List[ServerAddr]: """The list of servers for the connected interfaces.""" with self.interfaces_lock: return list(self.interfaces) @@ -516,21 +526,18 @@ def get_servers(self): # hardcoded servers out.update(constants.net.DEFAULT_SERVERS) # add recent servers - for s in self.recent_servers: - try: - host, port, protocol = deserialize_server(s) - except: - continue - if host in out: - out[host].update({protocol: port}) + for server in self.recent_servers: + port = str(server.port) + if server.host in out: + out[server.host].update({server.protocol: port}) else: - out[host] = {protocol: port} + out[server.host] = {server.protocol: port} # potentially filter out some if self.config.get('noonion'): out = filter_noonion(out) return out - def _start_interface(self, server: str): + def _start_interface(self, server: ServerAddr): if server not in self.interfaces and server not in self.connecting: if server == self.default_server: self.logger.info(f"connecting to {server} as new interface") @@ -538,10 +545,10 @@ def _start_interface(self, server: str): self.connecting.add(server) self.server_queue.put(server) - def _start_random_interface(self): + def _start_random_interface(self) -> Optional[ServerAddr]: with self.interfaces_lock: exclude_set = self.disconnected_servers | set(self.interfaces) | self.connecting - server = pick_random_server(self.get_servers(), self.protocol, exclude_set) + server = pick_random_server(self.get_servers(), protocol=self.protocol, exclude_set=exclude_set) if server: self._start_interface(server) return server @@ -557,10 +564,9 @@ async def set_parameters(self, net_params: NetworkParameters): proxy = net_params.proxy proxy_str = serialize_proxy(proxy) host, port, protocol = net_params.host, net_params.port, net_params.protocol - server_str = serialize_server(host, port, protocol) # sanitize parameters try: - deserialize_server(serialize_server(host, port, protocol)) + server = ServerAddr(host, port, protocol=protocol) if proxy: proxy_modes.index(proxy['mode']) + 1 int(proxy['port']) @@ -569,9 +575,9 @@ async def set_parameters(self, net_params: NetworkParameters): self.config.set_key('auto_connect', net_params.auto_connect, False) self.config.set_key('oneserver', net_params.oneserver, False) self.config.set_key('proxy', proxy_str, False) - self.config.set_key('server', server_str, True) + self.config.set_key('server', str(server), True) # abort if changes were not allowed by config - if self.config.get('server') != server_str \ + if self.config.get('server') != str(server) \ or self.config.get('proxy') != proxy_str \ or self.config.get('oneserver') != net_params.oneserver: return @@ -581,10 +587,10 @@ async def set_parameters(self, net_params: NetworkParameters): if self.proxy != proxy or self.protocol != protocol or self.oneserver != net_params.oneserver: # Restart the network defaulting to the given server await self._stop() - self.default_server = server_str + self.default_server = server await self._start() - elif self.default_server != server_str: - await self.switch_to_interface(server_str) + elif self.default_server != server: + await self.switch_to_interface(server) else: await self.switch_lagging_interface() @@ -646,7 +652,7 @@ async def switch_unwanted_fork_interface(self): # FIXME switch to best available? self.logger.info("tried to switch to best chain but no interfaces are on it") - async def switch_to_interface(self, server: str): + async def switch_to_interface(self, server: ServerAddr): """Switch to server as our main interface. If no connection exists, queue interface to be started. The actual switch will happen when the interface becomes ready. @@ -722,8 +728,8 @@ def get_network_timeout_seconds(self, request_type=NetworkTimeout.Generic) -> in @ignore_exceptions # do not kill main_taskgroup @log_exceptions - async def _run_new_interface(self, server): - interface = Interface(self, server, self.proxy) + async def _run_new_interface(self, server: ServerAddr): + interface = Interface(network=self, server=server, proxy=self.proxy) # note: using longer timeouts here as DNS can sometimes be slow! timeout = self.get_network_timeout_seconds(NetworkTimeout.Generic) try: @@ -1070,23 +1076,26 @@ async def follow_chain_given_id(self, chain_id: str) -> None: with self.interfaces_lock: interfaces = list(self.interfaces.values()) interfaces_on_selected_chain = list(filter(lambda iface: iface.blockchain == bc, interfaces)) if len(interfaces_on_selected_chain) == 0: return - chosen_iface = random.choice(interfaces_on_selected_chain) + chosen_iface = random.choice(interfaces_on_selected_chain) # type: Interface # switch to server (and save to config) net_params = self.get_parameters() - host, port, protocol = deserialize_server(chosen_iface.server) - net_params = net_params._replace(host=host, port=port, protocol=protocol) + server = chosen_iface.server + net_params = net_params._replace(host=server.host, + port=str(server.port), + protocol=server.protocol) await self.set_parameters(net_params) - async def follow_chain_given_server(self, server_str: str) -> None: + async def follow_chain_given_server(self, server: ServerAddr) -> None: # note that server_str should correspond to a connected interface - iface = self.interfaces.get(server_str) + iface = self.interfaces.get(server) if iface is None: return self._set_preferred_chain(iface.blockchain) # switch to server (and save to config) net_params = self.get_parameters() - host, port, protocol = deserialize_server(server_str) - net_params = net_params._replace(host=host, port=port, protocol=protocol) + net_params = net_params._replace(host=server.host, + port=str(server.port), + protocol=server.protocol) await self.set_parameters(net_params) def get_local_height(self): @@ -1107,7 +1116,7 @@ async def _start(self): assert not self.connecting and not self.server_queue self.logger.info('starting network') self.disconnected_servers = set([]) - self.protocol = deserialize_server(self.default_server)[2] + self.protocol = self.default_server.protocol self.server_queue = queue.Queue() self._set_proxy(deserialize_proxy(self.config.get('proxy'))) self._set_oneserver(self.config.get('oneserver', False)) @@ -1147,9 +1156,9 @@ async def _stop(self, full_shutdown=False): await asyncio.wait_for(self.taskgroup.cancel_remaining(), timeout=2) except (asyncio.TimeoutError, asyncio.CancelledError) as e: self.logger.info(f"exc during main_taskgroup cancellation: {repr(e)}") - self.taskgroup = None # type: TaskGroup - self.interface = None # type: Interface - self.interfaces = {} # type: Dict[str, Interface] + self.taskgroup = None + self.interface = None + self.interfaces = {} self.connecting.clear() self.server_queue = None if not full_shutdown: @@ -1268,8 +1277,8 @@ async def get_peers(self): async def send_multiple_requests(self, servers: List[str], method: str, params: Sequence): responses = dict() - async def get_response(server): - interface = Interface(self, server, self.proxy) + async def get_response(server: ServerAddr): + interface = Interface(network=self, server=server, proxy=self.proxy) timeout = self.get_network_timeout_seconds(NetworkTimeout.Urgent) try: await asyncio.wait_for(interface.ready, timeout) @@ -1283,5 +1292,6 @@ async def get_response(server): responses[interface.server] = res async with TaskGroup() as group: for server in servers: + server = ServerAddr.from_str(server) await group.spawn(get_response(server)) return responses From 8baa79be882745375226c1bd1a241c0c168bc2ab Mon Sep 17 00:00:00 2001 From: SomberNight Date: Tue, 14 Apr 2020 18:28:41 +0200 Subject: [PATCH 070/117] network: implement exponential backoff for retries --- electrum/network.py | 106 +++++++++++++++++++++++++++----------------- 1 file changed, 65 insertions(+), 41 deletions(-) diff --git a/electrum/network.py b/electrum/network.py index 4ea4a88d87..237ae11b6e 100644 --- a/electrum/network.py +++ b/electrum/network.py @@ -71,11 +71,12 @@ _logger = get_logger(__name__) - -NODES_RETRY_INTERVAL = 60 -SERVER_RETRY_INTERVAL = 10 NUM_TARGET_CONNECTED_SERVERS = 10 NUM_RECENT_SERVERS = 20 +MAX_RETRY_DELAY_FOR_SERVERS = 600 # sec +INIT_RETRY_DELAY_FOR_SERVERS = 15 # sec +MAX_RETRY_DELAY_FOR_MAIN_SERVER = 10 # sec +INIT_RETRY_DELAY_FOR_MAIN_SERVER = 1 # sec def parse_servers(result: Sequence[Tuple[str, str, List[str]]]) -> Dict[str, dict]: @@ -245,8 +246,8 @@ class Network(Logger): interfaces: Dict[ServerAddr, Interface] connecting: Set[ServerAddr] server_queue: 'Optional[queue.Queue[ServerAddr]]' - disconnected_servers: Set[ServerAddr] default_server: ServerAddr + _recent_servers: List[ServerAddr] def __init__(self, config: SimpleConfig, *, daemon: 'Daemon' = None): global _INSTANCE @@ -291,7 +292,7 @@ def __init__(self, config: SimpleConfig, *, daemon: 'Daemon' = None): self.interfaces_lock = threading.Lock() # for mutating/iterating self.interfaces self.server_peers = {} # returned by interface (servers that the main interface knows about) - self.recent_servers = self._read_recent_servers() # note: needs self.recent_servers_lock + self._recent_servers = self._read_recent_servers() # note: needs self.recent_servers_lock self.banner = '' self.donation_address = '' @@ -301,8 +302,7 @@ def __init__(self, config: SimpleConfig, *, daemon: 'Daemon' = None): util.make_dir(dir_path) # retry times - self.server_retry_time = time.time() - self.nodes_retry_time = time.time() + self._last_tried_server = {} # type: Dict[ServerAddr, Tuple[float, int]] # unix ts, num_attempts # the main server we are currently communicating with self.interface = None self.default_server_changed_event = asyncio.Event() @@ -373,7 +373,7 @@ def _save_recent_servers(self): if not self.config.path: return path = os.path.join(self.config.path, "recent_servers") - s = json.dumps(self.recent_servers, indent=4, sort_keys=True, cls=MyEncoder) + s = json.dumps(self._recent_servers, indent=4, sort_keys=True, cls=MyEncoder) try: with open(path, "w", encoding='utf-8') as f: f.write(s) @@ -526,7 +526,7 @@ def get_servers(self): # hardcoded servers out.update(constants.net.DEFAULT_SERVERS) # add recent servers - for server in self.recent_servers: + for server in self._recent_servers: port = str(server.port) if server.host in out: out[server.host].update({server.protocol: port}) @@ -538,20 +538,52 @@ def get_servers(self): return out def _start_interface(self, server: ServerAddr): - if server not in self.interfaces and server not in self.connecting: - if server == self.default_server: - self.logger.info(f"connecting to {server} as new interface") - self._set_status('connecting') - self.connecting.add(server) - self.server_queue.put(server) - - def _start_random_interface(self) -> Optional[ServerAddr]: + if server in self.interfaces or server in self.connecting: + return + if server == self.default_server: + self.logger.info(f"connecting to {server} as new interface") + self._set_status('connecting') + self.connecting.add(server) + self.server_queue.put(server) + # update _last_tried_server + last_time, num_attempts = self._last_tried_server.get(server, (0, 0)) + self._last_tried_server[server] = time.time(), num_attempts + 1 + + def _can_retry_server(self, server: ServerAddr, *, now: float = None) -> bool: + if now is None: + now = time.time() + last_time, num_attempts = self._last_tried_server.get(server, (0, 0)) + if server == self.default_server: + delay = min(MAX_RETRY_DELAY_FOR_MAIN_SERVER, + INIT_RETRY_DELAY_FOR_MAIN_SERVER * 2 ** num_attempts) + else: + delay = min(MAX_RETRY_DELAY_FOR_SERVERS, + INIT_RETRY_DELAY_FOR_SERVERS * 2 ** num_attempts) + next_time = last_time + delay + return next_time < now + + def _get_next_server_to_try(self) -> Optional[ServerAddr]: + now = time.time() with self.interfaces_lock: - exclude_set = self.disconnected_servers | set(self.interfaces) | self.connecting - server = pick_random_server(self.get_servers(), protocol=self.protocol, exclude_set=exclude_set) - if server: - self._start_interface(server) - return server + exclude_set = set(self.interfaces) | self.connecting + # first try from recent servers + with self.recent_servers_lock: + recent_servers = list(self._recent_servers) + recent_servers = [s for s in recent_servers if s.protocol == self.protocol] + for server in recent_servers: + if server in exclude_set: + continue + if not self._can_retry_server(server, now=now): + continue + return server + # try all servers we know about + hostmap = self.get_servers() + servers = set(filter_protocol(hostmap, self.protocol)) - exclude_set + for server in servers: + if not self._can_retry_server(server, now=now): + continue + return server + return None def _set_proxy(self, proxy: Optional[dict]): self.proxy = proxy @@ -701,11 +733,12 @@ async def _close_interface(self, interface: Interface): @with_recent_servers_lock def _add_recent_server(self, server): + self._last_tried_server[server] = time.time(), 0 # list is ordered - if server in self.recent_servers: - self.recent_servers.remove(server) - self.recent_servers.insert(0, server) - self.recent_servers = self.recent_servers[:NUM_RECENT_SERVERS] + if server in self._recent_servers: + self._recent_servers.remove(server) + self._recent_servers.insert(0, server) + self._recent_servers = self._recent_servers[:NUM_RECENT_SERVERS] self._save_recent_servers() async def connection_down(self, interface: Interface): @@ -713,7 +746,6 @@ async def connection_down(self, interface: Interface): We distinguish by whether it is in self.interfaces.''' if not interface: return server = interface.server - self.disconnected_servers.add(server) if server == self.default_server: self._set_status('disconnected') await self._close_interface(interface) @@ -752,7 +784,7 @@ async def _run_new_interface(self, server: ServerAddr): self._add_recent_server(server) util.trigger_callback('network_updated') - def check_interface_against_healthy_spread_of_connected_servers(self, iface_to_check) -> bool: + def check_interface_against_healthy_spread_of_connected_servers(self, iface_to_check: Interface) -> bool: # main interface is exempt. this makes switching servers easier if iface_to_check.is_main_server(): return True @@ -1115,7 +1147,7 @@ async def _start(self): assert not self.interface and not self.interfaces assert not self.connecting and not self.server_queue self.logger.info('starting network') - self.disconnected_servers = set([]) + self._last_tried_server.clear() self.protocol = self.default_server.protocol self.server_queue = queue.Queue() self._set_proxy(deserialize_proxy(self.config.get('proxy'))) @@ -1174,17 +1206,12 @@ def stop(self): async def _ensure_there_is_a_main_interface(self): if self.is_connected(): return - now = time.time() # if auto_connect is set, try a different server if self.auto_connect and not self.is_connecting(): await self._switch_to_random_interface() # if auto_connect is not set, or still no main interface, retry current if not self.is_connected() and not self.is_connecting(): - if self.default_server in self.disconnected_servers: - if now - self.server_retry_time > SERVER_RETRY_INTERVAL: - self.disconnected_servers.remove(self.default_server) - self.server_retry_time = now - else: + if self._can_retry_server(self.default_server): await self.switch_to_interface(self.default_server) async def _maintain_sessions(self): @@ -1193,14 +1220,11 @@ async def launch_already_queued_up_new_interfaces(): server = self.server_queue.get() await self.taskgroup.spawn(self._run_new_interface(server)) async def maybe_queue_new_interfaces_to_be_launched_later(): - now = time.time() for i in range(self.num_server - len(self.interfaces) - len(self.connecting)): # FIXME this should try to honour "healthy spread of connected servers" - self._start_random_interface() - if now - self.nodes_retry_time > NODES_RETRY_INTERVAL: - self.logger.info('network: retrying connections') - self.disconnected_servers = set([]) - self.nodes_retry_time = now + server = self._get_next_server_to_try() + if server: + self._start_interface(server) async def maintain_healthy_spread_of_connected_servers(): with self.interfaces_lock: interfaces = list(self.interfaces.values()) random.shuffle(interfaces) From 34e3e48ba5942614a1d1c44f57f7e9a4da3d701f Mon Sep 17 00:00:00 2001 From: SomberNight Date: Tue, 14 Apr 2020 18:44:45 +0200 Subject: [PATCH 071/117] network: rm server_queue it's no longer needed; now it was just an extra level of indirection --- electrum/network.py | 47 +++++++++++++++++---------------------------- 1 file changed, 18 insertions(+), 29 deletions(-) diff --git a/electrum/network.py b/electrum/network.py index 237ae11b6e..db378187f7 100644 --- a/electrum/network.py +++ b/electrum/network.py @@ -245,7 +245,6 @@ class Network(Logger): interface: Optional[Interface] interfaces: Dict[ServerAddr, Interface] connecting: Set[ServerAddr] - server_queue: 'Optional[queue.Queue[ServerAddr]]' default_server: ServerAddr _recent_servers: List[ServerAddr] @@ -310,7 +309,6 @@ def __init__(self, config: SimpleConfig, *, daemon: 'Daemon' = None): self.interfaces = {} self.auto_connect = self.config.get('auto_connect', True) self.connecting = set() - self.server_queue = None self.proxy = None # Dump network messages (all interfaces). Set at runtime from the console. @@ -537,18 +535,6 @@ def get_servers(self): out = filter_noonion(out) return out - def _start_interface(self, server: ServerAddr): - if server in self.interfaces or server in self.connecting: - return - if server == self.default_server: - self.logger.info(f"connecting to {server} as new interface") - self._set_status('connecting') - self.connecting.add(server) - self.server_queue.put(server) - # update _last_tried_server - last_time, num_attempts = self._last_tried_server.get(server, (0, 0)) - self._last_tried_server[server] = time.time(), num_attempts + 1 - def _can_retry_server(self, server: ServerAddr, *, now: float = None) -> bool: if now is None: now = time.time() @@ -700,11 +686,11 @@ async def switch_to_interface(self, server: ServerAddr): if old_server and old_server != server: await self._close_interface(old_interface) if len(self.interfaces) <= self.num_server: - self._start_interface(old_server) + await self.taskgroup.spawn(self._run_new_interface(old_server)) if server not in self.interfaces: self.interface = None - self._start_interface(server) + await self.taskgroup.spawn(self._run_new_interface(server)) return i = self.interfaces[server] @@ -758,9 +744,19 @@ def get_network_timeout_seconds(self, request_type=NetworkTimeout.Generic) -> in return request_type.RELAXED return request_type.NORMAL - @ignore_exceptions # do not kill main_taskgroup + @ignore_exceptions # do not kill outer taskgroup @log_exceptions async def _run_new_interface(self, server: ServerAddr): + if server in self.interfaces or server in self.connecting: + return + self.connecting.add(server) + if server == self.default_server: + self.logger.info(f"connecting to {server} as new interface") + self._set_status('connecting') + # update _last_tried_server + last_time, num_attempts = self._last_tried_server.get(server, (0, 0)) + self._last_tried_server[server] = time.time(), num_attempts + 1 + interface = Interface(network=self, server=server, proxy=self.proxy) # note: using longer timeouts here as DNS can sometimes be slow! timeout = self.get_network_timeout_seconds(NetworkTimeout.Generic) @@ -1145,14 +1141,13 @@ async def _start(self): assert not self.taskgroup self.taskgroup = taskgroup = SilentTaskGroup() assert not self.interface and not self.interfaces - assert not self.connecting and not self.server_queue + assert not self.connecting self.logger.info('starting network') self._last_tried_server.clear() self.protocol = self.default_server.protocol - self.server_queue = queue.Queue() self._set_proxy(deserialize_proxy(self.config.get('proxy'))) self._set_oneserver(self.config.get('oneserver', False)) - self._start_interface(self.default_server) + await self.taskgroup.spawn(self._run_new_interface(self.default_server)) async def main(): self.logger.info("starting taskgroup.") @@ -1192,7 +1187,6 @@ async def _stop(self, full_shutdown=False): self.interface = None self.interfaces = {} self.connecting.clear() - self.server_queue = None if not full_shutdown: util.trigger_callback('network_updated') @@ -1215,16 +1209,12 @@ async def _ensure_there_is_a_main_interface(self): await self.switch_to_interface(self.default_server) async def _maintain_sessions(self): - async def launch_already_queued_up_new_interfaces(): - while self.server_queue.qsize() > 0: - server = self.server_queue.get() - await self.taskgroup.spawn(self._run_new_interface(server)) - async def maybe_queue_new_interfaces_to_be_launched_later(): + async def maybe_start_new_interfaces(): for i in range(self.num_server - len(self.interfaces) - len(self.connecting)): # FIXME this should try to honour "healthy spread of connected servers" server = self._get_next_server_to_try() if server: - self._start_interface(server) + await self.taskgroup.spawn(self._run_new_interface(server)) async def maintain_healthy_spread_of_connected_servers(): with self.interfaces_lock: interfaces = list(self.interfaces.values()) random.shuffle(interfaces) @@ -1241,8 +1231,7 @@ async def maintain_main_interface(): while True: try: - await launch_already_queued_up_new_interfaces() - await maybe_queue_new_interfaces_to_be_launched_later() + await maybe_start_new_interfaces() await maintain_healthy_spread_of_connected_servers() await maintain_main_interface() except asyncio.CancelledError: From ac749f3a19ec10ee6f826c88d670e6cebab9aa17 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Tue, 14 Apr 2020 19:58:22 +0200 Subject: [PATCH 072/117] network: introduce NUM_STICKY_SERVERS --- electrum/network.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/electrum/network.py b/electrum/network.py index db378187f7..a3ed4988a4 100644 --- a/electrum/network.py +++ b/electrum/network.py @@ -72,6 +72,7 @@ NUM_TARGET_CONNECTED_SERVERS = 10 +NUM_STICKY_SERVERS = 4 NUM_RECENT_SERVERS = 20 MAX_RETRY_DELAY_FOR_SERVERS = 600 # sec INIT_RETRY_DELAY_FOR_SERVERS = 15 # sec @@ -551,20 +552,27 @@ def _can_retry_server(self, server: ServerAddr, *, now: float = None) -> bool: def _get_next_server_to_try(self) -> Optional[ServerAddr]: now = time.time() with self.interfaces_lock: - exclude_set = set(self.interfaces) | self.connecting - # first try from recent servers + connected_servers = set(self.interfaces) | self.connecting + # First try from recent servers. (which are persisted) + # As these are servers we successfully connected to recently, they are + # most likely to work. This also makes servers "sticky". + # Note: with sticky servers, it is more difficult for an attacker to eclipse the client, + # however if they succeed, the eclipsing would persist. To try to balance this, + # we only give priority to recent_servers up to NUM_STICKY_SERVERS. with self.recent_servers_lock: recent_servers = list(self._recent_servers) recent_servers = [s for s in recent_servers if s.protocol == self.protocol] - for server in recent_servers: - if server in exclude_set: - continue - if not self._can_retry_server(server, now=now): - continue - return server - # try all servers we know about + if len(connected_servers & set(recent_servers)) < NUM_STICKY_SERVERS: + for server in recent_servers: + if server in connected_servers: + continue + if not self._can_retry_server(server, now=now): + continue + return server + # try all servers we know about, pick one at random hostmap = self.get_servers() - servers = set(filter_protocol(hostmap, self.protocol)) - exclude_set + servers = list(set(filter_protocol(hostmap, self.protocol)) - connected_servers) + random.shuffle(servers) for server in servers: if not self._can_retry_server(server, now=now): continue From 86b29603cb87f8b2314bfffb22dd61fbe4d059b0 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Wed, 15 Apr 2020 16:01:34 +0200 Subject: [PATCH 073/117] network: (trivial) rename field to indicate private --- electrum/network.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/electrum/network.py b/electrum/network.py index a3ed4988a4..b08ab4b8ca 100644 --- a/electrum/network.py +++ b/electrum/network.py @@ -245,7 +245,7 @@ class Network(Logger): taskgroup: Optional[TaskGroup] interface: Optional[Interface] interfaces: Dict[ServerAddr, Interface] - connecting: Set[ServerAddr] + _connecting: Set[ServerAddr] default_server: ServerAddr _recent_servers: List[ServerAddr] @@ -309,7 +309,7 @@ def __init__(self, config: SimpleConfig, *, daemon: 'Daemon' = None): # set of servers we have an ongoing connection with self.interfaces = {} self.auto_connect = self.config.get('auto_connect', True) - self.connecting = set() + self._connecting = set() self.proxy = None # Dump network messages (all interfaces). Set at runtime from the console. @@ -552,7 +552,7 @@ def _can_retry_server(self, server: ServerAddr, *, now: float = None) -> bool: def _get_next_server_to_try(self) -> Optional[ServerAddr]: now = time.time() with self.interfaces_lock: - connected_servers = set(self.interfaces) | self.connecting + connected_servers = set(self.interfaces) | self._connecting # First try from recent servers. (which are persisted) # As these are servers we successfully connected to recently, they are # most likely to work. This also makes servers "sticky". @@ -755,9 +755,9 @@ def get_network_timeout_seconds(self, request_type=NetworkTimeout.Generic) -> in @ignore_exceptions # do not kill outer taskgroup @log_exceptions async def _run_new_interface(self, server: ServerAddr): - if server in self.interfaces or server in self.connecting: + if server in self.interfaces or server in self._connecting: return - self.connecting.add(server) + self._connecting.add(server) if server == self.default_server: self.logger.info(f"connecting to {server} as new interface") self._set_status('connecting') @@ -779,7 +779,7 @@ async def _run_new_interface(self, server: ServerAddr): assert server not in self.interfaces self.interfaces[server] = interface finally: - try: self.connecting.remove(server) + try: self._connecting.remove(server) except KeyError: pass if server == self.default_server: @@ -1149,7 +1149,7 @@ async def _start(self): assert not self.taskgroup self.taskgroup = taskgroup = SilentTaskGroup() assert not self.interface and not self.interfaces - assert not self.connecting + assert not self._connecting self.logger.info('starting network') self._last_tried_server.clear() self.protocol = self.default_server.protocol @@ -1194,7 +1194,7 @@ async def _stop(self, full_shutdown=False): self.taskgroup = None self.interface = None self.interfaces = {} - self.connecting.clear() + self._connecting.clear() if not full_shutdown: util.trigger_callback('network_updated') @@ -1218,7 +1218,7 @@ async def _ensure_there_is_a_main_interface(self): async def _maintain_sessions(self): async def maybe_start_new_interfaces(): - for i in range(self.num_server - len(self.interfaces) - len(self.connecting)): + for i in range(self.num_server - len(self.interfaces) - len(self._connecting)): # FIXME this should try to honour "healthy spread of connected servers" server = self._get_next_server_to_try() if server: From 90cb032721ffd8fad883762b3942de7a08b1949a Mon Sep 17 00:00:00 2001 From: SomberNight Date: Wed, 15 Apr 2020 16:40:16 +0200 Subject: [PATCH 074/117] lnworker: implement exponential backoff for retries --- electrum/lnpeer.py | 5 +-- electrum/lntransport.py | 4 +++ electrum/lnworker.py | 70 ++++++++++++++++++++++++++--------------- 3 files changed, 49 insertions(+), 30 deletions(-) diff --git a/electrum/lnpeer.py b/electrum/lnpeer.py index 7157761a20..4479234eaa 100644 --- a/electrum/lnpeer.py +++ b/electrum/lnpeer.py @@ -219,10 +219,7 @@ def on_init(self, payload): if constants.net.rev_genesis_bytes() not in their_chains: raise GracefulDisconnect(f"no common chain found with remote. (they sent: {their_chains})") # all checks passed - if self.channel_db and isinstance(self.transport, LNTransport): - self.channel_db.add_recent_peer(self.transport.peer_addr) - for chan in self.channels.values(): - chan.add_or_update_peer_addr(self.transport.peer_addr) + self.lnworker.on_peer_successfully_established(self) self._received_init = True self.maybe_set_initialized() diff --git a/electrum/lntransport.py b/electrum/lntransport.py index c006f36ec2..257f02b12a 100644 --- a/electrum/lntransport.py +++ b/electrum/lntransport.py @@ -155,6 +155,8 @@ def close(self): class LNResponderTransport(LNTransportBase): + """Transport initiated by remote party.""" + def __init__(self, privkey: bytes, reader: StreamReader, writer: StreamWriter): LNTransportBase.__init__(self) self.reader = reader @@ -211,7 +213,9 @@ async def handshake(self, **kwargs): self.init_counters(ck) return rs + class LNTransport(LNTransportBase): + """Transport initiated by local party.""" def __init__(self, privkey: bytes, peer_addr: LNPeerAddr): LNTransportBase.__init__(self) diff --git a/electrum/lnworker.py b/electrum/lnworker.py index cc8e55bd9e..6c1fb4b6f6 100644 --- a/electrum/lnworker.py +++ b/electrum/lnworker.py @@ -77,9 +77,11 @@ NUM_PEERS_TARGET = 4 -PEER_RETRY_INTERVAL = 600 # seconds -PEER_RETRY_INTERVAL_FOR_CHANNELS = 30 # seconds -GRAPH_DOWNLOAD_SECONDS = 600 + +MAX_RETRY_DELAY_FOR_PEERS = 3600 # sec +INIT_RETRY_DELAY_FOR_PEERS = 600 # sec +MAX_RETRY_DELAY_FOR_CHANNEL_PEERS = 300 # sec +INIT_RETRY_DELAY_FOR_CHANNEL_PEERS = 4 # sec FALLBACK_NODE_LIST_TESTNET = ( LNPeerAddr(host='203.132.95.10', port=9735, pubkey=bfh('038863cf8ab91046230f561cd5b386cbff8309fa02e3f0c3ed161a3aeb64a643b9')), @@ -156,6 +158,8 @@ def __init__(self, xprv): self.features |= LnFeatures.VAR_ONION_OPT self.features |= LnFeatures.PAYMENT_SECRET_OPT + self._last_tried_peer = {} # type: Dict[LNPeerAddr, Tuple[float, int]] # LNPeerAddr -> (unix ts, num_attempts) + def channels_for_peer(self, node_id): return {} @@ -204,8 +208,7 @@ async def _maintain_connectivity(self): continue peers = await self._get_next_peers_to_try() for peer in peers: - last_tried = self._last_tried_peer.get(peer, 0) - if last_tried + PEER_RETRY_INTERVAL < now: + if self._can_retry_peer(peer, now=now): await self._add_peer(peer.host, peer.port, peer.pubkey) async def _add_peer(self, host, port, node_id) -> Peer: @@ -214,7 +217,8 @@ async def _add_peer(self, host, port, node_id) -> Peer: port = int(port) peer_addr = LNPeerAddr(host, port, node_id) transport = LNTransport(self.node_keypair.privkey, peer_addr) - self._last_tried_peer[peer_addr] = time.time() + last_time, num_attempts = self._last_tried_peer.get(peer_addr, (0, 0)) + self._last_tried_peer[peer_addr] = time.time(), num_attempts + 1 self.logger.info(f"adding peer {peer_addr}") peer = Peer(self, node_id, transport) await self.taskgroup.spawn(peer.main_loop()) @@ -233,7 +237,6 @@ def start_network(self, network: 'Network'): self.network = network self.config = network.config self.channel_db = self.network.channel_db - self._last_tried_peer = {} # type: Dict[LNPeerAddr, float] # LNPeerAddr -> unix timestamp self._add_peers_from_config() asyncio.run_coroutine_threadsafe(self.main_loop(), self.network.asyncio_loop) @@ -259,20 +262,43 @@ def is_good_peer(self, peer): #self.logger.info(f'is_good {peer.host}') return True + def on_peer_successfully_established(self, peer: Peer) -> None: + if isinstance(peer.transport, LNTransport): + peer_addr = peer.transport.peer_addr + # reset connection attempt count + self._last_tried_peer[peer_addr] = time.time(), 0 + # add into channel db + if self.channel_db: + self.channel_db.add_recent_peer(peer_addr) + # save network address into channels we might have with peer + for chan in peer.channels.values(): + chan.add_or_update_peer_addr(peer_addr) + + def _can_retry_peer(self, peer: LNPeerAddr, *, + now: float = None, for_channel: bool = False) -> bool: + if now is None: + now = time.time() + last_time, num_attempts = self._last_tried_peer.get(peer, (0, 0)) + if for_channel: + delay = min(MAX_RETRY_DELAY_FOR_CHANNEL_PEERS, + INIT_RETRY_DELAY_FOR_CHANNEL_PEERS * 2 ** num_attempts) + else: + delay = min(MAX_RETRY_DELAY_FOR_PEERS, + INIT_RETRY_DELAY_FOR_PEERS * 2 ** num_attempts) + next_time = last_time + delay + return next_time < now + async def _get_next_peers_to_try(self) -> Sequence[LNPeerAddr]: now = time.time() await self.channel_db.data_loaded.wait() - recent_peers = self.channel_db.get_recent_peers() - # maintenance for last tried times - # due to this, below we can just test membership in _last_tried_peer - for peer in list(self._last_tried_peer): - if now >= self._last_tried_peer[peer] + PEER_RETRY_INTERVAL: - del self._last_tried_peer[peer] # first try from recent peers + recent_peers = self.channel_db.get_recent_peers() for peer in recent_peers: + if not peer: + continue if peer.pubkey in self.peers: continue - if peer in self._last_tried_peer: + if not self._can_retry_peer(peer, now=now): continue if not self.is_good_peer(peer): continue @@ -289,7 +315,7 @@ async def _get_next_peers_to_try(self) -> Sequence[LNPeerAddr]: peer = LNPeerAddr(host, port, node_id) except ValueError: continue - if peer in self._last_tried_peer: + if not self._can_retry_peer(peer, now=now): continue if not self.is_good_peer(peer): continue @@ -304,7 +330,7 @@ async def _get_next_peers_to_try(self) -> Sequence[LNPeerAddr]: else: return [] # regtest?? - fallback_list = [peer for peer in fallback_list if peer not in self._last_tried_peer] + fallback_list = [peer for peer in fallback_list if self._can_retry_peer(peer, now=now)] if fallback_list: return [random.choice(fallback_list)] @@ -1269,18 +1295,10 @@ async def reestablish_peer_for_given_channel(self, chan: Channel) -> None: peer_addresses.append(LNPeerAddr(host, port, chan.node_id)) # will try addresses stored in channel storage peer_addresses += list(chan.get_peer_addresses()) + # Done gathering addresses. # Now select first one that has not failed recently. - # Use long retry interval to check. This ensures each address we gathered gets a chance. - for peer in peer_addresses: - last_tried = self._last_tried_peer.get(peer, 0) - if last_tried + PEER_RETRY_INTERVAL < now: - await self._add_peer(peer.host, peer.port, peer.pubkey) - return - # Still here? That means all addresses failed ~recently. - # Use short retry interval now. for peer in peer_addresses: - last_tried = self._last_tried_peer.get(peer, 0) - if last_tried + PEER_RETRY_INTERVAL_FOR_CHANNELS < now: + if self._can_retry_peer(peer, for_channel=True, now=now): await self._add_peer(peer.host, peer.port, peer.pubkey) return From 76f0ad3271a611457a88e4bfe213334afcf1a5ae Mon Sep 17 00:00:00 2001 From: SomberNight Date: Wed, 15 Apr 2020 17:17:11 +0200 Subject: [PATCH 075/117] util: add NetworkRetryManager, a baseclass for LNWorker and Network --- electrum/lnworker.py | 48 ++++++++++++++------------------------- electrum/network.py | 46 +++++++++++++------------------------- electrum/util.py | 53 +++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 85 insertions(+), 62 deletions(-) diff --git a/electrum/lnworker.py b/electrum/lnworker.py index 6c1fb4b6f6..0dc047a776 100644 --- a/electrum/lnworker.py +++ b/electrum/lnworker.py @@ -25,7 +25,7 @@ from . import keystore from .util import profiler from .util import PR_UNPAID, PR_EXPIRED, PR_PAID, PR_INFLIGHT, PR_FAILED, PR_ROUTING -from .util import PR_TYPE_LN +from .util import PR_TYPE_LN, NetworkRetryManager from .lnutil import LN_MAX_FUNDING_SAT from .keystore import BIP32_KeyStore from .bitcoin import COIN @@ -78,10 +78,6 @@ NUM_PEERS_TARGET = 4 -MAX_RETRY_DELAY_FOR_PEERS = 3600 # sec -INIT_RETRY_DELAY_FOR_PEERS = 600 # sec -MAX_RETRY_DELAY_FOR_CHANNEL_PEERS = 300 # sec -INIT_RETRY_DELAY_FOR_CHANNEL_PEERS = 4 # sec FALLBACK_NODE_LIST_TESTNET = ( LNPeerAddr(host='203.132.95.10', port=9735, pubkey=bfh('038863cf8ab91046230f561cd5b386cbff8309fa02e3f0c3ed161a3aeb64a643b9')), @@ -143,10 +139,17 @@ def __str__(self): return _('No path found') -class LNWorker(Logger): +class LNWorker(Logger, NetworkRetryManager[LNPeerAddr]): def __init__(self, xprv): Logger.__init__(self) + NetworkRetryManager.__init__( + self, + max_retry_delay_normal=3600, + init_retry_delay_normal=600, + max_retry_delay_urgent=300, + init_retry_delay_urgent=4, + ) self.node_keypair = generate_keypair(BIP32Node.from_xkey(xprv), LnKeyFamily.NODE_KEY) self.peers = {} # type: Dict[bytes, Peer] # pubkey -> Peer self.taskgroup = SilentTaskGroup() @@ -158,8 +161,6 @@ def __init__(self, xprv): self.features |= LnFeatures.VAR_ONION_OPT self.features |= LnFeatures.PAYMENT_SECRET_OPT - self._last_tried_peer = {} # type: Dict[LNPeerAddr, Tuple[float, int]] # LNPeerAddr -> (unix ts, num_attempts) - def channels_for_peer(self, node_id): return {} @@ -208,17 +209,16 @@ async def _maintain_connectivity(self): continue peers = await self._get_next_peers_to_try() for peer in peers: - if self._can_retry_peer(peer, now=now): + if self._can_retry_addr(peer, now=now): await self._add_peer(peer.host, peer.port, peer.pubkey) - async def _add_peer(self, host, port, node_id) -> Peer: + async def _add_peer(self, host: str, port: int, node_id: bytes) -> Peer: if node_id in self.peers: return self.peers[node_id] port = int(port) peer_addr = LNPeerAddr(host, port, node_id) transport = LNTransport(self.node_keypair.privkey, peer_addr) - last_time, num_attempts = self._last_tried_peer.get(peer_addr, (0, 0)) - self._last_tried_peer[peer_addr] = time.time(), num_attempts + 1 + self._trying_addr_now(peer_addr) self.logger.info(f"adding peer {peer_addr}") peer = Peer(self, node_id, transport) await self.taskgroup.spawn(peer.main_loop()) @@ -266,7 +266,7 @@ def on_peer_successfully_established(self, peer: Peer) -> None: if isinstance(peer.transport, LNTransport): peer_addr = peer.transport.peer_addr # reset connection attempt count - self._last_tried_peer[peer_addr] = time.time(), 0 + self._on_connection_successfully_established(peer_addr) # add into channel db if self.channel_db: self.channel_db.add_recent_peer(peer_addr) @@ -274,20 +274,6 @@ def on_peer_successfully_established(self, peer: Peer) -> None: for chan in peer.channels.values(): chan.add_or_update_peer_addr(peer_addr) - def _can_retry_peer(self, peer: LNPeerAddr, *, - now: float = None, for_channel: bool = False) -> bool: - if now is None: - now = time.time() - last_time, num_attempts = self._last_tried_peer.get(peer, (0, 0)) - if for_channel: - delay = min(MAX_RETRY_DELAY_FOR_CHANNEL_PEERS, - INIT_RETRY_DELAY_FOR_CHANNEL_PEERS * 2 ** num_attempts) - else: - delay = min(MAX_RETRY_DELAY_FOR_PEERS, - INIT_RETRY_DELAY_FOR_PEERS * 2 ** num_attempts) - next_time = last_time + delay - return next_time < now - async def _get_next_peers_to_try(self) -> Sequence[LNPeerAddr]: now = time.time() await self.channel_db.data_loaded.wait() @@ -298,7 +284,7 @@ async def _get_next_peers_to_try(self) -> Sequence[LNPeerAddr]: continue if peer.pubkey in self.peers: continue - if not self._can_retry_peer(peer, now=now): + if not self._can_retry_addr(peer, now=now): continue if not self.is_good_peer(peer): continue @@ -315,7 +301,7 @@ async def _get_next_peers_to_try(self) -> Sequence[LNPeerAddr]: peer = LNPeerAddr(host, port, node_id) except ValueError: continue - if not self._can_retry_peer(peer, now=now): + if not self._can_retry_addr(peer, now=now): continue if not self.is_good_peer(peer): continue @@ -330,7 +316,7 @@ async def _get_next_peers_to_try(self) -> Sequence[LNPeerAddr]: else: return [] # regtest?? - fallback_list = [peer for peer in fallback_list if self._can_retry_peer(peer, now=now)] + fallback_list = [peer for peer in fallback_list if self._can_retry_addr(peer, now=now)] if fallback_list: return [random.choice(fallback_list)] @@ -1298,7 +1284,7 @@ async def reestablish_peer_for_given_channel(self, chan: Channel) -> None: # Done gathering addresses. # Now select first one that has not failed recently. for peer in peer_addresses: - if self._can_retry_peer(peer, for_channel=True, now=now): + if self._can_retry_addr(peer, urgent=True, now=now): await self._add_peer(peer.host, peer.port, peer.pubkey) return diff --git a/electrum/network.py b/electrum/network.py index b08ab4b8ca..09c2c6ad3d 100644 --- a/electrum/network.py +++ b/electrum/network.py @@ -44,7 +44,7 @@ from . import util from .util import (log_exceptions, ignore_exceptions, bfh, SilentTaskGroup, make_aiohttp_session, send_exception_to_crash_reporter, - is_hash256_str, is_non_negative_integer, MyEncoder) + is_hash256_str, is_non_negative_integer, MyEncoder, NetworkRetryManager) from .bitcoin import COIN from . import constants @@ -74,10 +74,6 @@ NUM_TARGET_CONNECTED_SERVERS = 10 NUM_STICKY_SERVERS = 4 NUM_RECENT_SERVERS = 20 -MAX_RETRY_DELAY_FOR_SERVERS = 600 # sec -INIT_RETRY_DELAY_FOR_SERVERS = 15 # sec -MAX_RETRY_DELAY_FOR_MAIN_SERVER = 10 # sec -INIT_RETRY_DELAY_FOR_MAIN_SERVER = 1 # sec def parse_servers(result: Sequence[Tuple[str, str, List[str]]]) -> Dict[str, dict]: @@ -235,7 +231,7 @@ def __repr__(self): _INSTANCE = None -class Network(Logger): +class Network(Logger, NetworkRetryManager[ServerAddr]): """The Network class manages a set of connections to remote electrum servers, each connected socket is handled by an Interface() object. """ @@ -255,6 +251,13 @@ def __init__(self, config: SimpleConfig, *, daemon: 'Daemon' = None): _INSTANCE = self Logger.__init__(self) + NetworkRetryManager.__init__( + self, + max_retry_delay_normal=600, + init_retry_delay_normal=15, + max_retry_delay_urgent=10, + init_retry_delay_urgent=1, + ) self.asyncio_loop = asyncio.get_event_loop() assert self.asyncio_loop.is_running(), "event loop not running" @@ -301,8 +304,6 @@ def __init__(self, config: SimpleConfig, *, daemon: 'Daemon' = None): dir_path = os.path.join(self.config.path, 'certs') util.make_dir(dir_path) - # retry times - self._last_tried_server = {} # type: Dict[ServerAddr, Tuple[float, int]] # unix ts, num_attempts # the main server we are currently communicating with self.interface = None self.default_server_changed_event = asyncio.Event() @@ -536,19 +537,6 @@ def get_servers(self): out = filter_noonion(out) return out - def _can_retry_server(self, server: ServerAddr, *, now: float = None) -> bool: - if now is None: - now = time.time() - last_time, num_attempts = self._last_tried_server.get(server, (0, 0)) - if server == self.default_server: - delay = min(MAX_RETRY_DELAY_FOR_MAIN_SERVER, - INIT_RETRY_DELAY_FOR_MAIN_SERVER * 2 ** num_attempts) - else: - delay = min(MAX_RETRY_DELAY_FOR_SERVERS, - INIT_RETRY_DELAY_FOR_SERVERS * 2 ** num_attempts) - next_time = last_time + delay - return next_time < now - def _get_next_server_to_try(self) -> Optional[ServerAddr]: now = time.time() with self.interfaces_lock: @@ -566,7 +554,7 @@ def _get_next_server_to_try(self) -> Optional[ServerAddr]: for server in recent_servers: if server in connected_servers: continue - if not self._can_retry_server(server, now=now): + if not self._can_retry_addr(server, now=now): continue return server # try all servers we know about, pick one at random @@ -574,7 +562,7 @@ def _get_next_server_to_try(self) -> Optional[ServerAddr]: servers = list(set(filter_protocol(hostmap, self.protocol)) - connected_servers) random.shuffle(servers) for server in servers: - if not self._can_retry_server(server, now=now): + if not self._can_retry_addr(server, now=now): continue return server return None @@ -726,8 +714,8 @@ async def _close_interface(self, interface: Interface): await interface.close() @with_recent_servers_lock - def _add_recent_server(self, server): - self._last_tried_server[server] = time.time(), 0 + def _add_recent_server(self, server: ServerAddr) -> None: + self._on_connection_successfully_established(server) # list is ordered if server in self._recent_servers: self._recent_servers.remove(server) @@ -761,9 +749,7 @@ async def _run_new_interface(self, server: ServerAddr): if server == self.default_server: self.logger.info(f"connecting to {server} as new interface") self._set_status('connecting') - # update _last_tried_server - last_time, num_attempts = self._last_tried_server.get(server, (0, 0)) - self._last_tried_server[server] = time.time(), num_attempts + 1 + self._trying_addr_now(server) interface = Interface(network=self, server=server, proxy=self.proxy) # note: using longer timeouts here as DNS can sometimes be slow! @@ -1151,7 +1137,7 @@ async def _start(self): assert not self.interface and not self.interfaces assert not self._connecting self.logger.info('starting network') - self._last_tried_server.clear() + self._clear_addr_retry_times() self.protocol = self.default_server.protocol self._set_proxy(deserialize_proxy(self.config.get('proxy'))) self._set_oneserver(self.config.get('oneserver', False)) @@ -1213,7 +1199,7 @@ async def _ensure_there_is_a_main_interface(self): await self._switch_to_random_interface() # if auto_connect is not set, or still no main interface, retry current if not self.is_connected() and not self.is_connecting(): - if self._can_retry_server(self.default_server): + if self._can_retry_addr(self.default_server, urgent=True): await self.switch_to_interface(self.default_server) async def _maintain_sessions(self): diff --git a/electrum/util.py b/electrum/util.py index 1c47ec8c16..48bd6910a0 100644 --- a/electrum/util.py +++ b/electrum/util.py @@ -23,7 +23,8 @@ import binascii import os, sys, re, json from collections import defaultdict, OrderedDict -from typing import NamedTuple, Union, TYPE_CHECKING, Tuple, Optional, Callable, Any, Sequence +from typing import (NamedTuple, Union, TYPE_CHECKING, Tuple, Optional, Callable, Any, + Sequence, Dict, Generic, TypeVar) from datetime import datetime import decimal from decimal import Decimal @@ -1342,3 +1343,53 @@ def trigger_callback(self, event, *args): trigger_callback = callback_mgr.trigger_callback register_callback = callback_mgr.register_callback unregister_callback = callback_mgr.unregister_callback + + +_NetAddrType = TypeVar("_NetAddrType") + + +class NetworkRetryManager(Generic[_NetAddrType]): + """Truncated Exponential Backoff for network connections.""" + + def __init__( + self, *, + max_retry_delay_normal: float, + init_retry_delay_normal: float, + max_retry_delay_urgent: float = None, + init_retry_delay_urgent: float = None, + ): + self._last_tried_addr = {} # type: Dict[_NetAddrType, Tuple[float, int]] # (unix ts, num_attempts) + + # note: these all use "seconds" as unit + if max_retry_delay_urgent is None: + max_retry_delay_urgent = max_retry_delay_normal + if init_retry_delay_urgent is None: + init_retry_delay_urgent = init_retry_delay_normal + self._max_retry_delay_normal = max_retry_delay_normal + self._init_retry_delay_normal = init_retry_delay_normal + self._max_retry_delay_urgent = max_retry_delay_urgent + self._init_retry_delay_urgent = init_retry_delay_urgent + + def _trying_addr_now(self, addr: _NetAddrType) -> None: + last_time, num_attempts = self._last_tried_addr.get(addr, (0, 0)) + self._last_tried_addr[addr] = time.time(), num_attempts + 1 + + def _on_connection_successfully_established(self, addr: _NetAddrType) -> None: + self._last_tried_addr[addr] = time.time(), 0 + + def _can_retry_addr(self, peer: _NetAddrType, *, + now: float = None, urgent: bool = False) -> bool: + if now is None: + now = time.time() + last_time, num_attempts = self._last_tried_addr.get(peer, (0, 0)) + if urgent: + delay = min(self._max_retry_delay_urgent, + self._init_retry_delay_urgent * 2 ** num_attempts) + else: + delay = min(self._max_retry_delay_normal, + self._init_retry_delay_normal * 2 ** num_attempts) + next_time = last_time + delay + return next_time < now + + def _clear_addr_retry_times(self) -> None: + self._last_tried_addr.clear() From 7257172e1c091cfe63923d3cbd30ded4b63aba25 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Wed, 15 Apr 2020 17:23:02 +0200 Subject: [PATCH 076/117] NetworkRetryManager: add random noise to time --- electrum/util.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/electrum/util.py b/electrum/util.py index 48bd6910a0..8e7d441d27 100644 --- a/electrum/util.py +++ b/electrum/util.py @@ -42,6 +42,7 @@ from typing import NamedTuple, Optional import ssl import ipaddress +import random import aiohttp from aiohttp_socks import ProxyConnector, ProxyType @@ -1372,7 +1373,10 @@ def __init__( def _trying_addr_now(self, addr: _NetAddrType) -> None: last_time, num_attempts = self._last_tried_addr.get(addr, (0, 0)) - self._last_tried_addr[addr] = time.time(), num_attempts + 1 + # we add up to 1 second of noise to the time, so that clients are less likely + # to get synchronised and bombard the remote in connection waves: + cur_time = time.time() + random.random() + self._last_tried_addr[addr] = cur_time, num_attempts + 1 def _on_connection_successfully_established(self, addr: _NetAddrType) -> None: self._last_tried_addr[addr] = time.time(), 0 From 1600241b0221f74b30d8130e8f2e25c03814e61d Mon Sep 17 00:00:00 2001 From: SomberNight Date: Wed, 15 Apr 2020 17:39:39 +0200 Subject: [PATCH 077/117] fix tests: follow-up prev few commits --- electrum/tests/test_lnpeer.py | 6 ++++-- electrum/tests/test_network.py | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/electrum/tests/test_lnpeer.py b/electrum/tests/test_lnpeer.py index beb4f0e402..af2db0611f 100644 --- a/electrum/tests/test_lnpeer.py +++ b/electrum/tests/test_lnpeer.py @@ -17,7 +17,7 @@ from electrum import simple_config, lnutil from electrum.lnaddr import lnencode, LnAddr, lndecode from electrum.bitcoin import COIN, sha256 -from electrum.util import bh2u, create_and_start_event_loop +from electrum.util import bh2u, create_and_start_event_loop, NetworkRetryManager from electrum.lnpeer import Peer from electrum.lnutil import LNPeerAddr, Keypair, privkey_to_pubkey from electrum.lnutil import LightningPeerConnectionClosed, RemoteMisbehaving @@ -95,9 +95,10 @@ def save_db(self): def is_lightning_backup(self): return False -class MockLNWallet(Logger): +class MockLNWallet(Logger, NetworkRetryManager[LNPeerAddr]): def __init__(self, remote_keypair, local_keypair, chan: 'Channel', tx_queue): Logger.__init__(self) + NetworkRetryManager.__init__(self, max_retry_delay_normal=1, init_retry_delay_normal=1) self.remote_keypair = remote_keypair self.node_keypair = local_keypair self.network = MockNetwork(tx_queue) @@ -160,6 +161,7 @@ def save_channel(self, chan): force_close_channel = LNWallet.force_close_channel try_force_closing = LNWallet.try_force_closing get_first_timestamp = lambda self: 0 + on_peer_successfully_established = LNWallet.on_peer_successfully_established class MockTransport: diff --git a/electrum/tests/test_network.py b/electrum/tests/test_network.py index 3d56e62820..b433a5e268 100644 --- a/electrum/tests/test_network.py +++ b/electrum/tests/test_network.py @@ -5,7 +5,7 @@ from electrum import constants from electrum.simple_config import SimpleConfig from electrum import blockchain -from electrum.interface import Interface +from electrum.interface import Interface, ServerAddr from electrum.crypto import sha256 from electrum.util import bh2u @@ -24,7 +24,7 @@ def __init__(self, config): self.config = config network = MockNetwork() network.config = config - super().__init__(network, 'mock-server:50000:t', None) + super().__init__(network=network, server=ServerAddr.from_str('mock-server:50000:t'), proxy=None) self.q = asyncio.Queue() self.blockchain = blockchain.Blockchain(config=self.config, forkpoint=0, parent=None, forkpoint_hash=constants.net.GENESIS, prev_hash=None) From c2d6a902dde63b117ff234764d2e7c60cd50c43c Mon Sep 17 00:00:00 2001 From: SomberNight Date: Wed, 15 Apr 2020 18:06:59 +0200 Subject: [PATCH 078/117] build: update some packages in dockerfiles Ubuntu no longer serves old version --- contrib/build-linux/appimage/Dockerfile | 2 +- contrib/build-wine/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/build-linux/appimage/Dockerfile b/contrib/build-linux/appimage/Dockerfile index 48c4e6f5ce..3a6d5768e8 100644 --- a/contrib/build-linux/appimage/Dockerfile +++ b/contrib/build-linux/appimage/Dockerfile @@ -4,7 +4,7 @@ ENV LC_ALL=C.UTF-8 LANG=C.UTF-8 RUN apt-get update -q && \ apt-get install -qy \ - git=1:2.7.4-0ubuntu1.7 \ + git=1:2.7.4-0ubuntu1.8 \ wget=1.17.1-1ubuntu1.5 \ make=4.1-6 \ autotools-dev=20150820.1 \ diff --git a/contrib/build-wine/Dockerfile b/contrib/build-wine/Dockerfile index 20c0efe9db..7064b38ee1 100644 --- a/contrib/build-wine/Dockerfile +++ b/contrib/build-wine/Dockerfile @@ -13,7 +13,7 @@ RUN dpkg --add-architecture i386 && \ RUN apt-get update -q && \ apt-get install -qy \ - git=1:2.17.1-1ubuntu0.5 \ + git=1:2.17.1-1ubuntu0.6 \ p7zip-full=16.02+dfsg-6 \ make=4.1-9.1ubuntu1 \ mingw-w64=5.0.3-1 \ From 47ab8f8dc5ec05f56f4ef9baa6947f3071d357dd Mon Sep 17 00:00:00 2001 From: SomberNight Date: Wed, 15 Apr 2020 19:34:52 +0200 Subject: [PATCH 079/117] daemon.on_stop: adapt to python 3.8 (py3.8 has breaking changes re asyncio.CancelledError and asyncio.TimeoutError) follow-up 308517d473d18fdc5a377e7296e0e6ab3f6c92b8 --- electrum/daemon.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/electrum/daemon.py b/electrum/daemon.py index 93b4baeb92..93a4f4dccb 100644 --- a/electrum/daemon.py +++ b/electrum/daemon.py @@ -32,6 +32,8 @@ from typing import Dict, Optional, Tuple, Iterable from base64 import b64decode, b64encode from collections import defaultdict +import concurrent +from concurrent import futures import aiohttp from aiohttp import web, client_exceptions @@ -507,7 +509,7 @@ def on_stop(self): fut = asyncio.run_coroutine_threadsafe(self.taskgroup.cancel_remaining(), self.asyncio_loop) try: fut.result(timeout=2) - except (asyncio.TimeoutError, asyncio.CancelledError): + except (concurrent.futures.TimeoutError, concurrent.futures.CancelledError, asyncio.CancelledError): pass self.logger.info("removing lockfile") remove_lockfile(get_lockfile(self.config)) From b5811e8072dafda7e2e740f8e5ca604680271291 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Wed, 15 Apr 2020 21:32:53 +0200 Subject: [PATCH 080/117] lnworker.peers: fix threading issues --- electrum/lnworker.py | 40 ++++++++++++++++++++--------------- electrum/tests/test_lnpeer.py | 4 ++++ 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/electrum/lnworker.py b/electrum/lnworker.py index 0dc047a776..f8d863db0d 100644 --- a/electrum/lnworker.py +++ b/electrum/lnworker.py @@ -7,7 +7,7 @@ from decimal import Decimal import random import time -from typing import Optional, Sequence, Tuple, List, Dict, TYPE_CHECKING, NamedTuple, Union +from typing import Optional, Sequence, Tuple, List, Dict, TYPE_CHECKING, NamedTuple, Union, Mapping import threading import socket import json @@ -150,8 +150,9 @@ def __init__(self, xprv): max_retry_delay_urgent=300, init_retry_delay_urgent=4, ) + self.lock = threading.RLock() self.node_keypair = generate_keypair(BIP32Node.from_xkey(xprv), LnKeyFamily.NODE_KEY) - self.peers = {} # type: Dict[bytes, Peer] # pubkey -> Peer + self._peers = {} # type: Dict[bytes, Peer] # pubkey -> Peer # needs self.lock self.taskgroup = SilentTaskGroup() # set some feature flags as baseline for both LNWallet and LNGossip # note that e.g. DATA_LOSS_PROTECT is needed for LNGossip as many peers require it @@ -161,6 +162,12 @@ def __init__(self, xprv): self.features |= LnFeatures.VAR_ONION_OPT self.features |= LnFeatures.PAYMENT_SECRET_OPT + @property + def peers(self) -> Mapping[bytes, Peer]: + """Returns a read-only copy of peers.""" + with self.lock: + return self._peers.copy() + def channels_for_peer(self, node_id): return {} @@ -180,7 +187,7 @@ async def cb(reader, writer): self.logger.info('handshake failure from incoming connection') return peer = Peer(self, node_id, transport) - self.peers[node_id] = peer + self._peers[node_id] = peer await self.taskgroup.spawn(peer.main_loop()) try: # FIXME: server.close(), server.wait_closed(), etc... ? @@ -205,7 +212,7 @@ async def _maintain_connectivity(self): while True: await asyncio.sleep(1) now = time.time() - if len(self.peers) >= NUM_PEERS_TARGET: + if len(self._peers) >= NUM_PEERS_TARGET: continue peers = await self._get_next_peers_to_try() for peer in peers: @@ -213,8 +220,8 @@ async def _maintain_connectivity(self): await self._add_peer(peer.host, peer.port, peer.pubkey) async def _add_peer(self, host: str, port: int, node_id: bytes) -> Peer: - if node_id in self.peers: - return self.peers[node_id] + if node_id in self._peers: + return self._peers[node_id] port = int(port) peer_addr = LNPeerAddr(host, port, node_id) transport = LNTransport(self.node_keypair.privkey, peer_addr) @@ -222,12 +229,12 @@ async def _add_peer(self, host: str, port: int, node_id: bytes) -> Peer: self.logger.info(f"adding peer {peer_addr}") peer = Peer(self, node_id, transport) await self.taskgroup.spawn(peer.main_loop()) - self.peers[node_id] = peer + self._peers[node_id] = peer return peer def peer_closed(self, peer: Peer) -> None: - if peer.pubkey in self.peers: - self.peers.pop(peer.pubkey) + if peer.pubkey in self._peers: + self._peers.pop(peer.pubkey) def num_peers(self) -> int: return sum([p.is_initialized() for p in self.peers.values()]) @@ -282,7 +289,7 @@ async def _get_next_peers_to_try(self) -> Sequence[LNPeerAddr]: for peer in recent_peers: if not peer: continue - if peer.pubkey in self.peers: + if peer.pubkey in self._peers: continue if not self._can_retry_addr(peer, now=now): continue @@ -442,7 +449,6 @@ def __init__(self, wallet: 'Abstract_Wallet', xprv): self.payments = self.db.get_dict('lightning_payments') # RHASH -> amount, direction, is_paid self.preimages = self.db.get_dict('lightning_preimages') # RHASH -> preimage self.sweep_address = wallet.get_receiving_address() - self.lock = threading.RLock() self.logs = defaultdict(list) # type: Dict[str, List[PaymentAttemptLog]] # key is RHASH # (not persisted) self.is_routing = set() # (not persisted) keys of invoices that are in PR_ROUTING state # used in tests @@ -680,12 +686,12 @@ async def on_channel_update(self, chan): await self.try_force_closing(chan.channel_id) elif chan.get_state() == ChannelState.FUNDED: - peer = self.peers.get(chan.node_id) + peer = self._peers.get(chan.node_id) if peer and peer.is_initialized(): peer.send_funding_locked(chan) elif chan.get_state() == ChannelState.OPEN: - peer = self.peers.get(chan.node_id) + peer = self._peers.get(chan.node_id) if peer: await peer.maybe_update_fee(chan) conf = self.lnwatcher.get_tx_height(chan.funding_outpoint.txid).conf @@ -736,7 +742,7 @@ def add_new_channel(self, chan): @log_exceptions async def add_peer(self, connect_str: str) -> Peer: node_id, rest = extract_nodeid(connect_str) - peer = self.peers.get(node_id) + peer = self._peers.get(node_id) if not peer: if rest is not None: host, port = split_host_port(rest) @@ -842,7 +848,7 @@ async def _pay(self, invoice, amount_sat=None, attempts=1) -> bool: async def _pay_to_route(self, route: LNPaymentRoute, lnaddr: LnAddr) -> PaymentAttemptLog: short_channel_id = route[0].short_channel_id chan = self.get_channel_by_short_id(short_channel_id) - peer = self.peers.get(route[0].node_id) + peer = self._peers.get(route[0].node_id) if not peer: raise Exception('Dropped peer') await peer.initialized @@ -1238,7 +1244,7 @@ def num_sats_can_receive(self) -> Union[Decimal, int]: async def close_channel(self, chan_id): chan = self.channels[chan_id] - peer = self.peers[chan.node_id] + peer = self._peers[chan.node_id] return await peer.close_channel(chan_id) async def force_close_channel(self, chan_id): @@ -1299,7 +1305,7 @@ async def reestablish_peers_and_channels(self): # reestablish if not chan.should_try_to_reestablish_peer(): continue - peer = self.peers.get(chan.node_id, None) + peer = self._peers.get(chan.node_id, None) if peer: await peer.taskgroup.spawn(peer.reestablish_channel(chan)) else: diff --git a/electrum/tests/test_lnpeer.py b/electrum/tests/test_lnpeer.py index af2db0611f..51237cc2e4 100644 --- a/electrum/tests/test_lnpeer.py +++ b/electrum/tests/test_lnpeer.py @@ -124,6 +124,10 @@ def lock(self): @property def peers(self): + return self._peers + + @property + def _peers(self): return {self.remote_keypair.pubkey: self.peer} def channels_for_peer(self, pubkey): From 223b62554ead397bb94013c0d9c95b63a0708ea6 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Wed, 15 Apr 2020 21:41:33 +0200 Subject: [PATCH 081/117] lntransport: use network proxy if available fixes #4824 --- electrum/interface.py | 21 ++------------------ electrum/lnpeer.py | 3 ++- electrum/lntransport.py | 13 ++++++++++--- electrum/lnworker.py | 13 +++++++++++-- electrum/tests/test_lntransport.py | 2 +- electrum/util.py | 31 ++++++++++++++++++++++++++++++ 6 files changed, 57 insertions(+), 26 deletions(-) diff --git a/electrum/interface.py b/electrum/interface.py index 86fa5e0c8d..26677aefee 100644 --- a/electrum/interface.py +++ b/electrum/interface.py @@ -43,7 +43,7 @@ from aiorpcx.rawsocket import RSClient import certifi -from .util import ignore_exceptions, log_exceptions, bfh, SilentTaskGroup +from .util import ignore_exceptions, log_exceptions, bfh, SilentTaskGroup, MySocksProxy from . import util from . import x509 from . import pem @@ -277,7 +277,7 @@ def __init__(self, *, network: 'Network', server: ServerAddr, proxy: Optional[di self.blockchain = None # type: Optional[Blockchain] self._requested_chunks = set() # type: Set[int] self.network = network - self._set_proxy(proxy) + self.proxy = MySocksProxy.from_proxy_dict(proxy) self.session = None # type: Optional[NotificationSession] self._ipaddr_bucket = None @@ -310,23 +310,6 @@ def diagnostic_name(self): def __str__(self): return f"" - def _set_proxy(self, proxy: dict): - if proxy: - username, pw = proxy.get('user'), proxy.get('password') - if not username or not pw: - auth = None - else: - auth = aiorpcx.socks.SOCKSUserAuth(username, pw) - addr = NetAddress(proxy['host'], proxy['port']) - if proxy['mode'] == "socks4": - self.proxy = aiorpcx.socks.SOCKSProxy(addr, aiorpcx.socks.SOCKS4a, auth) - elif proxy['mode'] == "socks5": - self.proxy = aiorpcx.socks.SOCKSProxy(addr, aiorpcx.socks.SOCKS5, auth) - else: - raise NotImplementedError # http proxy not available with aiorpcx - else: - self.proxy = None - async def is_server_ca_signed(self, ca_ssl_context): """Given a CA enforcing SSL context, returns True if the connection can be established. Returns False if the server has a self-signed diff --git a/electrum/lnpeer.py b/electrum/lnpeer.py index 4479234eaa..e92a30cb68 100644 --- a/electrum/lnpeer.py +++ b/electrum/lnpeer.py @@ -251,7 +251,8 @@ async def wrapper_func(self, *args, **kwargs): return await func(self, *args, **kwargs) except GracefulDisconnect as e: self.logger.log(e.log_level, f"Disconnecting: {repr(e)}") - except (LightningPeerConnectionClosed, IncompatibleLightningFeatures) as e: + except (LightningPeerConnectionClosed, IncompatibleLightningFeatures, + aiorpcx.socks.SOCKSError) as e: self.logger.info(f"Disconnecting: {repr(e)}") finally: self.close_and_cleanup() diff --git a/electrum/lntransport.py b/electrum/lntransport.py index 257f02b12a..a28d2da24b 100644 --- a/electrum/lntransport.py +++ b/electrum/lntransport.py @@ -8,12 +8,14 @@ import hashlib import asyncio from asyncio import StreamReader, StreamWriter +from typing import Optional from .crypto import sha256, hmac_oneshot, chacha20_poly1305_encrypt, chacha20_poly1305_decrypt from .lnutil import (get_ecdh, privkey_to_pubkey, LightningPeerConnectionClosed, HandshakeFailed, LNPeerAddr) from . import ecc -from .util import bh2u +from .util import bh2u, MySocksProxy + class HandshakeState(object): prologue = b"lightning" @@ -217,17 +219,22 @@ async def handshake(self, **kwargs): class LNTransport(LNTransportBase): """Transport initiated by local party.""" - def __init__(self, privkey: bytes, peer_addr: LNPeerAddr): + def __init__(self, privkey: bytes, peer_addr: LNPeerAddr, *, + proxy: Optional[dict]): LNTransportBase.__init__(self) assert type(privkey) is bytes and len(privkey) == 32 self.privkey = privkey self.peer_addr = peer_addr + self.proxy = MySocksProxy.from_proxy_dict(proxy) def name(self): return self.peer_addr.net_addr_str() async def handshake(self): - self.reader, self.writer = await asyncio.open_connection(self.peer_addr.host, self.peer_addr.port) + if not self.proxy: + self.reader, self.writer = await asyncio.open_connection(self.peer_addr.host, self.peer_addr.port) + else: + self.reader, self.writer = await self.proxy.open_connection(self.peer_addr.host, self.peer_addr.port) hs = HandshakeState(self.peer_addr.pubkey) # Get a new ephemeral key epriv, epub = create_ephemeral_key() diff --git a/electrum/lnworker.py b/electrum/lnworker.py index f8d863db0d..49999b7033 100644 --- a/electrum/lnworker.py +++ b/electrum/lnworker.py @@ -162,6 +162,8 @@ def __init__(self, xprv): self.features |= LnFeatures.VAR_ONION_OPT self.features |= LnFeatures.PAYMENT_SECRET_OPT + util.register_callback(self.on_proxy_changed, ['proxy_set']) + @property def peers(self) -> Mapping[bytes, Peer]: """Returns a read-only copy of peers.""" @@ -191,6 +193,7 @@ async def cb(reader, writer): await self.taskgroup.spawn(peer.main_loop()) try: # FIXME: server.close(), server.wait_closed(), etc... ? + # TODO: onion hidden service? server = await asyncio.start_server(cb, addr, int(port)) except OSError as e: self.logger.error(f"cannot listen for lightning p2p. error: {e!r}") @@ -224,7 +227,8 @@ async def _add_peer(self, host: str, port: int, node_id: bytes) -> Peer: return self._peers[node_id] port = int(port) peer_addr = LNPeerAddr(host, port, node_id) - transport = LNTransport(self.node_keypair.privkey, peer_addr) + transport = LNTransport(self.node_keypair.privkey, peer_addr, + proxy=self.network.proxy) self._trying_addr_now(peer_addr) self.logger.info(f"adding peer {peer_addr}") peer = Peer(self, node_id, transport) @@ -381,6 +385,10 @@ def choose_preferred_address(addr_list: Sequence[Tuple[str, int, int]]) -> Tuple choice = random.choice(addr_list) return choice + def on_proxy_changed(self, event, *args): + for peer in self.peers.values(): + peer.close_and_cleanup() + class LNGossip(LNWorker): max_age = 14*24*3600 @@ -1415,7 +1423,8 @@ def remove_channel_backup(self, channel_id): async def request_force_close(self, channel_id): cb = self.channel_backups[channel_id].cb peer_addr = LNPeerAddr(cb.host, cb.port, cb.node_id) - transport = LNTransport(cb.privkey, peer_addr) + transport = LNTransport(cb.privkey, peer_addr, + proxy=self.network.proxy) peer = Peer(self, cb.node_id, transport) await self.taskgroup.spawn(peer._message_loop()) await peer.initialized diff --git a/electrum/tests/test_lntransport.py b/electrum/tests/test_lntransport.py index dff48ca5c1..2dc1b4950d 100644 --- a/electrum/tests/test_lntransport.py +++ b/electrum/tests/test_lntransport.py @@ -57,7 +57,7 @@ async def cb(reader, writer): server = server_future.result() # type: asyncio.Server async def connect(): peer_addr = LNPeerAddr('127.0.0.1', 42898, responder_key.get_public_key_bytes()) - t = LNTransport(initiator_key.get_secret_bytes(), peer_addr) + t = LNTransport(initiator_key.get_secret_bytes(), peer_addr, proxy=None) await t.handshake() t.send_bytes(b'hello from client') self.assertEqual(await t.read_messages().__anext__(), b'hello from server') diff --git a/electrum/util.py b/electrum/util.py index 8e7d441d27..6f998bb7f9 100644 --- a/electrum/util.py +++ b/electrum/util.py @@ -46,6 +46,7 @@ import aiohttp from aiohttp_socks import ProxyConnector, ProxyType +import aiorpcx from aiorpcx import TaskGroup import certifi import dns.resolver @@ -1397,3 +1398,33 @@ def _can_retry_addr(self, peer: _NetAddrType, *, def _clear_addr_retry_times(self) -> None: self._last_tried_addr.clear() + + +class MySocksProxy(aiorpcx.SOCKSProxy): + + async def open_connection(self, host=None, port=None, **kwargs): + loop = asyncio.get_event_loop() + reader = asyncio.StreamReader(loop=loop) + protocol = asyncio.StreamReaderProtocol(reader, loop=loop) + transport, _ = await self.create_connection( + lambda: protocol, host, port, **kwargs) + writer = asyncio.StreamWriter(transport, protocol, reader, loop) + return reader, writer + + @classmethod + def from_proxy_dict(cls, proxy: dict = None) -> Optional['MySocksProxy']: + if not proxy: + return None + username, pw = proxy.get('user'), proxy.get('password') + if not username or not pw: + auth = None + else: + auth = aiorpcx.socks.SOCKSUserAuth(username, pw) + addr = aiorpcx.NetAddress(proxy['host'], proxy['port']) + if proxy['mode'] == "socks4": + ret = cls(addr, aiorpcx.socks.SOCKS4a, auth) + elif proxy['mode'] == "socks5": + ret = cls(addr, aiorpcx.socks.SOCKS5, auth) + else: + raise NotImplementedError # http proxy not available with aiorpcx + return ret From 95fa5d37c365165aaf3f7af943bb6ffc86b955d8 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Wed, 15 Apr 2020 22:41:16 +0200 Subject: [PATCH 082/117] lnworker.peers: follow-up b5811e8072dafda7e2e740f8e5ca604680271291 somehow I forgot writes... --- electrum/lnworker.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/electrum/lnworker.py b/electrum/lnworker.py index 49999b7033..df777ef002 100644 --- a/electrum/lnworker.py +++ b/electrum/lnworker.py @@ -189,7 +189,8 @@ async def cb(reader, writer): self.logger.info('handshake failure from incoming connection') return peer = Peer(self, node_id, transport) - self._peers[node_id] = peer + with self.lock: + self._peers[node_id] = peer await self.taskgroup.spawn(peer.main_loop()) try: # FIXME: server.close(), server.wait_closed(), etc... ? @@ -233,12 +234,13 @@ async def _add_peer(self, host: str, port: int, node_id: bytes) -> Peer: self.logger.info(f"adding peer {peer_addr}") peer = Peer(self, node_id, transport) await self.taskgroup.spawn(peer.main_loop()) - self._peers[node_id] = peer + with self.lock: + self._peers[node_id] = peer return peer def peer_closed(self, peer: Peer) -> None: - if peer.pubkey in self._peers: - self._peers.pop(peer.pubkey) + with self.lock: + self._peers.pop(peer.pubkey, None) def num_peers(self) -> int: return sum([p.is_initialized() for p in self.peers.values()]) From 82da581d45cd1570fa5a361681324b6cee507ab5 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Wed, 15 Apr 2020 22:47:14 +0200 Subject: [PATCH 083/117] lnworker: clear peer retry times if proxy settings change maybe there were failures due to the previous proxy details being incorrect --- electrum/lnworker.py | 1 + 1 file changed, 1 insertion(+) diff --git a/electrum/lnworker.py b/electrum/lnworker.py index df777ef002..c0460b937b 100644 --- a/electrum/lnworker.py +++ b/electrum/lnworker.py @@ -390,6 +390,7 @@ def choose_preferred_address(addr_list: Sequence[Tuple[str, int, int]]) -> Tuple def on_proxy_changed(self, event, *args): for peer in self.peers.values(): peer.close_and_cleanup() + self._clear_addr_retry_times() class LNGossip(LNWorker): From c454564ed6e094aa36be9fbe7aa9e9113a28710c Mon Sep 17 00:00:00 2001 From: ThomasV Date: Thu, 16 Apr 2020 10:58:40 +0200 Subject: [PATCH 084/117] sql_db: do not require network object --- electrum/channel_db.py | 2 +- electrum/sql_db.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/electrum/channel_db.py b/electrum/channel_db.py index e374519d5e..50cb212347 100644 --- a/electrum/channel_db.py +++ b/electrum/channel_db.py @@ -242,7 +242,7 @@ class ChannelDB(SqlDB): def __init__(self, network: 'Network'): path = os.path.join(get_headers_dir(network.config), 'gossip_db') - super().__init__(network, path, commit_interval=100) + super().__init__(network.asyncio_loop, path, commit_interval=100) self.lock = threading.RLock() self.num_nodes = 0 self.num_channels = 0 diff --git a/electrum/sql_db.py b/electrum/sql_db.py index 8cd793c0c7..fddd03b233 100644 --- a/electrum/sql_db.py +++ b/electrum/sql_db.py @@ -19,9 +19,9 @@ def wrapper(self, *args, **kwargs): class SqlDB(Logger): - def __init__(self, network, path, commit_interval=None): + def __init__(self, asyncio_loop, path, commit_interval=None): Logger.__init__(self) - self.network = network + self.asyncio_loop = asyncio_loop self.path = path self.commit_interval = commit_interval self.db_requests = queue.Queue() @@ -34,7 +34,7 @@ def run_sql(self): self.logger.info("Creating database") self.create_database() i = 0 - while self.network.asyncio_loop.is_running(): + while self.asyncio_loop.is_running(): try: future, func, args, kwargs = self.db_requests.get(timeout=0.1) except queue.Empty: From ef5ad5f22f3d42d0dc7e1e81eaacaad84b3fcda2 Mon Sep 17 00:00:00 2001 From: ThomasV Date: Thu, 16 Apr 2020 12:39:12 +0200 Subject: [PATCH 085/117] extend 'add_peer', 'list_peers' commands to gossip --- electrum/commands.py | 13 ++++++++---- electrum/lnpeer.py | 9 ++++---- electrum/lnworker.py | 49 +++++++++++++++++++++----------------------- 3 files changed, 37 insertions(+), 34 deletions(-) diff --git a/electrum/commands.py b/electrum/commands.py index 76fe1d7c90..c9db66a61a 100644 --- a/electrum/commands.py +++ b/electrum/commands.py @@ -53,6 +53,7 @@ from .address_synchronizer import TX_HEIGHT_LOCAL from .mnemonic import Mnemonic from .lnutil import SENT, RECEIVED +from .lnutil import LnFeatures from .lnutil import ln_dummy_address from .lnpeer import channel_id_from_funding_tx from .plugin import run_hook @@ -965,18 +966,21 @@ async def help(self): # lightning network commands @command('wn') - async def add_peer(self, connection_string, timeout=20, wallet: Abstract_Wallet = None): - await wallet.lnworker.add_peer(connection_string) + async def add_peer(self, connection_string, timeout=20, gossip=False, wallet: Abstract_Wallet = None): + lnworker = self.network.lngossip if gossip else wallet.lnworker + await lnworker.add_peer(connection_string) return True @command('wn') - async def list_peers(self, wallet: Abstract_Wallet = None): + async def list_peers(self, gossip=False, wallet: Abstract_Wallet = None): + lnworker = self.network.lngossip if gossip else wallet.lnworker return [{ 'node_id':p.pubkey.hex(), 'address':p.transport.name(), 'initialized':p.is_initialized(), + 'features': str(LnFeatures(p.features)), 'channels': [c.funding_outpoint.to_str() for c in p.channels.values()], - } for p in wallet.lnworker.peers.values()] + } for p in lnworker.peers.values()] @command('wpn') async def open_channel(self, connection_string, amount, push_amount=0, password=None, wallet: Abstract_Wallet = None): @@ -1165,6 +1169,7 @@ def eval_bool(x: str) -> bool: 'from_height': (None, "Only show transactions that confirmed after given block height"), 'to_height': (None, "Only show transactions that confirmed before given block height"), 'iknowwhatimdoing': (None, "Acknowledge that I understand the full implications of what I am about to do"), + 'gossip': (None, "Apply command to gossip node instead of wallet"), } diff --git a/electrum/lnpeer.py b/electrum/lnpeer.py index e92a30cb68..1dcdded2cf 100644 --- a/electrum/lnpeer.py +++ b/electrum/lnpeer.py @@ -74,6 +74,7 @@ def __init__(self, lnworker: Union['LNGossip', 'LNWallet'], pubkey:bytes, transp self.lnworker = lnworker self.privkey = self.transport.privkey # local privkey self.features = self.lnworker.features + self.their_features = 0 self.node_ids = [self.pubkey, privkey_to_pubkey(self.privkey)] self.network = lnworker.network self.channel_db = lnworker.network.channel_db @@ -200,15 +201,15 @@ def on_init(self, payload): if self._received_init: self.logger.info("ALREADY INITIALIZED BUT RECEIVED INIT") return - their_features = LnFeatures(int.from_bytes(payload['features'], byteorder="big")) + self.their_features = LnFeatures(int.from_bytes(payload['features'], byteorder="big")) their_globalfeatures = int.from_bytes(payload['globalfeatures'], byteorder="big") - their_features |= their_globalfeatures + self.their_features |= their_globalfeatures # check transitive dependencies for received features - if not their_features.validate_transitive_dependecies(): + if not self.their_features.validate_transitive_dependecies(): raise GracefulDisconnect("remote did not set all dependencies for the features they sent") # check if features are compatible, and set self.features to what we negotiated try: - self.features = ln_compare_features(self.features, their_features) + self.features = ln_compare_features(self.features, self.their_features) except IncompatibleLightningFeatures as e: self.initialized.set_exception(e) raise GracefulDisconnect(f"{str(e)}") diff --git a/electrum/lnworker.py b/electrum/lnworker.py index c0460b937b..889acad5e0 100644 --- a/electrum/lnworker.py +++ b/electrum/lnworker.py @@ -392,6 +392,29 @@ def on_proxy_changed(self, event, *args): peer.close_and_cleanup() self._clear_addr_retry_times() + @log_exceptions + async def add_peer(self, connect_str: str) -> Peer: + node_id, rest = extract_nodeid(connect_str) + peer = self._peers.get(node_id) + if not peer: + if rest is not None: + host, port = split_host_port(rest) + else: + addrs = self.channel_db.get_node_addresses(node_id) + if not addrs: + raise ConnStringFormatError(_('Don\'t know any addresses for node:') + ' ' + bh2u(node_id)) + host, port, timestamp = self.choose_preferred_address(addrs) + port = int(port) + # Try DNS-resolving the host (if needed). This is simply so that + # the caller gets a nice exception if it cannot be resolved. + try: + await asyncio.get_event_loop().getaddrinfo(host, port) + except socket.gaierror: + raise ConnStringFormatError(_('Hostname does not resolve (getaddrinfo failed)')) + # add peer + peer = await self._add_peer(host, port, node_id) + return peer + class LNGossip(LNWorker): max_age = 14*24*3600 @@ -716,9 +739,6 @@ async def on_channel_update(self, chan): self.logger.info('REBROADCASTING CLOSING TX') await self.network.try_broadcasting(force_close_tx, 'force-close') - - - @log_exceptions async def _open_channel_coroutine(self, *, connect_str: str, funding_tx: PartialTransaction, funding_sat: int, push_sat: int, @@ -750,29 +770,6 @@ def add_new_channel(self, chan): channels_db[chan.channel_id.hex()] = chan.storage self.wallet.save_backup() - @log_exceptions - async def add_peer(self, connect_str: str) -> Peer: - node_id, rest = extract_nodeid(connect_str) - peer = self._peers.get(node_id) - if not peer: - if rest is not None: - host, port = split_host_port(rest) - else: - addrs = self.channel_db.get_node_addresses(node_id) - if not addrs: - raise ConnStringFormatError(_('Don\'t know any addresses for node:') + ' ' + bh2u(node_id)) - host, port, timestamp = self.choose_preferred_address(addrs) - port = int(port) - # Try DNS-resolving the host (if needed). This is simply so that - # the caller gets a nice exception if it cannot be resolved. - try: - await asyncio.get_event_loop().getaddrinfo(host, port) - except socket.gaierror: - raise ConnStringFormatError(_('Hostname does not resolve (getaddrinfo failed)')) - # add peer - peer = await self._add_peer(host, port, node_id) - return peer - def mktx_for_open_channel(self, *, coins: Sequence[PartialTxInput], funding_sat: int, fee_est=None) -> PartialTransaction: dummy_address = ln_dummy_address() From ea64b2af64a29d4f640714e60a136cf4e00d6d61 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Thu, 16 Apr 2020 17:31:58 +0200 Subject: [PATCH 086/117] interface.get_certificate: use public asyncio APIs --- electrum/interface.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/electrum/interface.py b/electrum/interface.py index 26677aefee..f17a340179 100644 --- a/electrum/interface.py +++ b/electrum/interface.py @@ -463,13 +463,12 @@ async def save_certificate(self): async def get_certificate(self): sslc = ssl.SSLContext() - try: - async with _RSClient(session_factory=RPCSession, - host=self.host, port=self.port, - ssl=sslc, proxy=self.proxy) as session: - return session.transport._asyncio_transport._ssl_protocol._sslpipe._sslobj.getpeercert(True) - except ValueError: - return None + async with _RSClient(session_factory=RPCSession, + host=self.host, port=self.port, + ssl=sslc, proxy=self.proxy) as session: + asyncio_transport = session.transport._asyncio_transport # type: asyncio.BaseTransport + ssl_object = asyncio_transport.get_extra_info("ssl_object") # type: ssl.SSLObject + return ssl_object.getpeercert(binary_form=True) async def get_block_header(self, height, assert_mode): self.logger.info(f'requesting block header {height} in mode {assert_mode}') From 872380a5259020b1140884b7214ec3d44f86222d Mon Sep 17 00:00:00 2001 From: Luke Childs Date: Thu, 16 Apr 2020 22:49:48 +0700 Subject: [PATCH 087/117] Add electrum_data to .gitignore (#6092) --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index cfcca9979c..67101964b5 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ bin/ .idea .mypy_cache .vscode +electrum_data # icons electrum/gui/kivy/theming/light-0.png From adc3784bc24ab0e83e412846a4bfc776b33feef7 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Thu, 16 Apr 2020 19:56:30 +0200 Subject: [PATCH 088/117] network: allow mixed protocols among interfaces Previously all the interfaces used either "t" or "s". Now the network only tries to use "s" for all interfaces, except for the main interface, which the user can manually specify to use "t". (so e.g. if you run with "--server localhost:50002:t", the main server will use "t", but all the rest will use "s") --- electrum/gui/qt/network_dialog.py | 34 ++++++++-------------------- electrum/network.py | 37 +++++++++++++++++++------------ electrum/scripts/peers.py | 2 +- electrum/scripts/txradar.py | 2 +- 4 files changed, 34 insertions(+), 41 deletions(-) diff --git a/electrum/gui/qt/network_dialog.py b/electrum/gui/qt/network_dialog.py index bbf2df91f8..e8cf3d6e3d 100644 --- a/electrum/gui/qt/network_dialog.py +++ b/electrum/gui/qt/network_dialog.py @@ -37,7 +37,7 @@ from electrum.i18n import _ from electrum import constants, blockchain, util from electrum.interface import ServerAddr -from electrum.network import Network +from electrum.network import Network, PREFERRED_NETWORK_PROTOCOL from electrum.logging import get_logger from .util import (Buttons, CloseButton, HelpButton, read_QIcon, char_width_in_lineedit, @@ -72,6 +72,8 @@ def on_update(self): class NodesListWidget(QTreeWidget): + """List of connected servers.""" + SERVER_ADDR_ROLE = Qt.UserRole + 100 CHAIN_ID_ROLE = Qt.UserRole + 101 IS_SERVER_ROLE = Qt.UserRole + 102 @@ -129,6 +131,7 @@ def update(self, network: Network): item = QTreeWidgetItem([i.host + star, '%d'%i.tip]) item.setData(0, self.IS_SERVER_ROLE, 1) item.setData(0, self.SERVER_ADDR_ROLE, i.server) + item.setToolTip(0, str(i.server)) x.addChild(item) if n_chains > 1: self.addTopLevelItem(x) @@ -143,6 +146,8 @@ def update(self, network: Network): class ServerListWidget(QTreeWidget): + """List of all known servers.""" + class Columns(IntEnum): HOST = 0 PORT = 1 @@ -182,8 +187,9 @@ def on_activated(self, item, column): pt.setX(50) self.customContextMenuRequested.emit(pt) - def update(self, servers, protocol, use_tor): + def update(self, servers, use_tor): self.clear() + protocol = PREFERRED_NETWORK_PROTOCOL for _host, d in sorted(servers.items()): if _host.endswith('.onion') and not use_tor: continue @@ -207,7 +213,6 @@ class NetworkChoiceLayout(object): def __init__(self, network: Network, config, wizard=False): self.network = network self.config = config - self.protocol = None self.tor_proxy = None self.tabs = tabs = QTabWidget() @@ -370,9 +375,8 @@ def update(self): host = interface.host if interface else _('None') self.server_label.setText(host) - self.set_protocol(protocol) self.servers = self.network.get_servers() - self.servers_list.update(self.servers, self.protocol, self.tor_cb.isChecked()) + self.servers_list.update(self.servers, self.tor_cb.isChecked()) self.enable_set_server() height_str = "%d "%(self.network.get_local_height()) + _('blocks') @@ -413,22 +417,6 @@ def fill_in_proxy_settings(self): def layout(self): return self.layout_ - def set_protocol(self, protocol): - if protocol != self.protocol: - self.protocol = protocol - - def change_protocol(self, use_ssl): - p = 's' if use_ssl else 't' - host = self.server_host.text() - pp = self.servers.get(host, constants.net.DEFAULT_PORTS) - if p not in pp.keys(): - p = list(pp.keys())[0] - port = pp[p] - self.server_host.setText(host) - self.server_port.setText(port) - self.set_protocol(p) - self.set_server() - def follow_branch(self, chain_id): self.network.run_from_another_thread(self.network.follow_chain_given_id(chain_id)) self.update() @@ -437,10 +425,6 @@ def follow_server(self, server: ServerAddr): self.network.run_from_another_thread(self.network.follow_chain_given_server(server)) self.update() - def server_changed(self, x): - if x: - self.change_server(str(x.text(0)), self.protocol) - def change_server(self, host, protocol): pp = self.servers.get(host, constants.net.DEFAULT_PORTS) if protocol and protocol not in protocol_letters: diff --git a/electrum/network.py b/electrum/network.py index 09c2c6ad3d..e7cdb68502 100644 --- a/electrum/network.py +++ b/electrum/network.py @@ -75,6 +75,10 @@ NUM_STICKY_SERVERS = 4 NUM_RECENT_SERVERS = 20 +_KNOWN_NETWORK_PROTOCOLS = {'t', 's'} +PREFERRED_NETWORK_PROTOCOL = 's' +assert PREFERRED_NETWORK_PROTOCOL in _KNOWN_NETWORK_PROTOCOLS + def parse_servers(result: Sequence[Tuple[str, str, List[str]]]) -> Dict[str, dict]: """ parse servers list into dict format""" @@ -115,23 +119,27 @@ def filter_noonion(servers): return {k: v for k, v in servers.items() if not k.endswith('.onion')} -def filter_protocol(hostmap, protocol='s') -> Sequence[ServerAddr]: +def filter_protocol(hostmap, *, allowed_protocols: Iterable[str] = None) -> Sequence[ServerAddr]: """Filters the hostmap for those implementing protocol.""" + if allowed_protocols is None: + allowed_protocols = {PREFERRED_NETWORK_PROTOCOL} eligible = [] for host, portmap in hostmap.items(): - port = portmap.get(protocol) - if port: - eligible.append(ServerAddr(host, port, protocol=protocol)) + for protocol in allowed_protocols: + port = portmap.get(protocol) + if port: + eligible.append(ServerAddr(host, port, protocol=protocol)) return eligible -def pick_random_server(hostmap=None, *, protocol='s', +def pick_random_server(hostmap=None, *, allowed_protocols: Iterable[str], exclude_set: Set[ServerAddr] = None) -> Optional[ServerAddr]: if hostmap is None: hostmap = constants.net.DEFAULT_SERVERS if exclude_set is None: exclude_set = set() - eligible = list(set(filter_protocol(hostmap, protocol)) - exclude_set) + servers = set(filter_protocol(hostmap, allowed_protocols=allowed_protocols)) + eligible = list(servers - exclude_set) return random.choice(eligible) if eligible else None @@ -273,6 +281,9 @@ def __init__(self, config: SimpleConfig, *, daemon: 'Daemon' = None): self.logger.info(f"blockchains {list(map(lambda b: b.forkpoint, blockchain.blockchains.values()))}") self._blockchain_preferred_block = self.config.get('blockchain_preferred_block', None) # type: Optional[Dict] self._blockchain = blockchain.get_best_chain() + + self._allowed_protocols = {PREFERRED_NETWORK_PROTOCOL} + # Server for addresses and transactions self.default_server = self.config.get('server', None) # Sanitize default server @@ -283,7 +294,7 @@ def __init__(self, config: SimpleConfig, *, daemon: 'Daemon' = None): self.logger.warning('failed to parse server-string; falling back to localhost.') self.default_server = ServerAddr.from_str("localhost:50002:s") else: - self.default_server = pick_random_server() + self.default_server = pick_random_server(allowed_protocols=self._allowed_protocols) assert isinstance(self.default_server, ServerAddr), f"invalid type for default_server: {self.default_server!r}" self.taskgroup = None @@ -549,7 +560,7 @@ def _get_next_server_to_try(self) -> Optional[ServerAddr]: # we only give priority to recent_servers up to NUM_STICKY_SERVERS. with self.recent_servers_lock: recent_servers = list(self._recent_servers) - recent_servers = [s for s in recent_servers if s.protocol == self.protocol] + recent_servers = [s for s in recent_servers if s.protocol in self._allowed_protocols] if len(connected_servers & set(recent_servers)) < NUM_STICKY_SERVERS: for server in recent_servers: if server in connected_servers: @@ -559,7 +570,7 @@ def _get_next_server_to_try(self) -> Optional[ServerAddr]: return server # try all servers we know about, pick one at random hostmap = self.get_servers() - servers = list(set(filter_protocol(hostmap, self.protocol)) - connected_servers) + servers = list(set(filter_protocol(hostmap, allowed_protocols=self._allowed_protocols)) - connected_servers) random.shuffle(servers) for server in servers: if not self._can_retry_addr(server, now=now): @@ -574,7 +585,7 @@ def _set_proxy(self, proxy: Optional[dict]): util.trigger_callback('proxy_set', self.proxy) @log_exceptions - async def set_parameters(self, net_params: NetworkParameters): + async def set_parameters(self, net_params: NetworkParameters): # TODO proxy = net_params.proxy proxy_str = serialize_proxy(proxy) host, port, protocol = net_params.host, net_params.port, net_params.protocol @@ -598,7 +609,7 @@ async def set_parameters(self, net_params: NetworkParameters): async with self.restart_lock: self.auto_connect = net_params.auto_connect - if self.proxy != proxy or self.protocol != protocol or self.oneserver != net_params.oneserver: + if self.proxy != proxy or self.oneserver != net_params.oneserver: # Restart the network defaulting to the given server await self._stop() self.default_server = server @@ -1138,7 +1149,6 @@ async def _start(self): assert not self._connecting self.logger.info('starting network') self._clear_addr_retry_times() - self.protocol = self.default_server.protocol self._set_proxy(deserialize_proxy(self.config.get('proxy'))) self._set_oneserver(self.config.get('oneserver', False)) await self.taskgroup.spawn(self._run_new_interface(self.default_server)) @@ -1282,7 +1292,7 @@ async def get_peers(self): session = self.interface.session return parse_servers(await session.send_request('server.peers.subscribe')) - async def send_multiple_requests(self, servers: List[str], method: str, params: Sequence): + async def send_multiple_requests(self, servers: Sequence[ServerAddr], method: str, params: Sequence): responses = dict() async def get_response(server: ServerAddr): interface = Interface(network=self, server=server, proxy=self.proxy) @@ -1299,6 +1309,5 @@ async def get_response(server: ServerAddr): responses[interface.server] = res async with TaskGroup() as group: for server in servers: - server = ServerAddr.from_str(server) await group.spawn(get_response(server)) return responses diff --git a/electrum/scripts/peers.py b/electrum/scripts/peers.py index a26572b307..64c25e0cdf 100755 --- a/electrum/scripts/peers.py +++ b/electrum/scripts/peers.py @@ -17,7 +17,7 @@ async def f(): try: peers = await network.get_peers() - peers = filter_protocol(peers, 's') + peers = filter_protocol(peers) results = await network.send_multiple_requests(peers, 'blockchain.headers.subscribe', []) for server, header in sorted(results.items(), key=lambda x: x[1].get('height')): height = header.get('height') diff --git a/electrum/scripts/txradar.py b/electrum/scripts/txradar.py index 2166a87ba4..8e301fd5de 100755 --- a/electrum/scripts/txradar.py +++ b/electrum/scripts/txradar.py @@ -23,7 +23,7 @@ async def f(): try: peers = await network.get_peers() - peers = filter_protocol(peers, 's') + peers = filter_protocol(peers) results = await network.send_multiple_requests(peers, 'blockchain.transaction.get', [txid]) r1, r2 = [], [] for k, v in results.items(): From b2cfaddff25b93189e8fb26f04fe9667958fe2d6 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Thu, 16 Apr 2020 20:30:53 +0200 Subject: [PATCH 089/117] network.NetworkParameters: merge host+port+protocol into "server" field --- electrum/commands.py | 2 +- electrum/gui/kivy/main_window.py | 19 +++++++++++++++--- electrum/gui/kivy/uix/ui_screens/server.kv | 10 +++++----- electrum/gui/qt/main_window.py | 2 +- electrum/gui/qt/network_dialog.py | 15 +++++++++----- electrum/gui/text.py | 9 ++++----- electrum/network.py | 23 ++++++---------------- 7 files changed, 43 insertions(+), 37 deletions(-) diff --git a/electrum/commands.py b/electrum/commands.py index c9db66a61a..5dca403161 100644 --- a/electrum/commands.py +++ b/electrum/commands.py @@ -187,7 +187,7 @@ async def getinfo(self): net_params = self.network.get_parameters() response = { 'path': self.network.config.path, - 'server': net_params.host, + 'server': net_params.server.host, 'blockchain_height': self.network.get_local_height(), 'server_height': self.network.get_server_height(), 'spv_nodes': len(self.network.get_interfaces()), diff --git a/electrum/gui/kivy/main_window.py b/electrum/gui/kivy/main_window.py index 120a5f66ae..7cf50e46aa 100644 --- a/electrum/gui/kivy/main_window.py +++ b/electrum/gui/kivy/main_window.py @@ -145,6 +145,19 @@ def cb2(host): servers = self.network.get_servers() ChoiceDialog(_('Choose a server'), sorted(servers), popup.ids.host.text, cb2).open() + def maybe_switch_to_server(self, *, host: str, port: str): + from electrum.interface import ServerAddr + net_params = self.network.get_parameters() + try: + server = ServerAddr(host=host, + port=port, + protocol=net_params.server.protocol) + except Exception as e: + self.show_error(_("Invalid server details: {}").format(repr(e))) + return + net_params = net_params._replace(server=server) + self.network.run_from_another_thread(self.network.set_parameters(net_params)) + def choose_blockchain_dialog(self, dt): from .uix.dialogs.choice_dialog import ChoiceDialog chains = self.network.get_blockchains() @@ -348,8 +361,8 @@ def __init__(self, **kwargs): self.num_blocks = self.network.get_local_height() self.num_nodes = len(self.network.get_interfaces()) net_params = self.network.get_parameters() - self.server_host = net_params.host - self.server_port = net_params.port + self.server_host = net_params.server.host + self.server_port = str(net_params.server.port) self.auto_connect = net_params.auto_connect self.oneserver = net_params.oneserver self.proxy_config = net_params.proxy if net_params.proxy else {} @@ -814,7 +827,7 @@ def update_interfaces(self, dt): if interface: self.server_host = interface.host else: - self.server_host = str(net_params.host) + ' (connecting...)' + self.server_host = str(net_params.server.host) + ' (connecting...)' self.proxy_config = net_params.proxy or {} self.update_proxy_str(self.proxy_config) diff --git a/electrum/gui/kivy/uix/ui_screens/server.kv b/electrum/gui/kivy/uix/ui_screens/server.kv index 67ce06750d..aee4395355 100644 --- a/electrum/gui/kivy/uix/ui_screens/server.kv +++ b/electrum/gui/kivy/uix/ui_screens/server.kv @@ -1,3 +1,5 @@ +#:import ServerAddr electrum.interface.ServerAddr + Popup: id: nd title: _('Server') @@ -23,7 +25,7 @@ Popup: height: '36dp' size_hint_x: 3 size_hint_y: None - text: app.network.get_parameters().host + text: app.network.get_parameters().server.host Label: height: '36dp' size_hint_x: 1 @@ -36,7 +38,7 @@ Popup: height: '36dp' size_hint_x: 3 size_hint_y: None - text: app.network.get_parameters().port + text: str(app.network.get_parameters().server.port) Widget Button: id: chooser @@ -56,7 +58,5 @@ Popup: height: '48dp' text: _('OK') on_release: - net_params = app.network.get_parameters() - net_params = net_params._replace(host=str(root.ids.host.text), port=str(root.ids.port.text)) - app.network.run_from_another_thread(app.network.set_parameters(net_params)) + app.maybe_switch_to_server(host=str(root.ids.host.text), port=str(root.ids.port.text)) nd.dismiss() diff --git a/electrum/gui/qt/main_window.py b/electrum/gui/qt/main_window.py index 062f153bd3..2c65c895f5 100644 --- a/electrum/gui/qt/main_window.py +++ b/electrum/gui/qt/main_window.py @@ -738,7 +738,7 @@ def add_toggle_action(view_menu, tab): def donate_to_server(self): d = self.network.get_donation_address() if d: - host = self.network.get_parameters().host + host = self.network.get_parameters().server.host self.pay_to_URI('bitcoin:%s?message=donation for %s'%(d, host)) else: self.show_error(_('No donation address for this server')) diff --git a/electrum/gui/qt/network_dialog.py b/electrum/gui/qt/network_dialog.py index e8cf3d6e3d..b34f80e98e 100644 --- a/electrum/gui/qt/network_dialog.py +++ b/electrum/gui/qt/network_dialog.py @@ -364,11 +364,11 @@ def enable_set_server(self): def update(self): net_params = self.network.get_parameters() - host, port, protocol = net_params.host, net_params.port, net_params.protocol + server = net_params.server proxy_config, auto_connect = net_params.proxy, net_params.auto_connect if not self.server_host.hasFocus() and not self.server_port.hasFocus(): - self.server_host.setText(host) - self.server_port.setText(str(port)) + self.server_host.setText(server.host) + self.server_port.setText(str(server.port)) self.autoconnect_cb.setChecked(auto_connect) interface = self.network.interface @@ -448,8 +448,13 @@ def accept(self): def set_server(self): net_params = self.network.get_parameters() - net_params = net_params._replace(host=str(self.server_host.text()), - port=str(self.server_port.text()), + try: + server = ServerAddr(host=str(self.server_host.text()), + port=str(self.server_port.text()), + protocol=net_params.server.protocol) + except Exception: + return + net_params = net_params._replace(server=server, auto_connect=self.autoconnect_cb.isChecked()) self.network.run_from_another_thread(self.network.set_parameters(net_params)) diff --git a/electrum/gui/text.py b/electrum/gui/text.py index 71c6a49dd6..8758b2c11c 100644 --- a/electrum/gui/text.py +++ b/electrum/gui/text.py @@ -409,7 +409,7 @@ def network_dialog(self): if not self.network: return net_params = self.network.get_parameters() - host, port, protocol = net_params.host, net_params.port, net_params.protocol + server_addr = net_params.server proxy_config, auto_connect = net_params.proxy, net_params.auto_connect srv = 'auto-connect' if auto_connect else str(self.network.default_server) out = self.run_dialog('Network', [ @@ -426,12 +426,11 @@ def network_dialog(self): except Exception: self.show_message("Error:" + server_str + "\nIn doubt, type \"auto-connect\"") return False - host = server_addr.host - port = str(server_addr.port) - protocol = server_addr.protocol if out.get('server') or out.get('proxy'): proxy = electrum.network.deserialize_proxy(out.get('proxy')) if out.get('proxy') else proxy_config - net_params = NetworkParameters(host, port, protocol, proxy, auto_connect) + net_params = NetworkParameters(server=server_addr, + proxy=proxy, + auto_connect=auto_connect) self.network.run_from_another_thread(self.network.set_parameters(net_params)) def settings_dialog(self): diff --git a/electrum/network.py b/electrum/network.py index e7cdb68502..409fd02489 100644 --- a/electrum/network.py +++ b/electrum/network.py @@ -144,9 +144,7 @@ def pick_random_server(hostmap=None, *, allowed_protocols: Iterable[str], class NetworkParameters(NamedTuple): - host: str - port: str - protocol: str + server: ServerAddr proxy: Optional[dict] auto_connect: bool oneserver: bool = False @@ -483,10 +481,7 @@ def notify(self, key): util.trigger_callback(key, self.get_status_value(key)) def get_parameters(self) -> NetworkParameters: - server = self.default_server - return NetworkParameters(host=server.host, - port=str(server.port), - protocol=server.protocol, + return NetworkParameters(server=self.default_server, proxy=self.proxy, auto_connect=self.auto_connect, oneserver=self.oneserver) @@ -585,13 +580,12 @@ def _set_proxy(self, proxy: Optional[dict]): util.trigger_callback('proxy_set', self.proxy) @log_exceptions - async def set_parameters(self, net_params: NetworkParameters): # TODO + async def set_parameters(self, net_params: NetworkParameters): proxy = net_params.proxy proxy_str = serialize_proxy(proxy) - host, port, protocol = net_params.host, net_params.port, net_params.protocol + server = net_params.server # sanitize parameters try: - server = ServerAddr(host, port, protocol=protocol) if proxy: proxy_modes.index(proxy['mode']) + 1 int(proxy['port']) @@ -1112,10 +1106,7 @@ async def follow_chain_given_id(self, chain_id: str) -> None: chosen_iface = random.choice(interfaces_on_selected_chain) # type: Interface # switch to server (and save to config) net_params = self.get_parameters() - server = chosen_iface.server - net_params = net_params._replace(host=server.host, - port=str(server.port), - protocol=server.protocol) + net_params = net_params._replace(server=chosen_iface.server) await self.set_parameters(net_params) async def follow_chain_given_server(self, server: ServerAddr) -> None: @@ -1126,9 +1117,7 @@ async def follow_chain_given_server(self, server: ServerAddr) -> None: self._set_preferred_chain(iface.blockchain) # switch to server (and save to config) net_params = self.get_parameters() - net_params = net_params._replace(host=server.host, - port=str(server.port), - protocol=server.protocol) + net_params = net_params._replace(server=server) await self.set_parameters(net_params) def get_local_height(self): From 9e57ae630ba96e6d2c40288d2633aeca8d20764d Mon Sep 17 00:00:00 2001 From: SomberNight Date: Thu, 16 Apr 2020 21:12:23 +0200 Subject: [PATCH 090/117] network/gui: unify host/port input fields to single server str This allows optionally specifying the protocol for the main server. fixes #6095 fixes #5278 --- electrum/gui/kivy/main_window.py | 6 +-- electrum/gui/kivy/uix/ui_screens/server.kv | 23 ++-------- electrum/gui/qt/network_dialog.py | 51 +++++----------------- electrum/interface.py | 24 +++++++++- electrum/network.py | 6 +-- 5 files changed, 42 insertions(+), 68 deletions(-) diff --git a/electrum/gui/kivy/main_window.py b/electrum/gui/kivy/main_window.py index 7cf50e46aa..63ea82eeea 100644 --- a/electrum/gui/kivy/main_window.py +++ b/electrum/gui/kivy/main_window.py @@ -145,13 +145,11 @@ def cb2(host): servers = self.network.get_servers() ChoiceDialog(_('Choose a server'), sorted(servers), popup.ids.host.text, cb2).open() - def maybe_switch_to_server(self, *, host: str, port: str): + def maybe_switch_to_server(self, server_str: str): from electrum.interface import ServerAddr net_params = self.network.get_parameters() try: - server = ServerAddr(host=host, - port=port, - protocol=net_params.server.protocol) + server = ServerAddr.from_str_with_inference(server_str) except Exception as e: self.show_error(_("Invalid server details: {}").format(repr(e))) return diff --git a/electrum/gui/kivy/uix/ui_screens/server.kv b/electrum/gui/kivy/uix/ui_screens/server.kv index aee4395355..d67be71aa0 100644 --- a/electrum/gui/kivy/uix/ui_screens/server.kv +++ b/electrum/gui/kivy/uix/ui_screens/server.kv @@ -1,5 +1,3 @@ -#:import ServerAddr electrum.interface.ServerAddr - Popup: id: nd title: _('Server') @@ -18,27 +16,14 @@ Popup: height: '36dp' size_hint_x: 1 size_hint_y: None - text: _('Host') + ':' + text: _('Server') + ':' TextInput: - id: host + id: server_str multiline: False height: '36dp' size_hint_x: 3 size_hint_y: None - text: app.network.get_parameters().server.host - Label: - height: '36dp' - size_hint_x: 1 - size_hint_y: None - text: _('Port') + ':' - TextInput: - id: port - multiline: False - input_type: 'number' - height: '36dp' - size_hint_x: 3 - size_hint_y: None - text: str(app.network.get_parameters().server.port) + text: app.network.get_parameters().server.net_addr_str() Widget Button: id: chooser @@ -58,5 +43,5 @@ Popup: height: '48dp' text: _('OK') on_release: - app.maybe_switch_to_server(host=str(root.ids.host.text), port=str(root.ids.port.text)) + app.maybe_switch_to_server(str(root.ids.server_str.text)) nd.dismiss() diff --git a/electrum/gui/qt/network_dialog.py b/electrum/gui/qt/network_dialog.py index b34f80e98e..bbede847cb 100644 --- a/electrum/gui/qt/network_dialog.py +++ b/electrum/gui/qt/network_dialog.py @@ -36,8 +36,8 @@ from electrum.i18n import _ from electrum import constants, blockchain, util -from electrum.interface import ServerAddr -from electrum.network import Network, PREFERRED_NETWORK_PROTOCOL +from electrum.interface import ServerAddr, PREFERRED_NETWORK_PROTOCOL +from electrum.network import Network from electrum.logging import get_logger from .util import (Buttons, CloseButton, HelpButton, read_QIcon, char_width_in_lineedit, @@ -171,8 +171,7 @@ def create_menu(self, position): menu.exec_(self.viewport().mapToGlobal(position)) def set_server(self, server: ServerAddr): - self.parent.server_host.setText(server.host) - self.parent.server_port.setText(str(server.port)) + self.parent.server_e.setText(server.net_addr_str()) self.parent.set_server() def keyPressEvent(self, event): @@ -230,15 +229,12 @@ def __init__(self, network: Network, config, wizard=False): grid = QGridLayout(server_tab) grid.setSpacing(8) - self.server_host = QLineEdit() - self.server_host.setFixedWidth(fixed_width_hostname) - self.server_port = QLineEdit() - self.server_port.setFixedWidth(fixed_width_port) + self.server_e = QLineEdit() + self.server_e.setFixedWidth(fixed_width_hostname + fixed_width_port) self.autoconnect_cb = QCheckBox(_('Select server automatically')) self.autoconnect_cb.setEnabled(self.config.is_modifiable('auto_connect')) - self.server_host.editingFinished.connect(self.set_server) - self.server_port.editingFinished.connect(self.set_server) + self.server_e.editingFinished.connect(self.set_server) self.autoconnect_cb.clicked.connect(self.set_server) self.autoconnect_cb.clicked.connect(self.update) @@ -250,8 +246,7 @@ def __init__(self, network: Network, config, wizard=False): grid.addWidget(HelpButton(msg), 0, 4) grid.addWidget(QLabel(_('Server') + ':'), 1, 0) - grid.addWidget(self.server_host, 1, 1, 1, 2) - grid.addWidget(self.server_port, 1, 3) + grid.addWidget(self.server_e, 1, 1, 1, 3) label = _('Server peers') if network.is_connected() else _('Default Servers') grid.addWidget(QLabel(label), 2, 0, 1, 5) @@ -355,20 +350,18 @@ def check_disable_proxy(self, b): def enable_set_server(self): if self.config.is_modifiable('server'): enabled = not self.autoconnect_cb.isChecked() - self.server_host.setEnabled(enabled) - self.server_port.setEnabled(enabled) + self.server_e.setEnabled(enabled) self.servers_list.setEnabled(enabled) else: - for w in [self.autoconnect_cb, self.server_host, self.server_port, self.servers_list]: + for w in [self.autoconnect_cb, self.server_e, self.servers_list]: w.setEnabled(False) def update(self): net_params = self.network.get_parameters() server = net_params.server proxy_config, auto_connect = net_params.proxy, net_params.auto_connect - if not self.server_host.hasFocus() and not self.server_port.hasFocus(): - self.server_host.setText(server.host) - self.server_port.setText(str(server.port)) + if not self.server_e.hasFocus(): + self.server_e.setText(server.net_addr_str()) self.autoconnect_cb.setChecked(auto_connect) interface = self.network.interface @@ -425,33 +418,13 @@ def follow_server(self, server: ServerAddr): self.network.run_from_another_thread(self.network.follow_chain_given_server(server)) self.update() - def change_server(self, host, protocol): - pp = self.servers.get(host, constants.net.DEFAULT_PORTS) - if protocol and protocol not in protocol_letters: - protocol = None - if protocol: - port = pp.get(protocol) - if port is None: - protocol = None - if not protocol: - if 's' in pp.keys(): - protocol = 's' - port = pp.get(protocol) - else: - protocol = list(pp.keys())[0] - port = pp.get(protocol) - self.server_host.setText(host) - self.server_port.setText(port) - def accept(self): pass def set_server(self): net_params = self.network.get_parameters() try: - server = ServerAddr(host=str(self.server_host.text()), - port=str(self.server_port.text()), - protocol=net_params.server.protocol) + server = ServerAddr.from_str_with_inference(str(self.server_e.text())) except Exception: return net_params = net_params._replace(server=server, diff --git a/electrum/interface.py b/electrum/interface.py index f17a340179..5600524e5a 100644 --- a/electrum/interface.py +++ b/electrum/interface.py @@ -65,6 +65,10 @@ MAX_INCOMING_MSG_SIZE = 1_000_000 # in bytes +_KNOWN_NETWORK_PROTOCOLS = {'t', 's'} +PREFERRED_NETWORK_PROTOCOL = 's' +assert PREFERRED_NETWORK_PROTOCOL in _KNOWN_NETWORK_PROTOCOLS + class NetworkTimeout: # seconds @@ -212,7 +216,7 @@ def __init__(self, host: str, port: Union[int, str], *, protocol: str = None): net_addr = NetAddress(host, port) # this validates host and port except Exception as e: raise ValueError(f"cannot construct ServerAddr: invalid host or port (host={host}, port={port})") from e - if protocol not in ('s', 't'): + if protocol not in _KNOWN_NETWORK_PROTOCOLS: raise ValueError(f"invalid network protocol: {protocol}") self.host = str(net_addr.host) # canonical form (if e.g. IPv6 address) self.port = int(net_addr.port) @@ -225,6 +229,24 @@ def from_str(cls, s: str) -> 'ServerAddr': host, port, protocol = str(s).rsplit(':', 2) return ServerAddr(host=host, port=port, protocol=protocol) + @classmethod + def from_str_with_inference(cls, s: str) -> Optional['ServerAddr']: + """Construct ServerAddr from str, guessing missing details. + Ongoing compatibility not guaranteed. + """ + if not s: + return None + items = str(s).rsplit(':', 2) + if len(items) < 2: + return None # although maybe we could guess the port too? + host = items[0] + port = items[1] + if len(items) >= 3: + protocol = items[2] + else: + protocol = PREFERRED_NETWORK_PROTOCOL + return ServerAddr(host=host, port=port, protocol=protocol) + def __str__(self): return '{}:{}'.format(self.net_addr_str(), self.protocol) diff --git a/electrum/network.py b/electrum/network.py index 409fd02489..d8995d7bb5 100644 --- a/electrum/network.py +++ b/electrum/network.py @@ -53,7 +53,7 @@ from . import dns_hacks from .transaction import Transaction from .blockchain import Blockchain, HEADER_SIZE -from .interface import (Interface, +from .interface import (Interface, PREFERRED_NETWORK_PROTOCOL, RequestTimedOut, NetworkTimeout, BUCKET_NAME_OF_ONION_SERVERS, NetworkException, RequestCorrupted, ServerAddr) from .version import PROTOCOL_VERSION @@ -75,10 +75,6 @@ NUM_STICKY_SERVERS = 4 NUM_RECENT_SERVERS = 20 -_KNOWN_NETWORK_PROTOCOLS = {'t', 's'} -PREFERRED_NETWORK_PROTOCOL = 's' -assert PREFERRED_NETWORK_PROTOCOL in _KNOWN_NETWORK_PROTOCOLS - def parse_servers(result: Sequence[Tuple[str, str, List[str]]]) -> Dict[str, dict]: """ parse servers list into dict format""" From cd199390e2d6376cdf9458c4a14906c564beea1c Mon Sep 17 00:00:00 2001 From: Luke Childs Date: Fri, 17 Apr 2020 02:39:05 +0700 Subject: [PATCH 091/117] Use non-standard localhost port for server-string fallback (#6087) * Use non-standard localhost port for server-string fallback Co-authored-by: Luke Childs --- electrum/network.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/electrum/network.py b/electrum/network.py index d8995d7bb5..b86a1747a7 100644 --- a/electrum/network.py +++ b/electrum/network.py @@ -285,8 +285,8 @@ def __init__(self, config: SimpleConfig, *, daemon: 'Daemon' = None): try: self.default_server = ServerAddr.from_str(self.default_server) except: - self.logger.warning('failed to parse server-string; falling back to localhost.') - self.default_server = ServerAddr.from_str("localhost:50002:s") + self.logger.warning('failed to parse server-string; falling back to localhost:1:s.') + self.default_server = ServerAddr.from_str("localhost:1:s") else: self.default_server = pick_random_server(allowed_protocols=self._allowed_protocols) assert isinstance(self.default_server, ServerAddr), f"invalid type for default_server: {self.default_server!r}" From 98d2ab5bd6f47961316c607f2a7ccd01fe38c472 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Fri, 17 Apr 2020 19:25:18 +0200 Subject: [PATCH 092/117] hww: fix HardwareClientBase not having reference to plugin it was incorrectly documented that it did (previously only for some plugins) --- electrum/plugins/bitbox02/bitbox02.py | 6 +++--- electrum/plugins/coldcard/coldcard.py | 5 +++-- electrum/plugins/digitalbitbox/digitalbitbox.py | 2 +- electrum/plugins/hw_wallet/plugin.py | 4 +++- electrum/plugins/keepkey/clientbase.py | 1 + electrum/plugins/ledger/ledger.py | 6 ++++-- electrum/plugins/safe_t/clientbase.py | 1 + electrum/plugins/trezor/clientbase.py | 2 +- 8 files changed, 17 insertions(+), 10 deletions(-) diff --git a/electrum/plugins/bitbox02/bitbox02.py b/electrum/plugins/bitbox02/bitbox02.py index 3b28923277..15a0f64dc7 100644 --- a/electrum/plugins/bitbox02/bitbox02.py +++ b/electrum/plugins/bitbox02/bitbox02.py @@ -44,7 +44,8 @@ class BitBox02Client(HardwareClientBase): # handler is a BitBox02_Handler, importing it would lead to a circular dependency - def __init__(self, handler: Any, device: Device, config: SimpleConfig): + def __init__(self, handler: Any, device: Device, config: SimpleConfig, *, plugin: HW_PluginBase): + HardwareClientBase.__init__(self, plugin=plugin) self.bitbox02_device = None self.handler = handler self.device_descriptor = device @@ -556,12 +557,11 @@ def get_library_version(self): else: raise ImportError() - # handler is a BitBox02_Handler def create_client(self, device: Device, handler: Any) -> BitBox02Client: if not handler: self.handler = handler - return BitBox02Client(handler, device, self.config) + return BitBox02Client(handler, device, self.config, plugin=self) def setup_device( self, device_info: DeviceInfo, wizard: BaseWizard, purpose: int diff --git a/electrum/plugins/coldcard/coldcard.py b/electrum/plugins/coldcard/coldcard.py index b8cae820c3..fcfa2d1925 100644 --- a/electrum/plugins/coldcard/coldcard.py +++ b/electrum/plugins/coldcard/coldcard.py @@ -60,7 +60,8 @@ def mitm_verify(self, sig, expect_xpub): class CKCCClient(HardwareClientBase): - def __init__(self, plugin, handler, dev_path, is_simulator=False): + def __init__(self, plugin, handler, dev_path, *, is_simulator=False): + HardwareClientBase.__init__(self, plugin=plugin) self.device = plugin.device self.handler = handler @@ -515,7 +516,7 @@ def create_client(self, device, handler): # the 'path' is unabiguous, so we'll use that. try: rv = CKCCClient(self, handler, device.path, - is_simulator=(device.product_key[1] == CKCC_SIMULATED_PID)) + is_simulator=(device.product_key[1] == CKCC_SIMULATED_PID)) return rv except: self.logger.info('late failure connecting to device?') diff --git a/electrum/plugins/digitalbitbox/digitalbitbox.py b/electrum/plugins/digitalbitbox/digitalbitbox.py index 15b91de7ce..add12ff415 100644 --- a/electrum/plugins/digitalbitbox/digitalbitbox.py +++ b/electrum/plugins/digitalbitbox/digitalbitbox.py @@ -66,7 +66,7 @@ def derive_keys(x): class DigitalBitbox_Client(HardwareClientBase): def __init__(self, plugin, hidDevice): - self.plugin = plugin + HardwareClientBase.__init__(self, plugin=plugin) self.dbb_hid = hidDevice self.opened = True self.password = None diff --git a/electrum/plugins/hw_wallet/plugin.py b/electrum/plugins/hw_wallet/plugin.py index c7c0c1eefc..e9b6a78396 100644 --- a/electrum/plugins/hw_wallet/plugin.py +++ b/electrum/plugins/hw_wallet/plugin.py @@ -191,9 +191,11 @@ def create_handler(self, window) -> 'HardwareHandlerBase': class HardwareClientBase: - plugin: 'HW_PluginBase' handler = None # type: Optional['HardwareHandlerBase'] + def __init__(self, *, plugin: 'HW_PluginBase'): + self.plugin = plugin + def is_pairable(self) -> bool: raise NotImplementedError() diff --git a/electrum/plugins/keepkey/clientbase.py b/electrum/plugins/keepkey/clientbase.py index 92cae82078..3c106edbe8 100644 --- a/electrum/plugins/keepkey/clientbase.py +++ b/electrum/plugins/keepkey/clientbase.py @@ -103,6 +103,7 @@ class KeepKeyClientBase(HardwareClientBase, GuiMixin, Logger): def __init__(self, handler, plugin, proto): assert hasattr(self, 'tx_api') # ProtocolMixin already constructed? + HardwareClientBase.__init__(self, plugin=plugin) self.proto = proto self.device = plugin.device self.handler = handler diff --git a/electrum/plugins/ledger/ledger.py b/electrum/plugins/ledger/ledger.py index 08cea77a38..1770166473 100644 --- a/electrum/plugins/ledger/ledger.py +++ b/electrum/plugins/ledger/ledger.py @@ -62,7 +62,9 @@ def catch_exception(self, *args, **kwargs): class Ledger_Client(HardwareClientBase): - def __init__(self, hidDevice, *, product_key: Tuple[int, int]): + def __init__(self, hidDevice, *, product_key: Tuple[int, int], + plugin: HW_PluginBase): + HardwareClientBase.__init__(self, plugin=plugin) self.dongleObject = btchip(hidDevice) self.preflightDone = False self._product_key = product_key @@ -602,7 +604,7 @@ def create_client(self, device, handler): client = self.get_btchip_device(device) if client is not None: - client = Ledger_Client(client, product_key=device.product_key) + client = Ledger_Client(client, product_key=device.product_key, plugin=self) return client def setup_device(self, device_info, wizard, purpose): diff --git a/electrum/plugins/safe_t/clientbase.py b/electrum/plugins/safe_t/clientbase.py index 18bbdbab4b..9b15037f31 100644 --- a/electrum/plugins/safe_t/clientbase.py +++ b/electrum/plugins/safe_t/clientbase.py @@ -105,6 +105,7 @@ class SafeTClientBase(HardwareClientBase, GuiMixin, Logger): def __init__(self, handler, plugin, proto): assert hasattr(self, 'tx_api') # ProtocolMixin already constructed? + HardwareClientBase.__init__(self, plugin=plugin) self.proto = proto self.device = plugin.device self.handler = handler diff --git a/electrum/plugins/trezor/clientbase.py b/electrum/plugins/trezor/clientbase.py index 829a245bca..3d83f57360 100644 --- a/electrum/plugins/trezor/clientbase.py +++ b/electrum/plugins/trezor/clientbase.py @@ -40,10 +40,10 @@ class TrezorClientBase(HardwareClientBase, Logger): def __init__(self, transport, handler, plugin): + HardwareClientBase.__init__(self, plugin=plugin) if plugin.is_outdated_fw_ignored(): TrezorClient.is_outdated = lambda *args, **kwargs: False self.client = TrezorClient(transport, ui=self) - self.plugin = plugin self.device = plugin.device self.handler = handler Logger.__init__(self) From 2cfa3bd6c82b5f2df0332307240fd202523cb522 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Fri, 17 Apr 2020 19:05:56 +0200 Subject: [PATCH 093/117] hww hidapi usage: try to mitigate some thread-safety issues related: #6097 --- electrum/plugin.py | 32 ++++++++++++++++++- electrum/plugins/bitbox02/bitbox02.py | 19 ++++++----- electrum/plugins/coldcard/coldcard.py | 9 +++--- .../plugins/digitalbitbox/digitalbitbox.py | 14 ++++---- electrum/plugins/hw_wallet/plugin.py | 3 ++ electrum/plugins/ledger/ledger.py | 14 ++++---- 6 files changed, 66 insertions(+), 25 deletions(-) diff --git a/electrum/plugin.py b/electrum/plugin.py index d86e767069..b643a763ef 100644 --- a/electrum/plugin.py +++ b/electrum/plugin.py @@ -30,6 +30,8 @@ import sys from typing import (NamedTuple, Any, Union, TYPE_CHECKING, Optional, Tuple, Dict, Iterable, List, Sequence) +import concurrent +from concurrent import futures from .i18n import _ from .util import (profiler, DaemonThread, UserCancelled, ThreadJob, UserFacingException) @@ -321,6 +323,20 @@ class HardwarePluginToScan(NamedTuple): PLACEHOLDER_HW_CLIENT_LABELS = {None, "", " "} +# hidapi is not thread-safe +# see https://github.com/signal11/hidapi/issues/205#issuecomment-527654560 +# https://github.com/libusb/hidapi/issues/45 +# https://github.com/signal11/hidapi/issues/45#issuecomment-4434598 +# https://github.com/signal11/hidapi/pull/414#issuecomment-445164238 +# It is not entirely clear to me, exactly what is safe and what isn't, when +# using multiple threads... +# For now, we use a dedicated thread to enumerate devices (_hid_executor), +# and we synchronize all device opens/closes/enumeration (_hid_lock). +# FIXME there are still probably threading issues with how we use hidapi... +_hid_executor = None # type: Optional[concurrent.futures.Executor] +_hid_lock = threading.Lock() + + class DeviceMgr(ThreadJob): '''Manages hardware clients. A client communicates over a hardware channel with the device. @@ -367,9 +383,15 @@ def __init__(self, config: SimpleConfig): # locks: if you need to take multiple ones, acquire them in the order they are defined here! self._scan_lock = threading.RLock() self.lock = threading.RLock() + self.hid_lock = _hid_lock self.config = config + global _hid_executor + if _hid_executor is None: + _hid_executor = concurrent.futures.ThreadPoolExecutor(max_workers=1, + thread_name_prefix='hid_enumerate_thread') + def with_scan_lock(func): def func_wrapper(self: 'DeviceMgr', *args, **kwargs): with self._scan_lock: @@ -636,7 +658,15 @@ def _scan_devices_with_hid(self) -> List['Device']: except ImportError: return [] - hid_list = hid.enumerate(0, 0) + def hid_enumerate(): + with self.hid_lock: + return hid.enumerate(0, 0) + + hid_list_fut = _hid_executor.submit(hid_enumerate) + try: + hid_list = hid_list_fut.result() + except (concurrent.futures.CancelledError, concurrent.futures.TimeoutError) as e: + return [] devices = [] for d in hid_list: diff --git a/electrum/plugins/bitbox02/bitbox02.py b/electrum/plugins/bitbox02/bitbox02.py index 15a0f64dc7..87089e4dbd 100644 --- a/electrum/plugins/bitbox02/bitbox02.py +++ b/electrum/plugins/bitbox02/bitbox02.py @@ -46,7 +46,7 @@ class BitBox02Client(HardwareClientBase): # handler is a BitBox02_Handler, importing it would lead to a circular dependency def __init__(self, handler: Any, device: Device, config: SimpleConfig, *, plugin: HW_PluginBase): HardwareClientBase.__init__(self, plugin=plugin) - self.bitbox02_device = None + self.bitbox02_device = None # type: Optional[bitbox02.BitBox02] self.handler = handler self.device_descriptor = device self.config = config @@ -73,10 +73,11 @@ def is_initialized(self) -> bool: return True def close(self): - try: - self.bitbox02_device.close() - except: - pass + with self.device_manager().hid_lock: + try: + self.bitbox02_device.close() + except: + pass def has_usable_connection_with_device(self) -> bool: if self.bitbox_hid_info is None: @@ -91,7 +92,8 @@ def pairing_step(code: str, device_response: Callable[[], bool]) -> bool: res = device_response() except: # Close the hid device on exception - hid_device.close() + with self.device_manager().hid_lock: + hid_device.close() raise finally: self.handler.finished() @@ -155,8 +157,9 @@ def set_app_static_privkey(self, privkey: bytes) -> None: return set_noise_privkey(privkey) if self.bitbox02_device is None: - hid_device = hid.device() - hid_device.open_path(self.bitbox_hid_info["path"]) + with self.device_manager().hid_lock: + hid_device = hid.device() + hid_device.open_path(self.bitbox_hid_info["path"]) self.bitbox02_device = bitbox02.BitBox02( transport=u2fhid.U2FHid(hid_device), diff --git a/electrum/plugins/coldcard/coldcard.py b/electrum/plugins/coldcard/coldcard.py index fcfa2d1925..f0936f08d7 100644 --- a/electrum/plugins/coldcard/coldcard.py +++ b/electrum/plugins/coldcard/coldcard.py @@ -72,9 +72,9 @@ def __init__(self, plugin, handler, dev_path, *, is_simulator=False): self.dev = ElectrumColdcardDevice(dev_path, encrypt=True) else: # open the real HID device - import hid - hd = hid.device(path=dev_path) - hd.open_path(dev_path) + with self.device_manager().hid_lock: + hd = hid.device(path=dev_path) + hd.open_path(dev_path) self.dev = ElectrumColdcardDevice(dev=hd, encrypt=True) @@ -127,7 +127,8 @@ def timeout(self, cutoff): def close(self): # close the HID device (so can be reused) - self.dev.close() + with self.device_manager().hid_lock: + self.dev.close() self.dev = None def is_initialized(self): diff --git a/electrum/plugins/digitalbitbox/digitalbitbox.py b/electrum/plugins/digitalbitbox/digitalbitbox.py index add12ff415..e35cdc9827 100644 --- a/electrum/plugins/digitalbitbox/digitalbitbox.py +++ b/electrum/plugins/digitalbitbox/digitalbitbox.py @@ -77,10 +77,11 @@ def __init__(self, plugin, hidDevice): def close(self): if self.opened: - try: - self.dbb_hid.close() - except: - pass + with self.device_manager().hid_lock: + try: + self.dbb_hid.close() + except: + pass self.opened = False @@ -681,8 +682,9 @@ def __init__(self, parent, config, name): def get_dbb_device(self, device): - dev = hid.device() - dev.open_path(device.path) + with self.device_manager().hid_lock: + dev = hid.device() + dev.open_path(device.path) return dev diff --git a/electrum/plugins/hw_wallet/plugin.py b/electrum/plugins/hw_wallet/plugin.py index e9b6a78396..512f0ca6ed 100644 --- a/electrum/plugins/hw_wallet/plugin.py +++ b/electrum/plugins/hw_wallet/plugin.py @@ -196,6 +196,9 @@ class HardwareClientBase: def __init__(self, *, plugin: 'HW_PluginBase'): self.plugin = plugin + def device_manager(self) -> 'DeviceMgr': + return self.plugin.device_manager() + def is_pairable(self) -> bool: raise NotImplementedError() diff --git a/electrum/plugins/ledger/ledger.py b/electrum/plugins/ledger/ledger.py index 1770166473..393a0c3697 100644 --- a/electrum/plugins/ledger/ledger.py +++ b/electrum/plugins/ledger/ledger.py @@ -74,7 +74,8 @@ def is_pairable(self): return True def close(self): - self.dongleObject.dongle.close() + with self.device_manager().hid_lock: + self.dongleObject.dongle.close() def timeout(self, cutoff): pass @@ -184,13 +185,13 @@ def perform_hw1_preflight(self): self.segwitSupported = self.nativeSegwitSupported or (firmwareInfo['specialVersion'] == 0x20 and versiontuple(firmware) >= versiontuple(SEGWIT_SUPPORT_SPECIAL)) if not checkFirmware(firmwareInfo): - self.dongleObject.dongle.close() + self.close() raise UserFacingException(MSG_NEEDS_FW_UPDATE_GENERIC) try: self.dongleObject.getOperationMode() except BTChipException as e: if (e.sw == 0x6985): - self.dongleObject.dongle.close() + self.close() self.handler.get_setup( ) # Acquire the new client on the next run else: @@ -593,9 +594,10 @@ def get_btchip_device(self, device): ledger = True else: return None # non-compatible interface of a Nano S or Blue - dev = hid.device() - dev.open_path(device.path) - dev.set_nonblocking(True) + with self.device_manager().hid_lock: + dev = hid.device() + dev.open_path(device.path) + dev.set_nonblocking(True) return HIDDongleHIDAPI(dev, ledger, BTCHIP_DEBUG) def create_client(self, device, handler): From b1d23896562d0311b3bbcb56588a67f99d76047c Mon Sep 17 00:00:00 2001 From: SomberNight Date: Fri, 17 Apr 2020 19:37:05 +0200 Subject: [PATCH 094/117] hww: stop keystore.thread when closing wallet previously left running? Qt on macOS was complaining: ``` QThread: Destroyed while thread is still running Abort trap: 6 ``` --- electrum/plugins/hw_wallet/plugin.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/electrum/plugins/hw_wallet/plugin.py b/electrum/plugins/hw_wallet/plugin.py index 512f0ca6ed..00fdcabadb 100644 --- a/electrum/plugins/hw_wallet/plugin.py +++ b/electrum/plugins/hw_wallet/plugin.py @@ -81,6 +81,8 @@ def close_wallet(self, wallet: 'Abstract_Wallet'): for keystore in wallet.get_keystores(): if isinstance(keystore, self.keystore_class): self.device_manager().unpair_xpub(keystore.xpub) + if keystore.thread: + keystore.thread.stop() def scan_and_create_client_for_device(self, *, device_id: str, wizard: 'BaseWizard') -> 'HardwareClientBase': devmgr = self.device_manager() From 8f4c384aad8531d1acecffde5583deefc251fc50 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Sat, 18 Apr 2020 05:48:11 +0200 Subject: [PATCH 095/117] qt crash reporter: html.escape traceback to avoid formatting issues fixes #6099 --- electrum/base_crash_reporter.py | 5 ++++- electrum/gui/qt/exception_window.py | 10 ++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/electrum/base_crash_reporter.py b/electrum/base_crash_reporter.py index 212a714115..4d7c268b13 100644 --- a/electrum/base_crash_reporter.py +++ b/electrum/base_crash_reporter.py @@ -121,9 +121,12 @@ def get_git_version(): ['git', 'describe', '--always', '--dirty'], cwd=dir) return str(version, "utf8").strip() + def _get_traceback_str(self) -> str: + return "".join(traceback.format_exception(*self.exc_args)) + def get_report_string(self): info = self.get_additional_info() - info["traceback"] = "".join(traceback.format_exception(*self.exc_args)) + info["traceback"] = self._get_traceback_str() return self.issue_template.format(**info) def get_user_description(self): diff --git a/electrum/gui/qt/exception_window.py b/electrum/gui/qt/exception_window.py index f33223999c..b19e6fb7df 100644 --- a/electrum/gui/qt/exception_window.py +++ b/electrum/gui/qt/exception_window.py @@ -22,6 +22,7 @@ # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. import sys +import html from PyQt5.QtCore import QObject import PyQt5.QtCore as QtCore @@ -58,8 +59,6 @@ def __init__(self, main_window, exctype, value, tb): main_box.addWidget(QLabel(BaseCrashReporter.REQUEST_HELP_MESSAGE)) collapse_info = QPushButton(_("Show report contents")) - # FIXME if traceback contains special HTML characters, e.g. '<' - # then formatting issues arise (due to rich_text=True) collapse_info.clicked.connect( lambda: self.msg_box(QMessageBox.NoIcon, self, _("Report contents"), self.get_report_string(), @@ -139,6 +138,13 @@ def get_user_description(self): def get_wallet_type(self): return self.main_window.wallet.wallet_type + def _get_traceback_str(self) -> str: + # The msg_box that shows the report uses rich_text=True, so + # if traceback contains special HTML characters, e.g. '<', + # they need to be escaped to avoid formatting issues. + traceback_str = super()._get_traceback_str() + return html.escape(traceback_str) + def _show_window(*args): if not Exception_Window._active_window: From 12d771737afe87f520ffefb7648fd3878076b923 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Sat, 18 Apr 2020 05:56:12 +0200 Subject: [PATCH 096/117] fix #6096: bugfix for creating zero amount LN invoice (also there was a unit-mismatch here...) --- electrum/lnworker.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/electrum/lnworker.py b/electrum/lnworker.py index 889acad5e0..7610e6fd44 100644 --- a/electrum/lnworker.py +++ b/electrum/lnworker.py @@ -1051,7 +1051,7 @@ def add_request(self, amount_sat, message, expiry): raise Exception(_("add invoice timed out")) @log_exceptions - async def _add_request_coro(self, amount_sat, message, expiry: int): + async def _add_request_coro(self, amount_sat: Optional[int], message, expiry: int): timestamp = int(time.time()) routing_hints = await self._calc_routing_hints_for_invoice(amount_sat) if not routing_hints: @@ -1190,16 +1190,20 @@ def payment_received(self, chan, payment_hash: bytes): util.trigger_callback('request_status', payment_hash.hex(), PR_PAID) util.trigger_callback('ln_payment_completed', payment_hash, chan.channel_id) - async def _calc_routing_hints_for_invoice(self, amount_sat): + async def _calc_routing_hints_for_invoice(self, amount_sat: Optional[int]): """calculate routing hints (BOLT-11 'r' field)""" routing_hints = [] with self.lock: channels = list(self.channels.values()) scid_to_my_channels = {chan.short_channel_id: chan for chan in channels if chan.short_channel_id is not None} + if amount_sat: + amount_msat = 1000 * amount_sat + else: # for no amt invoices, check if channel can receive at least 1 sat: + amount_msat = 1 # note: currently we add *all* our channels; but this might be a privacy leak? for chan in channels: - if not chan.can_receive(amount_sat, check_frozen=True): + if not chan.can_receive(amount_msat=amount_msat, check_frozen=True): continue chan_id = chan.short_channel_id assert isinstance(chan_id, bytes), chan_id From f52072e1693107b07e29e4d3a68ee944f1020381 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Sat, 18 Apr 2020 18:51:20 +0200 Subject: [PATCH 097/117] follow-up prev we can't just test with a 1 msat htlc as that might be below htlc_minimum_msat --- electrum/lnchannel.py | 19 ++++++++++++------- electrum/lnworker.py | 8 ++++++-- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/electrum/lnchannel.py b/electrum/lnchannel.py index 0f7405e4a4..fd34112f02 100644 --- a/electrum/lnchannel.py +++ b/electrum/lnchannel.py @@ -688,7 +688,8 @@ def set_frozen_for_receiving(self, b: bool) -> None: self.storage['frozen_for_receiving'] = bool(b) util.trigger_callback('channel', self) - def _assert_can_add_htlc(self, *, htlc_proposer: HTLCOwner, amount_msat: int) -> None: + def _assert_can_add_htlc(self, *, htlc_proposer: HTLCOwner, amount_msat: int, + ignore_min_htlc_value: bool = False) -> None: """Raises PaymentFailure if the htlc_proposer cannot add this new HTLC. (this is relevant both for forwarding and endpoint) """ @@ -712,10 +713,11 @@ def _assert_can_add_htlc(self, *, htlc_proposer: HTLCOwner, amount_msat: int) -> strict = (htlc_proposer == LOCAL) # check htlc raw value - if amount_msat <= 0: - raise PaymentFailure("HTLC value must be positive") - if amount_msat < chan_config.htlc_minimum_msat: - raise PaymentFailure(f'HTLC value too small: {amount_msat} msat') + if not ignore_min_htlc_value: + if amount_msat <= 0: + raise PaymentFailure("HTLC value must be positive") + if amount_msat < chan_config.htlc_minimum_msat: + raise PaymentFailure(f'HTLC value too small: {amount_msat} msat') if amount_msat > LN_MAX_HTLC_VALUE_MSAT and not self._ignore_max_htlc_value: raise PaymentFailure(f"HTLC value over protocol maximum: {amount_msat} > {LN_MAX_HTLC_VALUE_MSAT} msat") @@ -752,12 +754,15 @@ def can_pay(self, amount_msat: int, *, check_frozen=False) -> bool: return False return True - def can_receive(self, amount_msat: int, *, check_frozen=False) -> bool: + def can_receive(self, amount_msat: int, *, check_frozen=False, + ignore_min_htlc_value: bool = False) -> bool: """Returns whether the remote can add an HTLC of given value.""" if check_frozen and self.is_frozen_for_receiving(): return False try: - self._assert_can_add_htlc(htlc_proposer=REMOTE, amount_msat=amount_msat) + self._assert_can_add_htlc(htlc_proposer=REMOTE, + amount_msat=amount_msat, + ignore_min_htlc_value=ignore_min_htlc_value) except PaymentFailure: return False return True diff --git a/electrum/lnworker.py b/electrum/lnworker.py index 7610e6fd44..59064389af 100644 --- a/electrum/lnworker.py +++ b/electrum/lnworker.py @@ -1197,13 +1197,17 @@ async def _calc_routing_hints_for_invoice(self, amount_sat: Optional[int]): channels = list(self.channels.values()) scid_to_my_channels = {chan.short_channel_id: chan for chan in channels if chan.short_channel_id is not None} + ignore_min_htlc_value = False if amount_sat: amount_msat = 1000 * amount_sat - else: # for no amt invoices, check if channel can receive at least 1 sat: + else: + # for no amt invoices, check if channel can receive at least 1 msat amount_msat = 1 + ignore_min_htlc_value = True # note: currently we add *all* our channels; but this might be a privacy leak? for chan in channels: - if not chan.can_receive(amount_msat=amount_msat, check_frozen=True): + if not chan.can_receive(amount_msat=amount_msat, check_frozen=True, + ignore_min_htlc_value=ignore_min_htlc_value): continue chan_id = chan.short_channel_id assert isinstance(chan_id, bytes), chan_id From ec5330fc21b3f8d05dfea12067c30f4a82ed75ae Mon Sep 17 00:00:00 2001 From: ThomasV Date: Fri, 17 Apr 2020 11:03:19 +0200 Subject: [PATCH 098/117] separate method that runs Dijkstra and return distances --- electrum/lnrouter.py | 46 +++++++++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/electrum/lnrouter.py b/electrum/lnrouter.py index bf266de212..8ca3b59bdc 100644 --- a/electrum/lnrouter.py +++ b/electrum/lnrouter.py @@ -184,26 +184,13 @@ def _edge_cost(self, short_channel_id: bytes, start_node: bytes, end_node: bytes overall_cost = base_cost + fee_msat + cltv_cost return overall_cost, fee_msat - @profiler - def find_path_for_payment(self, nodeA: bytes, nodeB: bytes, - invoice_amount_msat: int, *, - my_channels: Dict[ShortChannelID, 'Channel'] = None) \ - -> Optional[Sequence[Tuple[bytes, bytes]]]: - """Return a path from nodeA to nodeB. - - Returns a list of (node_id, short_channel_id) representing a path. - To get from node ret[n][0] to ret[n+1][0], use channel ret[n+1][1]; - i.e. an element reads as, "to get to node_id, travel through short_channel_id" - """ - assert type(nodeA) is bytes - assert type(nodeB) is bytes - assert type(invoice_amount_msat) is int - if my_channels is None: my_channels = {} + def get_distances(self, nodeA: bytes, nodeB: bytes, + invoice_amount_msat: int, *, + my_channels: Dict[ShortChannelID, 'Channel'] = None) \ + -> Optional[Sequence[Tuple[bytes, bytes]]]: # note: we don't lock self.channel_db, so while the path finding runs, # the underlying graph could potentially change... (not good but maybe ~OK?) - # FIXME paths cannot be longer than 20 edges (onion packet)... - # run Dijkstra # The search is run in the REVERSE direction, from nodeB to nodeA, # to properly calculate compound routing fees. @@ -255,10 +242,33 @@ def inspect_edge(): channel_info = self.channel_db.get_channel_info(edge_channel_id, my_channels=my_channels) edge_startnode = channel_info.node2_id if channel_info.node1_id == edge_endnode else channel_info.node1_id inspect_edge() - else: + + return prev_node + + @profiler + def find_path_for_payment(self, nodeA: bytes, nodeB: bytes, + invoice_amount_msat: int, *, + my_channels: Dict[ShortChannelID, 'Channel'] = None) \ + -> Optional[Sequence[Tuple[bytes, bytes]]]: + """Return a path from nodeA to nodeB. + + Returns a list of (node_id, short_channel_id) representing a path. + To get from node ret[n][0] to ret[n+1][0], use channel ret[n+1][1]; + i.e. an element reads as, "to get to node_id, travel through short_channel_id" + """ + assert type(nodeA) is bytes + assert type(nodeB) is bytes + assert type(invoice_amount_msat) is int + if my_channels is None: + my_channels = {} + + prev_node = self.get_distances(nodeA, nodeB, invoice_amount_msat, my_channels=my_channels) + + if nodeA not in prev_node: return None # no path found # backtrack from search_end (nodeA) to search_start (nodeB) + # FIXME paths cannot be longer than 20 edges (onion packet)... edge_startnode = nodeA path = [] while edge_startnode != nodeB: From 1a4d33086b65be364a034d71f449150b08a68521 Mon Sep 17 00:00:00 2001 From: ThomasV Date: Fri, 17 Apr 2020 11:53:51 +0200 Subject: [PATCH 099/117] refactoring: remove inspect_edge --- electrum/lnrouter.py | 48 +++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/electrum/lnrouter.py b/electrum/lnrouter.py index 8ca3b59bdc..9f77dd9e6e 100644 --- a/electrum/lnrouter.py +++ b/electrum/lnrouter.py @@ -200,30 +200,6 @@ def get_distances(self, nodeA: bytes, nodeB: bytes, nodes_to_explore = queue.PriorityQueue() nodes_to_explore.put((0, invoice_amount_msat, nodeB)) # order of fields (in tuple) matters! - def inspect_edge(): - is_mine = edge_channel_id in my_channels - if is_mine: - if edge_startnode == nodeA: # payment outgoing, on our channel - if not my_channels[edge_channel_id].can_pay(amount_msat, check_frozen=True): - return - else: # payment incoming, on our channel. (funny business, cycle weirdness) - assert edge_endnode == nodeA, (bh2u(edge_startnode), bh2u(edge_endnode)) - if not my_channels[edge_channel_id].can_receive(amount_msat, check_frozen=True): - return - edge_cost, fee_for_edge_msat = self._edge_cost( - edge_channel_id, - start_node=edge_startnode, - end_node=edge_endnode, - payment_amt_msat=amount_msat, - ignore_costs=(edge_startnode == nodeA), - is_mine=is_mine, - my_channels=my_channels) - alt_dist_to_neighbour = distance_from_start[edge_endnode] + edge_cost - if alt_dist_to_neighbour < distance_from_start[edge_startnode]: - distance_from_start[edge_startnode] = alt_dist_to_neighbour - prev_node[edge_startnode] = edge_endnode, edge_channel_id - amount_to_forward_msat = amount_msat + fee_for_edge_msat - nodes_to_explore.put((alt_dist_to_neighbour, amount_to_forward_msat, edge_startnode)) # main loop of search while nodes_to_explore.qsize() > 0: @@ -241,7 +217,29 @@ def inspect_edge(): continue channel_info = self.channel_db.get_channel_info(edge_channel_id, my_channels=my_channels) edge_startnode = channel_info.node2_id if channel_info.node1_id == edge_endnode else channel_info.node1_id - inspect_edge() + is_mine = edge_channel_id in my_channels + if is_mine: + if edge_startnode == nodeA: # payment outgoing, on our channel + if not my_channels[edge_channel_id].can_pay(amount_msat, check_frozen=True): + continue + else: # payment incoming, on our channel. (funny business, cycle weirdness) + assert edge_endnode == nodeA, (bh2u(edge_startnode), bh2u(edge_endnode)) + if not my_channels[edge_channel_id].can_receive(amount_msat, check_frozen=True): + continue + edge_cost, fee_for_edge_msat = self._edge_cost( + edge_channel_id, + start_node=edge_startnode, + end_node=edge_endnode, + payment_amt_msat=amount_msat, + ignore_costs=(edge_startnode == nodeA), + is_mine=is_mine, + my_channels=my_channels) + alt_dist_to_neighbour = distance_from_start[edge_endnode] + edge_cost + if alt_dist_to_neighbour < distance_from_start[edge_startnode]: + distance_from_start[edge_startnode] = alt_dist_to_neighbour + prev_node[edge_startnode] = edge_endnode, edge_channel_id + amount_to_forward_msat = amount_msat + fee_for_edge_msat + nodes_to_explore.put((alt_dist_to_neighbour, amount_to_forward_msat, edge_startnode)) return prev_node From 4d01a550c4802d68783adeff7e8df97f308f189c Mon Sep 17 00:00:00 2001 From: ThomasV Date: Mon, 20 Apr 2020 18:48:41 +0200 Subject: [PATCH 100/117] fix #6103: local config contains remote reserve --- electrum/commands.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/electrum/commands.py b/electrum/commands.py index 5dca403161..3eb0dedde9 100644 --- a/electrum/commands.py +++ b/electrum/commands.py @@ -1032,8 +1032,8 @@ async def list_channels(self, wallet: Abstract_Wallet = None): 'remote_pubkey': bh2u(chan.node_id), 'local_balance': chan.balance(LOCAL)//1000, 'remote_balance': chan.balance(REMOTE)//1000, - 'local_reserve': chan.config[LOCAL].reserve_sat, - 'remote_reserve': chan.config[REMOTE].reserve_sat, + 'local_reserve': chan.config[REMOTE].reserve_sat, # their config has our reserve + 'remote_reserve': chan.config[LOCAL].reserve_sat, 'local_unsettled_sent': chan.balance_tied_up_in_htlcs_by_direction(LOCAL, direction=SENT) // 1000, 'remote_unsettled_sent': chan.balance_tied_up_in_htlcs_by_direction(REMOTE, direction=SENT) // 1000, } for channel_id, chan in l From 0b6ae1dbff08c6c17b53ed854fb2f0c9af36c6ce Mon Sep 17 00:00:00 2001 From: ThomasV Date: Mon, 20 Apr 2020 18:54:43 +0200 Subject: [PATCH 101/117] fix #6101 --- electrum/commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/electrum/commands.py b/electrum/commands.py index 3eb0dedde9..3433dc6094 100644 --- a/electrum/commands.py +++ b/electrum/commands.py @@ -1077,7 +1077,7 @@ async def export_channel_backup(self, channel_point, wallet: Abstract_Wallet = N @command('w') async def import_channel_backup(self, encrypted, wallet: Abstract_Wallet = None): - return wallet.lnworker.import_channel_backup(encrypted) + return wallet.lnbackups.import_channel_backup(encrypted) @command('wn') async def get_channel_ctx(self, channel_point, iknowwhatimdoing=False, wallet: Abstract_Wallet = None): From bdb870af00a3a509f7dba7f3e14b8bb58a3634af Mon Sep 17 00:00:00 2001 From: ThomasV Date: Tue, 21 Apr 2020 15:31:13 +0200 Subject: [PATCH 102/117] follow-up c454564ed6e094aa36be9fbe7aa9e9113a28710c --- electrum/lnwatcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/electrum/lnwatcher.py b/electrum/lnwatcher.py index 4787253561..c338b4b5c5 100644 --- a/electrum/lnwatcher.py +++ b/electrum/lnwatcher.py @@ -53,7 +53,7 @@ class TxMinedDepth(IntEnum): class SweepStore(SqlDB): def __init__(self, path, network): - super().__init__(network, path) + super().__init__(network.asyncio_loop, path) def create_database(self): c = self.conn.cursor() From 1846154ca3cc8e8ff3e8fcd14b700f7722a781a0 Mon Sep 17 00:00:00 2001 From: Jin Eguchi Date: Wed, 22 Apr 2020 07:48:01 +0900 Subject: [PATCH 103/117] build: update git in dockerfiles (#6107) --- contrib/build-linux/appimage/Dockerfile | 2 +- contrib/build-wine/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/build-linux/appimage/Dockerfile b/contrib/build-linux/appimage/Dockerfile index 3a6d5768e8..60f08cfea6 100644 --- a/contrib/build-linux/appimage/Dockerfile +++ b/contrib/build-linux/appimage/Dockerfile @@ -4,7 +4,7 @@ ENV LC_ALL=C.UTF-8 LANG=C.UTF-8 RUN apt-get update -q && \ apt-get install -qy \ - git=1:2.7.4-0ubuntu1.8 \ + git=1:2.7.4-0ubuntu1.9 \ wget=1.17.1-1ubuntu1.5 \ make=4.1-6 \ autotools-dev=20150820.1 \ diff --git a/contrib/build-wine/Dockerfile b/contrib/build-wine/Dockerfile index 7064b38ee1..5b10a04ce8 100644 --- a/contrib/build-wine/Dockerfile +++ b/contrib/build-wine/Dockerfile @@ -13,7 +13,7 @@ RUN dpkg --add-architecture i386 && \ RUN apt-get update -q && \ apt-get install -qy \ - git=1:2.17.1-1ubuntu0.6 \ + git=1:2.17.1-1ubuntu0.7 \ p7zip-full=16.02+dfsg-6 \ make=4.1-9.1ubuntu1 \ mingw-w64=5.0.3-1 \ From 64733a39dcf702d271236eb044fa23263e9948f2 Mon Sep 17 00:00:00 2001 From: Evgeny Zinoviev Date: Wed, 22 Apr 2020 02:01:41 +0300 Subject: [PATCH 104/117] set more restrictive file permissions for exported private keys (#6106) --- electrum/gui/qt/main_window.py | 1 + 1 file changed, 1 insertion(+) diff --git a/electrum/gui/qt/main_window.py b/electrum/gui/qt/main_window.py index 2c65c895f5..437ee5be33 100644 --- a/electrum/gui/qt/main_window.py +++ b/electrum/gui/qt/main_window.py @@ -2729,6 +2729,7 @@ def on_dialog_closed(*args): def do_export_privkeys(self, fileName, pklist, is_csv): with open(fileName, "w+") as f: + os.chmod(fileName, 0o600) if is_csv: transaction = csv.writer(f) transaction.writerow(["address", "private_key"]) From 2d0ef78a11f99f198b8b6d3802a98b60a319583a Mon Sep 17 00:00:00 2001 From: ThomasV Date: Fri, 24 Apr 2020 11:45:39 +0200 Subject: [PATCH 105/117] channel_db: add verbose option to add_channel_update --- electrum/channel_db.py | 94 ++++++++++++++++++++++-------------------- electrum/lnworker.py | 11 ++--- 2 files changed, 55 insertions(+), 50 deletions(-) diff --git a/electrum/channel_db.py b/electrum/channel_db.py index 50cb212347..8be056d7f3 100644 --- a/electrum/channel_db.py +++ b/electrum/channel_db.py @@ -32,6 +32,7 @@ import base64 import asyncio import threading +from enum import IntEnum from .sql_db import SqlDB, sql @@ -196,13 +197,17 @@ def read(n): return addresses +class UpdateStatus(IntEnum): + ORPHANED = 0 + EXPIRED = 1 + DEPRECATED = 2 + GOOD = 3 + class CategorizedChannelUpdates(NamedTuple): orphaned: List # no channel announcement for channel update expired: List # update older than two weeks deprecated: List # update older than database entry good: List # good updates - to_delete: List # database entries to delete - create_channel_info = """ @@ -374,62 +379,61 @@ def print_change(self, old_policy: Policy, new_policy: Policy): if old_policy.message_flags != new_policy.message_flags: self.logger.info(f'message_flags: {old_policy.message_flags} -> {new_policy.message_flags}') - def add_channel_updates(self, payloads, max_age=None, verify=True) -> CategorizedChannelUpdates: + def add_channel_update(self, payload, max_age=None, verify=False, verbose=True): + now = int(time.time()) + short_channel_id = ShortChannelID(payload['short_channel_id']) + timestamp = payload['timestamp'] + if max_age and now - timestamp > max_age: + return UpdateStatus.EXPIRED + channel_info = self._channels.get(short_channel_id) + if not channel_info: + return UpdateStatus.ORPHANED + flags = int.from_bytes(payload['channel_flags'], 'big') + direction = flags & FLAG_DIRECTION + start_node = channel_info.node1_id if direction == 0 else channel_info.node2_id + payload['start_node'] = start_node + # compare updates to existing database entries + timestamp = payload['timestamp'] + start_node = payload['start_node'] + short_channel_id = ShortChannelID(payload['short_channel_id']) + key = (start_node, short_channel_id) + old_policy = self._policies.get(key) + if old_policy and timestamp <= old_policy.timestamp: + return UpdateStatus.DEPRECATED + if verify: + self.verify_channel_update(payload) + policy = Policy.from_msg(payload) + with self.lock: + self._policies[key] = policy + self._update_num_policies_for_chan(short_channel_id) + if 'raw' in payload: + self._db_save_policy(policy.key, payload['raw']) + if old_policy and verbose: + self.print_change(old_policy, policy) + return UpdateStatus.GOOD + + def add_channel_updates(self, payloads, max_age=None) -> CategorizedChannelUpdates: orphaned = [] expired = [] deprecated = [] good = [] - to_delete = [] - # filter orphaned and expired first - known = [] - now = int(time.time()) for payload in payloads: - short_channel_id = ShortChannelID(payload['short_channel_id']) - timestamp = payload['timestamp'] - if max_age and now - timestamp > max_age: - expired.append(payload) - continue - channel_info = self._channels.get(short_channel_id) - if not channel_info: + r = self.add_channel_update(payload, max_age=max_age, verbose=False) + if r == UpdateStatus.ORPHANED: orphaned.append(payload) - continue - flags = int.from_bytes(payload['channel_flags'], 'big') - direction = flags & FLAG_DIRECTION - start_node = channel_info.node1_id if direction == 0 else channel_info.node2_id - payload['start_node'] = start_node - known.append(payload) - # compare updates to existing database entries - for payload in known: - timestamp = payload['timestamp'] - start_node = payload['start_node'] - short_channel_id = ShortChannelID(payload['short_channel_id']) - key = (start_node, short_channel_id) - old_policy = self._policies.get(key) - if old_policy and timestamp <= old_policy.timestamp: + elif r == UpdateStatus.EXPIRED: + expired.append(payload) + elif r == UpdateStatus.DEPRECATED: deprecated.append(payload) - continue - good.append(payload) - if verify: - self.verify_channel_update(payload) - policy = Policy.from_msg(payload) - with self.lock: - self._policies[key] = policy - self._update_num_policies_for_chan(short_channel_id) - if 'raw' in payload: - self._db_save_policy(policy.key, payload['raw']) - # + elif r == UpdateStatus.GOOD: + good.append(payload) self.update_counts() return CategorizedChannelUpdates( orphaned=orphaned, expired=expired, deprecated=deprecated, - good=good, - to_delete=to_delete, - ) + good=good) - def add_channel_update(self, payload): - # called from tests - self.add_channel_updates([payload], verify=False) def create_database(self): c = self.conn.cursor() diff --git a/electrum/lnworker.py b/electrum/lnworker.py index 59064389af..8748f70422 100644 --- a/electrum/lnworker.py +++ b/electrum/lnworker.py @@ -67,6 +67,7 @@ from .crypto import pw_encode_bytes, pw_decode_bytes, PW_HASH_VERSION_LATEST from .lnutil import ChannelBackupStorage from .lnchannel import ChannelBackup +from .channel_db import UpdateStatus if TYPE_CHECKING: from .network import Network @@ -930,20 +931,20 @@ def handle_error_code_from_failed_htlc(self, failure_msg, sender_idx, route, pee if payload['chain_hash'] != constants.net.rev_genesis_bytes(): self.logger.info(f'could not decode channel_update for failed htlc: {channel_update_as_received.hex()}') return True - categorized_chan_upds = self.channel_db.add_channel_updates([payload]) + r = self.channel_db.add_channel_update(payload) blacklist = False short_channel_id = ShortChannelID(payload['short_channel_id']) - if categorized_chan_upds.good: + if r == UpdateStatus.GOOD: self.logger.info(f"applied channel update to {short_channel_id}") peer.maybe_save_remote_update(payload) - elif categorized_chan_upds.orphaned: + elif r == UpdateStatus.ORPHANED: # maybe it is a private channel (and data in invoice was outdated) self.logger.info(f"Could not find {short_channel_id}. maybe update is for private channel?") start_node_id = route[sender_idx].node_id self.channel_db.add_channel_update_for_private_channel(payload, start_node_id) - elif categorized_chan_upds.expired: + elif r == UpdateStatus.EXPIRED: blacklist = True - elif categorized_chan_upds.deprecated: + elif r == UpdateStatus.DEPRECATED: self.logger.info(f'channel update is not more recent.') blacklist = True else: From f4dc93cb7d8bbcd33163e8bd957315905f8548b2 Mon Sep 17 00:00:00 2001 From: ThomasV Date: Fri, 24 Apr 2020 12:16:21 +0200 Subject: [PATCH 106/117] lnworker: blacklist channel if policy is unchanged but has a new timestamp. --- electrum/channel_db.py | 50 +++++++++++++++++++++++++++++++----------- electrum/lnworker.py | 2 ++ 2 files changed, 39 insertions(+), 13 deletions(-) diff --git a/electrum/channel_db.py b/electrum/channel_db.py index 8be056d7f3..25178cd25a 100644 --- a/electrum/channel_db.py +++ b/electrum/channel_db.py @@ -201,12 +201,14 @@ class UpdateStatus(IntEnum): ORPHANED = 0 EXPIRED = 1 DEPRECATED = 2 - GOOD = 3 + UNCHANGED = 3 + GOOD = 4 class CategorizedChannelUpdates(NamedTuple): orphaned: List # no channel announcement for channel update expired: List # update older than two weeks deprecated: List # update older than database entry + unchanged: List # unchanged policies good: List # good updates @@ -362,22 +364,39 @@ def add_verified_channel_info(self, msg: dict, *, capacity_sat: int = None) -> N if 'raw' in msg: self._db_save_channel(channel_info.short_channel_id, msg['raw']) - def print_change(self, old_policy: Policy, new_policy: Policy): - # print what changed between policies + def policy_changed(self, old_policy: Policy, new_policy: Policy, verbose: bool) -> bool: + changed = False if old_policy.cltv_expiry_delta != new_policy.cltv_expiry_delta: - self.logger.info(f'cltv_expiry_delta: {old_policy.cltv_expiry_delta} -> {new_policy.cltv_expiry_delta}') + changed |= True + if verbose: + self.logger.info(f'cltv_expiry_delta: {old_policy.cltv_expiry_delta} -> {new_policy.cltv_expiry_delta}') if old_policy.htlc_minimum_msat != new_policy.htlc_minimum_msat: - self.logger.info(f'htlc_minimum_msat: {old_policy.htlc_minimum_msat} -> {new_policy.htlc_minimum_msat}') + changed |= True + if verbose: + self.logger.info(f'htlc_minimum_msat: {old_policy.htlc_minimum_msat} -> {new_policy.htlc_minimum_msat}') if old_policy.htlc_maximum_msat != new_policy.htlc_maximum_msat: - self.logger.info(f'htlc_maximum_msat: {old_policy.htlc_maximum_msat} -> {new_policy.htlc_maximum_msat}') + changed |= True + if verbose: + self.logger.info(f'htlc_maximum_msat: {old_policy.htlc_maximum_msat} -> {new_policy.htlc_maximum_msat}') if old_policy.fee_base_msat != new_policy.fee_base_msat: - self.logger.info(f'fee_base_msat: {old_policy.fee_base_msat} -> {new_policy.fee_base_msat}') + changed |= True + if verbose: + self.logger.info(f'fee_base_msat: {old_policy.fee_base_msat} -> {new_policy.fee_base_msat}') if old_policy.fee_proportional_millionths != new_policy.fee_proportional_millionths: - self.logger.info(f'fee_proportional_millionths: {old_policy.fee_proportional_millionths} -> {new_policy.fee_proportional_millionths}') + changed |= True + if verbose: + self.logger.info(f'fee_proportional_millionths: {old_policy.fee_proportional_millionths} -> {new_policy.fee_proportional_millionths}') if old_policy.channel_flags != new_policy.channel_flags: - self.logger.info(f'channel_flags: {old_policy.channel_flags} -> {new_policy.channel_flags}') + changed |= True + if verbose: + self.logger.info(f'channel_flags: {old_policy.channel_flags} -> {new_policy.channel_flags}') if old_policy.message_flags != new_policy.message_flags: - self.logger.info(f'message_flags: {old_policy.message_flags} -> {new_policy.message_flags}') + changed |= True + if verbose: + self.logger.info(f'message_flags: {old_policy.message_flags} -> {new_policy.message_flags}') + if not changed and verbose: + self.logger.info(f'policy unchanged: {old_policy.timestamp} -> {new_policy.timestamp}') + return changed def add_channel_update(self, payload, max_age=None, verify=False, verbose=True): now = int(time.time()) @@ -408,14 +427,16 @@ def add_channel_update(self, payload, max_age=None, verify=False, verbose=True): self._update_num_policies_for_chan(short_channel_id) if 'raw' in payload: self._db_save_policy(policy.key, payload['raw']) - if old_policy and verbose: - self.print_change(old_policy, policy) - return UpdateStatus.GOOD + if old_policy and not self.policy_changed(old_policy, policy, verbose): + return UpdateStatus.UNCHANGED + else: + return UpdateStatus.GOOD def add_channel_updates(self, payloads, max_age=None) -> CategorizedChannelUpdates: orphaned = [] expired = [] deprecated = [] + unchanged = [] good = [] for payload in payloads: r = self.add_channel_update(payload, max_age=max_age, verbose=False) @@ -425,6 +446,8 @@ def add_channel_updates(self, payloads, max_age=None) -> CategorizedChannelUpdat expired.append(payload) elif r == UpdateStatus.DEPRECATED: deprecated.append(payload) + elif r == UpdateStatus.UNCHANGED: + unchanged.append(payload) elif r == UpdateStatus.GOOD: good.append(payload) self.update_counts() @@ -432,6 +455,7 @@ def add_channel_updates(self, payloads, max_age=None) -> CategorizedChannelUpdat orphaned=orphaned, expired=expired, deprecated=deprecated, + unchanged=unchanged, good=good) diff --git a/electrum/lnworker.py b/electrum/lnworker.py index 8748f70422..af0612cc1d 100644 --- a/electrum/lnworker.py +++ b/electrum/lnworker.py @@ -947,6 +947,8 @@ def handle_error_code_from_failed_htlc(self, failure_msg, sender_idx, route, pee elif r == UpdateStatus.DEPRECATED: self.logger.info(f'channel update is not more recent.') blacklist = True + elif r == UpdateStatus.UNCHANGED: + blacklist = True else: blacklist = True return blacklist From 54fdb011f99eb19ceaeb6e778c2f117ec074744c Mon Sep 17 00:00:00 2001 From: SomberNight Date: Fri, 24 Apr 2020 15:32:05 +0200 Subject: [PATCH 107/117] fixups for CallbackManager refactor 92244041081db96b92925c9e76b117035e241011 --- electrum/util.py | 2 +- electrum/wallet.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/electrum/util.py b/electrum/util.py index 6f998bb7f9..3c907d1d08 100644 --- a/electrum/util.py +++ b/electrum/util.py @@ -1152,7 +1152,7 @@ async def _start_tasks(self): raise NotImplementedError() # implemented by subclasses async def stop(self): - self.network.unregister_callback(self._restart) + unregister_callback(self._restart) await self._stop() async def _stop(self): diff --git a/electrum/wallet.py b/electrum/wallet.py index 68ff677d55..7bfb2e4618 100644 --- a/electrum/wallet.py +++ b/electrum/wallet.py @@ -46,6 +46,7 @@ from .i18n import _ from .bip32 import BIP32Node, convert_bip32_intpath_to_strpath, convert_bip32_path_to_list_of_uint32 from .crypto import sha256 +from . import util from .util import (NotEnoughFunds, UserCancelled, profiler, format_satoshis, format_fee_satoshis, NoDynamicFeeEstimates, WalletFileException, BitcoinException, MultipleSpendMaxTxOutputs, @@ -1615,7 +1616,7 @@ def receive_tx_callback(self, tx_hash, tx, tx_height): addr = self.get_txout_address(txo) if addr in self.receive_requests: status, conf = self.get_request_status(addr) - self.network.trigger_callback('payment_received', self, addr, status) + util.trigger_callback('payment_received', self, addr, status) def make_payment_request(self, addr, amount, message, expiration): timestamp = int(time.time()) From e2ae44beb99c8be98a8a978e7e69612534fe4621 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Fri, 24 Apr 2020 15:34:55 +0200 Subject: [PATCH 108/117] commands: "notify" cmd: stop watching addr if called with empty URL closes #5881 --- electrum/commands.py | 11 ++++++++--- electrum/synchronizer.py | 13 +++++++++++-- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/electrum/commands.py b/electrum/commands.py index 3433dc6094..6d8a780f39 100644 --- a/electrum/commands.py +++ b/electrum/commands.py @@ -898,11 +898,16 @@ async def clear_invoices(self, wallet: Abstract_Wallet = None): return True @command('n') - async def notify(self, address: str, URL: str): - """Watch an address. Every time the address changes, a http POST is sent to the URL.""" + async def notify(self, address: str, URL: Optional[str]): + """Watch an address. Every time the address changes, a http POST is sent to the URL. + Call with an empty URL to stop watching an address. + """ if not hasattr(self, "_notifier"): self._notifier = Notifier(self.network) - await self._notifier.start_watching_queue.put((address, URL)) + if URL: + await self._notifier.start_watching_addr(address, URL) + else: + await self._notifier.stop_watching_addr(address) return True @command('wn') diff --git a/electrum/synchronizer.py b/electrum/synchronizer.py index 6c47a9baf0..6363eeab81 100644 --- a/electrum/synchronizer.py +++ b/electrum/synchronizer.py @@ -263,7 +263,7 @@ class Notifier(SynchronizerBase): def __init__(self, network): SynchronizerBase.__init__(self, network) self.watched_addresses = defaultdict(list) # type: Dict[str, List[str]] - self.start_watching_queue = asyncio.Queue() + self._start_watching_queue = asyncio.Queue() # type: asyncio.Queue[Tuple[str, str]] async def main(self): # resend existing subscriptions if we were restarted @@ -271,11 +271,20 @@ async def main(self): await self._add_address(addr) # main loop while True: - addr, url = await self.start_watching_queue.get() + addr, url = await self._start_watching_queue.get() self.watched_addresses[addr].append(url) await self._add_address(addr) + async def start_watching_addr(self, addr: str, url: str): + await self._start_watching_queue.put((addr, url)) + + async def stop_watching_addr(self, addr: str): + self.watched_addresses.pop(addr, None) + # TODO blockchain.scripthash.unsubscribe + async def _on_address_status(self, addr, status): + if addr not in self.watched_addresses: + return self.logger.info(f'new status for addr {addr}') headers = {'content-type': 'application/json'} data = {'address': addr, 'status': status} From ca1046bce2c8d57c03c7803fab0e14b7c119b2c6 Mon Sep 17 00:00:00 2001 From: Luke Childs Date: Fri, 24 Apr 2020 21:11:40 +0700 Subject: [PATCH 109/117] Add --serverfingerprint option (#6094) * Add --fingerprint option * Simplify conditional checks * Improve warning wording * Throw error instead of logging and returning * --fingerprint => --serverfingerprint * Only run fingerprint checks against main server * Throw error if --serverfingerprint is set for a non SSL main server * Fix linting errors * Don't check certificate fingerprint in a seperate connection * Disallow CA signed certs when a fingerprint is provided * Show clear error and then exit for Qt GUI users * Remove leading newlines from error dialog * Always check is_main_server() when getting fingerprint * Document how to generate SSL cert fingerprint --- electrum/commands.py | 2 ++ electrum/gui/qt/main_window.py | 16 +++++++++++++++- electrum/interface.py | 26 +++++++++++++++++++++++++- 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/electrum/commands.py b/electrum/commands.py index 6d8a780f39..03cb5847e1 100644 --- a/electrum/commands.py +++ b/electrum/commands.py @@ -1268,6 +1268,8 @@ def subparser_call(self, parser, namespace, values, option_string=None): def add_network_options(parser): + parser.add_argument("-f", "--serverfingerprint", dest="serverfingerprint", default=None, help="only allow connecting to servers with a matching SSL certificate SHA256 fingerprint." + " " + + "To calculate this yourself: '$ openssl x509 -noout -fingerprint -sha256 -inform pem -in mycertfile.crt'. Enter as 64 hex chars.") parser.add_argument("-1", "--oneserver", action="store_true", dest="oneserver", default=None, help="connect to one server only") parser.add_argument("-s", "--server", dest="server", default=None, help="set server host:port:protocol, where protocol is either t (tcp) or s (ssl)") parser.add_argument("-p", "--proxy", dest="proxy", default=None, help="set proxy [type:]host[:port] (or 'none' to disable proxy), where type is socks4,socks5 or http") diff --git a/electrum/gui/qt/main_window.py b/electrum/gui/qt/main_window.py index 437ee5be33..60472e2a40 100644 --- a/electrum/gui/qt/main_window.py +++ b/electrum/gui/qt/main_window.py @@ -183,6 +183,7 @@ def __init__(self, gui_object: 'ElectrumGui', wallet: Abstract_Wallet): self.checking_accounts = False self.qr_window = None self.pluginsdialog = None + self.showing_cert_mismatch_error = False self.tl_windows = [] Logger.__init__(self) @@ -267,7 +268,8 @@ def add_optional_tab(tabs, tab, icon, description, name): 'banner', 'verified', 'fee', 'fee_histogram', 'on_quotes', 'on_history', 'channel', 'channels_updated', 'payment_failed', 'payment_succeeded', - 'invoice_status', 'request_status', 'ln_gossip_sync_progress'] + 'invoice_status', 'request_status', 'ln_gossip_sync_progress', + 'cert_mismatch'] # To avoid leaking references to "self" that prevent the # window from being GC-ed when closed, callbacks should be # methods of this class only, and specifically not be @@ -442,6 +444,8 @@ def on_network_qt(self, event, args=None): self.history_model.on_fee_histogram() elif event == 'ln_gossip_sync_progress': self.update_lightning_icon() + elif event == 'cert_mismatch': + self.show_cert_mismatch_error() else: self.logger.info(f"unexpected network event: {event} {args}") @@ -3119,3 +3123,13 @@ def save_transaction_into_wallet(self, tx: Transaction): "to see it, you need to broadcast it.")) win.msg_box(QPixmap(icon_path("offline_tx.png")), None, _('Success'), msg) return True + + def show_cert_mismatch_error(self): + if self.showing_cert_mismatch_error: + return + self.showing_cert_mismatch_error = True + self.show_critical(title=_("Certificate mismatch"), + msg=_("The SSL certificate provided by the main server did not match the fingerprint passed in with the --serverfingerprint option.") + "\n\n" + + _("Electrum will now exit.")) + self.showing_cert_mismatch_error = False + self.close() diff --git a/electrum/interface.py b/electrum/interface.py index 5600524e5a..15e6706e6c 100644 --- a/electrum/interface.py +++ b/electrum/interface.py @@ -34,6 +34,7 @@ from ipaddress import IPv4Network, IPv6Network, ip_address, IPv6Address import itertools import logging +import hashlib import aiorpcx from aiorpcx import TaskGroup @@ -190,6 +191,8 @@ class RequestCorrupted(GracefulDisconnect): pass class ErrorParsingSSLCert(Exception): pass class ErrorGettingSSLCertFromServer(Exception): pass +class ErrorSSLCertFingerprintMismatch(Exception): pass +class InvalidOptionCombination(Exception): pass class ConnectError(NetworkException): pass @@ -350,6 +353,8 @@ async def is_server_ca_signed(self, ca_ssl_context): async def _try_saving_ssl_cert_for_first_time(self, ca_ssl_context): ca_signed = await self.is_server_ca_signed(ca_ssl_context) if ca_signed: + if self.get_expected_fingerprint(): + raise InvalidOptionCombination("cannot use --serverfingerprint with CA signed servers") with open(self.cert_path, 'w') as f: # empty file means this is CA signed, not self-signed f.write('') @@ -362,6 +367,8 @@ def _is_saved_ssl_cert_available(self): with open(self.cert_path, 'r') as f: contents = f.read() if contents == '': # CA signed + if self.get_expected_fingerprint(): + raise InvalidOptionCombination("cannot use --serverfingerprint with CA signed servers") return True # pinned self-signed cert try: @@ -376,11 +383,12 @@ def _is_saved_ssl_cert_available(self): raise ErrorParsingSSLCert(e) from e try: x.check_date() - return True except x509.CertificateError as e: self.logger.info(f"certificate has expired: {e}") os.unlink(self.cert_path) # delete pinned cert only in this case return False + self.verify_certificate_fingerprint(bytearray(b)) + return True async def _get_ssl_context(self): if self.protocol != 's': @@ -468,6 +476,7 @@ async def save_certificate(self): dercert = await self.get_certificate() if dercert: self.logger.info("succeeded in getting cert") + self.verify_certificate_fingerprint(dercert) with open(self.cert_path, 'w') as f: cert = ssl.DER_cert_to_PEM_cert(dercert) # workaround android bug @@ -492,6 +501,21 @@ async def get_certificate(self): ssl_object = asyncio_transport.get_extra_info("ssl_object") # type: ssl.SSLObject return ssl_object.getpeercert(binary_form=True) + def get_expected_fingerprint(self): + if self.is_main_server(): + return self.network.config.get("serverfingerprint") + + def verify_certificate_fingerprint(self, certificate): + expected_fingerprint = self.get_expected_fingerprint() + if not expected_fingerprint: + return + fingerprint = hashlib.sha256(certificate).hexdigest() + fingerprints_match = fingerprint.lower() == expected_fingerprint.lower() + if not fingerprints_match: + util.trigger_callback('cert_mismatch') + raise ErrorSSLCertFingerprintMismatch('Refusing to connect to server due to cert fingerprint mismatch') + self.logger.info("cert fingerprint verification passed") + async def get_block_header(self, height, assert_mode): self.logger.info(f'requesting block header {height} in mode {assert_mode}') # use lower timeout as we usually have network.bhi_lock here From 69de3b94db5594ddff36db8c926d787b45baa92e Mon Sep 17 00:00:00 2001 From: SomberNight Date: Fri, 24 Apr 2020 17:17:12 +0200 Subject: [PATCH 110/117] config: "serverfingerprint" key requires "server" key follow-up prev --- electrum/gui/qt/network_dialog.py | 7 +++++-- electrum/simple_config.py | 18 ++++++++++++++++-- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/electrum/gui/qt/network_dialog.py b/electrum/gui/qt/network_dialog.py index bbede847cb..1473792dfb 100644 --- a/electrum/gui/qt/network_dialog.py +++ b/electrum/gui/qt/network_dialog.py @@ -26,7 +26,7 @@ import socket import time from enum import IntEnum -from typing import Tuple +from typing import Tuple, TYPE_CHECKING from PyQt5.QtCore import Qt, pyqtSignal, QThread from PyQt5.QtWidgets import (QTreeWidget, QTreeWidgetItem, QMenu, QGridLayout, QComboBox, @@ -43,6 +43,9 @@ from .util import (Buttons, CloseButton, HelpButton, read_QIcon, char_width_in_lineedit, PasswordLineEdit) +if TYPE_CHECKING: + from electrum.simple_config import SimpleConfig + _logger = get_logger(__name__) @@ -209,7 +212,7 @@ def update(self, servers, use_tor): class NetworkChoiceLayout(object): - def __init__(self, network: Network, config, wizard=False): + def __init__(self, network: Network, config: 'SimpleConfig', wizard=False): self.network = network self.config = config self.tor_proxy = None diff --git a/electrum/simple_config.py b/electrum/simple_config.py index bd454e798c..72fcc5974b 100644 --- a/electrum/simple_config.py +++ b/electrum/simple_config.py @@ -88,6 +88,8 @@ def __init__(self, options=None, read_user_config_function=None, # avoid new config getting upgraded self.user_config = {'config_version': FINAL_CONFIG_VERSION} + self._not_modifiable_keys = set() + # config "upgrade" - CLI options self.rename_config_keys( self.cmdline_options, {'auto_cycle': 'auto_connect'}, True) @@ -96,6 +98,8 @@ def __init__(self, options=None, read_user_config_function=None, if self.requires_upgrade(): self.upgrade() + self._check_dependent_keys() + def electrum_path(self): # Read electrum_path from command line # Otherwise use the user's default data directory. @@ -159,6 +163,12 @@ def get(self, key, default=None): out = self.user_config.get(key, default) return out + def _check_dependent_keys(self) -> None: + if self.get('serverfingerprint'): + if not self.get('server'): + raise Exception("config key 'serverfingerprint' requires 'server' to also be set") + self.make_key_not_modifiable('server') + def requires_upgrade(self): return self.get_config_version() < FINAL_CONFIG_VERSION @@ -221,8 +231,12 @@ def get_config_version(self): .format(config_version, FINAL_CONFIG_VERSION)) return config_version - def is_modifiable(self, key): - return key not in self.cmdline_options + def is_modifiable(self, key) -> bool: + return (key not in self.cmdline_options + and key not in self._not_modifiable_keys) + + def make_key_not_modifiable(self, key) -> None: + self._not_modifiable_keys.add(key) def save_user_config(self): if self.get('forget_config'): From 38980a4f5cdccd3816bc0130c8471104fa805aab Mon Sep 17 00:00:00 2001 From: SomberNight Date: Fri, 24 Apr 2020 17:18:05 +0200 Subject: [PATCH 111/117] interface: (trivial) make some methods private --- electrum/interface.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/electrum/interface.py b/electrum/interface.py index 15e6706e6c..8834286a26 100644 --- a/electrum/interface.py +++ b/electrum/interface.py @@ -353,13 +353,13 @@ async def is_server_ca_signed(self, ca_ssl_context): async def _try_saving_ssl_cert_for_first_time(self, ca_ssl_context): ca_signed = await self.is_server_ca_signed(ca_ssl_context) if ca_signed: - if self.get_expected_fingerprint(): + if self._get_expected_fingerprint(): raise InvalidOptionCombination("cannot use --serverfingerprint with CA signed servers") with open(self.cert_path, 'w') as f: # empty file means this is CA signed, not self-signed f.write('') else: - await self.save_certificate() + await self._save_certificate() def _is_saved_ssl_cert_available(self): if not os.path.exists(self.cert_path): @@ -367,7 +367,7 @@ def _is_saved_ssl_cert_available(self): with open(self.cert_path, 'r') as f: contents = f.read() if contents == '': # CA signed - if self.get_expected_fingerprint(): + if self._get_expected_fingerprint(): raise InvalidOptionCombination("cannot use --serverfingerprint with CA signed servers") return True # pinned self-signed cert @@ -387,7 +387,7 @@ def _is_saved_ssl_cert_available(self): self.logger.info(f"certificate has expired: {e}") os.unlink(self.cert_path) # delete pinned cert only in this case return False - self.verify_certificate_fingerprint(bytearray(b)) + self._verify_certificate_fingerprint(bytearray(b)) return True async def _get_ssl_context(self): @@ -469,14 +469,14 @@ def _mark_ready(self) -> None: self.ready.set_result(1) - async def save_certificate(self): + async def _save_certificate(self) -> None: if not os.path.exists(self.cert_path): # we may need to retry this a few times, in case the handshake hasn't completed for _ in range(10): - dercert = await self.get_certificate() + dercert = await self._fetch_certificate() if dercert: self.logger.info("succeeded in getting cert") - self.verify_certificate_fingerprint(dercert) + self._verify_certificate_fingerprint(dercert) with open(self.cert_path, 'w') as f: cert = ssl.DER_cert_to_PEM_cert(dercert) # workaround android bug @@ -492,7 +492,7 @@ async def save_certificate(self): else: raise GracefulDisconnect("could not get certificate after 10 tries") - async def get_certificate(self): + async def _fetch_certificate(self) -> bytes: sslc = ssl.SSLContext() async with _RSClient(session_factory=RPCSession, host=self.host, port=self.port, @@ -501,12 +501,12 @@ async def get_certificate(self): ssl_object = asyncio_transport.get_extra_info("ssl_object") # type: ssl.SSLObject return ssl_object.getpeercert(binary_form=True) - def get_expected_fingerprint(self): + def _get_expected_fingerprint(self) -> Optional[str]: if self.is_main_server(): return self.network.config.get("serverfingerprint") - def verify_certificate_fingerprint(self, certificate): - expected_fingerprint = self.get_expected_fingerprint() + def _verify_certificate_fingerprint(self, certificate): + expected_fingerprint = self._get_expected_fingerprint() if not expected_fingerprint: return fingerprint = hashlib.sha256(certificate).hexdigest() From 56a9ccca6df0c992b845ea363a6696c2efbcabb7 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Sat, 25 Apr 2020 06:38:26 +0200 Subject: [PATCH 112/117] interface: make localhost exempt from ip-range bucketing --- electrum/interface.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/electrum/interface.py b/electrum/interface.py index 8834286a26..7785019414 100644 --- a/electrum/interface.py +++ b/electrum/interface.py @@ -31,7 +31,7 @@ import socket from typing import Tuple, Union, List, TYPE_CHECKING, Optional, Set, NamedTuple from collections import defaultdict -from ipaddress import IPv4Network, IPv6Network, ip_address, IPv6Address +from ipaddress import IPv4Network, IPv6Network, ip_address, IPv6Address, IPv4Address import itertools import logging import hashlib @@ -794,11 +794,13 @@ def do_bucket(): if self.is_tor(): return BUCKET_NAME_OF_ONION_SERVERS try: - ip_addr = ip_address(self.ip_addr()) + ip_addr = ip_address(self.ip_addr()) # type: Union[IPv4Address, IPv6Address] except ValueError: return '' if not ip_addr: return '' + if ip_addr.is_loopback: # localhost is exempt + return '' if ip_addr.version == 4: slash16 = IPv4Network(ip_addr).supernet(prefixlen_diff=32-16) return str(slash16) From bf223470ce607a762700f144a6a0f4cb0205cf54 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Sat, 25 Apr 2020 06:53:25 +0200 Subject: [PATCH 113/117] network: handle unparseable server-str follow-up 9e57ae630ba96e6d2c40288d2633aeca8d20764d fixes #6113 --- electrum/gui/kivy/main_window.py | 1 + electrum/gui/qt/network_dialog.py | 1 + 2 files changed, 2 insertions(+) diff --git a/electrum/gui/kivy/main_window.py b/electrum/gui/kivy/main_window.py index 63ea82eeea..7793a034fe 100644 --- a/electrum/gui/kivy/main_window.py +++ b/electrum/gui/kivy/main_window.py @@ -150,6 +150,7 @@ def maybe_switch_to_server(self, server_str: str): net_params = self.network.get_parameters() try: server = ServerAddr.from_str_with_inference(server_str) + if not server: raise Exception("failed to parse") except Exception as e: self.show_error(_("Invalid server details: {}").format(repr(e))) return diff --git a/electrum/gui/qt/network_dialog.py b/electrum/gui/qt/network_dialog.py index 1473792dfb..8aed26ea57 100644 --- a/electrum/gui/qt/network_dialog.py +++ b/electrum/gui/qt/network_dialog.py @@ -428,6 +428,7 @@ def set_server(self): net_params = self.network.get_parameters() try: server = ServerAddr.from_str_with_inference(str(self.server_e.text())) + if not server: raise Exception("failed to parse") except Exception: return net_params = net_params._replace(server=server, From 58dee38ed2b7a6cde996d3cd2ff81c8fd9942c99 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Sat, 25 Apr 2020 06:15:35 +0200 Subject: [PATCH 114/117] qt network dialog: merge "Overview" and "Servers" tabs --- electrum/gui/qt/network_dialog.py | 200 ++++++++++++------------------ 1 file changed, 79 insertions(+), 121 deletions(-) diff --git a/electrum/gui/qt/network_dialog.py b/electrum/gui/qt/network_dialog.py index 8aed26ea57..3793687d2a 100644 --- a/electrum/gui/qt/network_dialog.py +++ b/electrum/gui/qt/network_dialog.py @@ -56,7 +56,7 @@ class NetworkDialog(QDialog): def __init__(self, network, config, network_updated_signal_obj): QDialog.__init__(self) self.setWindowTitle(_('Network')) - self.setMinimumSize(500, 300) + self.setMinimumSize(500, 500) self.nlayout = NetworkChoiceLayout(network, config) self.network_updated_signal_obj = network_updated_signal_obj vbox = QVBoxLayout(self) @@ -79,12 +79,18 @@ class NodesListWidget(QTreeWidget): SERVER_ADDR_ROLE = Qt.UserRole + 100 CHAIN_ID_ROLE = Qt.UserRole + 101 - IS_SERVER_ROLE = Qt.UserRole + 102 + ITEMTYPE_ROLE = Qt.UserRole + 102 + + class ItemType(IntEnum): + CHAIN = 0 + CONNECTED_SERVER = 1 + DISCONNECTED_SERVER = 2 + TOPLEVEL = 3 def __init__(self, parent): QTreeWidget.__init__(self) self.parent = parent # type: NetworkChoiceLayout - self.setHeaderLabels([_('Connected node'), _('Height')]) + self.setHeaderLabels([_('Server'), _('Height')]) self.setContextMenuPolicy(Qt.CustomContextMenu) self.customContextMenuRequested.connect(self.create_menu) @@ -92,14 +98,22 @@ def create_menu(self, position): item = self.currentItem() if not item: return - is_server = bool(item.data(0, self.IS_SERVER_ROLE)) + item_type = item.data(0, self.ITEMTYPE_ROLE) menu = QMenu() - if is_server: + if item_type == self.ItemType.CONNECTED_SERVER: server = item.data(0, self.SERVER_ADDR_ROLE) # type: ServerAddr menu.addAction(_("Use as server"), lambda: self.parent.follow_server(server)) - else: + elif item_type == self.ItemType.DISCONNECTED_SERVER: + server = item.data(0, self.SERVER_ADDR_ROLE) # type: ServerAddr + def func(): + self.parent.server_e.setText(server.net_addr_str()) + self.parent.set_server() + menu.addAction(_("Use as server"), func) + elif item_type == self.ItemType.CHAIN: chain_id = item.data(0, self.CHAIN_ID_ROLE) menu.addAction(_("Follow this branch"), lambda: self.parent.follow_branch(chain_id)) + else: + return menu.exec_(self.viewport().mapToGlobal(position)) def keyPressEvent(self, event): @@ -114,9 +128,12 @@ def on_activated(self, item, column): pt.setX(50) self.customContextMenuRequested.emit(pt) - def update(self, network: Network): + def update(self, *, network: Network, servers: dict, use_tor: bool): self.clear() - self.addChild = self.addTopLevelItem + + # connected servers + connected_servers_item = QTreeWidgetItem([_("Connected nodes"), '']) + connected_servers_item.setData(0, self.ITEMTYPE_ROLE, self.ItemType.TOPLEVEL) chains = network.get_blockchains() n_chains = len(chains) for chain_id, interfaces in chains.items(): @@ -125,87 +142,51 @@ def update(self, network: Network): name = b.get_name() if n_chains > 1: x = QTreeWidgetItem([name + '@%d'%b.get_max_forkpoint(), '%d'%b.height()]) - x.setData(0, self.IS_SERVER_ROLE, 0) + x.setData(0, self.ITEMTYPE_ROLE, self.ItemType.CHAIN) x.setData(0, self.CHAIN_ID_ROLE, b.get_id()) else: - x = self + x = connected_servers_item for i in interfaces: star = ' *' if i == network.interface else '' - item = QTreeWidgetItem([i.host + star, '%d'%i.tip]) - item.setData(0, self.IS_SERVER_ROLE, 1) + item = QTreeWidgetItem([f"{i.server.net_addr_str()}" + star, '%d'%i.tip]) + item.setData(0, self.ITEMTYPE_ROLE, self.ItemType.CONNECTED_SERVER) item.setData(0, self.SERVER_ADDR_ROLE, i.server) item.setToolTip(0, str(i.server)) x.addChild(item) if n_chains > 1: - self.addTopLevelItem(x) - x.setExpanded(True) + connected_servers_item.addChild(x) - h = self.header() - h.setStretchLastSection(False) - h.setSectionResizeMode(0, QHeaderView.Stretch) - h.setSectionResizeMode(1, QHeaderView.ResizeToContents) - - super().update() - - -class ServerListWidget(QTreeWidget): - """List of all known servers.""" - - class Columns(IntEnum): - HOST = 0 - PORT = 1 - - SERVER_ADDR_ROLE = Qt.UserRole + 100 - - def __init__(self, parent): - QTreeWidget.__init__(self) - self.parent = parent # type: NetworkChoiceLayout - self.setHeaderLabels([_('Host'), _('Port')]) - self.setContextMenuPolicy(Qt.CustomContextMenu) - self.customContextMenuRequested.connect(self.create_menu) - - def create_menu(self, position): - item = self.currentItem() - if not item: - return - menu = QMenu() - server = item.data(self.Columns.HOST, self.SERVER_ADDR_ROLE) - menu.addAction(_("Use as server"), lambda: self.set_server(server)) - menu.exec_(self.viewport().mapToGlobal(position)) - - def set_server(self, server: ServerAddr): - self.parent.server_e.setText(server.net_addr_str()) - self.parent.set_server() - - def keyPressEvent(self, event): - if event.key() in [ Qt.Key_F2, Qt.Key_Return ]: - self.on_activated(self.currentItem(), self.currentColumn()) - else: - QTreeWidget.keyPressEvent(self, event) - - def on_activated(self, item, column): - # on 'enter' we show the menu - pt = self.visualItemRect(item).bottomLeft() - pt.setX(50) - self.customContextMenuRequested.emit(pt) - - def update(self, servers, use_tor): - self.clear() + # disconnected servers + disconnected_servers_item = QTreeWidgetItem([_("Other known servers"), ""]) + disconnected_servers_item.setData(0, self.ITEMTYPE_ROLE, self.ItemType.TOPLEVEL) + connected_hosts = set([iface.host for ifaces in chains.values() for iface in ifaces]) protocol = PREFERRED_NETWORK_PROTOCOL for _host, d in sorted(servers.items()): + if _host in connected_hosts: + continue if _host.endswith('.onion') and not use_tor: continue port = d.get(protocol) if port: - x = QTreeWidgetItem([_host, port]) server = ServerAddr(_host, port, protocol=protocol) - x.setData(self.Columns.HOST, self.SERVER_ADDR_ROLE, server) - self.addTopLevelItem(x) + item = QTreeWidgetItem([server.net_addr_str(), ""]) + item.setData(0, self.ITEMTYPE_ROLE, self.ItemType.DISCONNECTED_SERVER) + item.setData(0, self.SERVER_ADDR_ROLE, server) + disconnected_servers_item.addChild(item) + + self.addTopLevelItem(connected_servers_item) + self.addTopLevelItem(disconnected_servers_item) + + connected_servers_item.setExpanded(True) + for i in range(connected_servers_item.childCount()): + connected_servers_item.child(i).setExpanded(True) + disconnected_servers_item.setExpanded(True) + # headers h = self.header() h.setStretchLastSection(False) - h.setSectionResizeMode(self.Columns.HOST, QHeaderView.Stretch) - h.setSectionResizeMode(self.Columns.PORT, QHeaderView.ResizeToContents) + h.setSectionResizeMode(0, QHeaderView.Stretch) + h.setSectionResizeMode(1, QHeaderView.ResizeToContents) super().update() @@ -218,44 +199,14 @@ def __init__(self, network: Network, config: 'SimpleConfig', wizard=False): self.tor_proxy = None self.tabs = tabs = QTabWidget() - server_tab = QWidget() proxy_tab = QWidget() blockchain_tab = QWidget() tabs.addTab(blockchain_tab, _('Overview')) - tabs.addTab(server_tab, _('Server')) tabs.addTab(proxy_tab, _('Proxy')) fixed_width_hostname = 24 * char_width_in_lineedit() fixed_width_port = 6 * char_width_in_lineedit() - # server tab - grid = QGridLayout(server_tab) - grid.setSpacing(8) - - self.server_e = QLineEdit() - self.server_e.setFixedWidth(fixed_width_hostname + fixed_width_port) - self.autoconnect_cb = QCheckBox(_('Select server automatically')) - self.autoconnect_cb.setEnabled(self.config.is_modifiable('auto_connect')) - - self.server_e.editingFinished.connect(self.set_server) - self.autoconnect_cb.clicked.connect(self.set_server) - self.autoconnect_cb.clicked.connect(self.update) - - msg = ' '.join([ - _("If auto-connect is enabled, Electrum will always use a server that is on the longest blockchain."), - _("If it is disabled, you have to choose a server you want to use. Electrum will warn you if your server is lagging.") - ]) - grid.addWidget(self.autoconnect_cb, 0, 0, 1, 3) - grid.addWidget(HelpButton(msg), 0, 4) - - grid.addWidget(QLabel(_('Server') + ':'), 1, 0) - grid.addWidget(self.server_e, 1, 1, 1, 3) - - label = _('Server peers') if network.is_connected() else _('Default Servers') - grid.addWidget(QLabel(label), 2, 0, 1, 5) - self.servers_list = ServerListWidget(self) - grid.addWidget(self.servers_list, 3, 0, 1, 5) - # Proxy tab grid = QGridLayout(proxy_tab) grid.setSpacing(8) @@ -315,23 +266,36 @@ def __init__(self, network: Network, config: 'SimpleConfig', wizard=False): grid.addWidget(self.status_label, 0, 1, 1, 3) grid.addWidget(HelpButton(msg), 0, 4) - self.server_label = QLabel('') - msg = _("Electrum sends your wallet addresses to a single server, in order to receive your transaction history.") - grid.addWidget(QLabel(_('Server') + ':'), 1, 0) - grid.addWidget(self.server_label, 1, 1, 1, 3) + self.autoconnect_cb = QCheckBox(_('Select server automatically')) + self.autoconnect_cb.setEnabled(self.config.is_modifiable('auto_connect')) + self.autoconnect_cb.clicked.connect(self.set_server) + self.autoconnect_cb.clicked.connect(self.update) + msg = ' '.join([ + _("If auto-connect is enabled, Electrum will always use a server that is on the longest blockchain."), + _("If it is disabled, you have to choose a server you want to use. Electrum will warn you if your server is lagging.") + ]) + grid.addWidget(self.autoconnect_cb, 1, 0, 1, 3) grid.addWidget(HelpButton(msg), 1, 4) + self.server_e = QLineEdit() + self.server_e.setFixedWidth(fixed_width_hostname + fixed_width_port) + self.server_e.editingFinished.connect(self.set_server) + msg = _("Electrum sends your wallet addresses to a single server, in order to receive your transaction history.") + grid.addWidget(QLabel(_('Server') + ':'), 2, 0) + grid.addWidget(self.server_e, 2, 1, 1, 3) + grid.addWidget(HelpButton(msg), 2, 4) + self.height_label = QLabel('') msg = _('This is the height of your local copy of the blockchain.') - grid.addWidget(QLabel(_('Blockchain') + ':'), 2, 0) - grid.addWidget(self.height_label, 2, 1) - grid.addWidget(HelpButton(msg), 2, 4) + grid.addWidget(QLabel(_('Blockchain') + ':'), 3, 0) + grid.addWidget(self.height_label, 3, 1) + grid.addWidget(HelpButton(msg), 3, 4) self.split_label = QLabel('') - grid.addWidget(self.split_label, 3, 0, 1, 3) + grid.addWidget(self.split_label, 4, 0, 1, 3) self.nodes_list_widget = NodesListWidget(self) - grid.addWidget(self.nodes_list_widget, 5, 0, 1, 5) + grid.addWidget(self.nodes_list_widget, 6, 0, 1, 5) vbox = QVBoxLayout() vbox.addWidget(tabs) @@ -354,27 +318,18 @@ def enable_set_server(self): if self.config.is_modifiable('server'): enabled = not self.autoconnect_cb.isChecked() self.server_e.setEnabled(enabled) - self.servers_list.setEnabled(enabled) else: - for w in [self.autoconnect_cb, self.server_e, self.servers_list]: + for w in [self.autoconnect_cb, self.server_e, self.nodes_list_widget]: w.setEnabled(False) def update(self): net_params = self.network.get_parameters() server = net_params.server - proxy_config, auto_connect = net_params.proxy, net_params.auto_connect + auto_connect = net_params.auto_connect if not self.server_e.hasFocus(): self.server_e.setText(server.net_addr_str()) self.autoconnect_cb.setChecked(auto_connect) - interface = self.network.interface - host = interface.host if interface else _('None') - self.server_label.setText(host) - - self.servers = self.network.get_servers() - self.servers_list.update(self.servers, self.tor_cb.isChecked()) - self.enable_set_server() - height_str = "%d "%(self.network.get_local_height()) + _('blocks') self.height_label.setText(height_str) n = len(self.network.get_interfaces()) @@ -391,7 +346,10 @@ def update(self): else: msg = '' self.split_label.setText(msg) - self.nodes_list_widget.update(self.network) + self.nodes_list_widget.update(network=self.network, + servers=self.network.get_servers(), + use_tor=self.tor_cb.isChecked()) + self.enable_set_server() def fill_in_proxy_settings(self): proxy_config = self.network.get_parameters().proxy From b59c3294b29c8899480a69b5c79bc37dd034b11e Mon Sep 17 00:00:00 2001 From: SomberNight Date: Sun, 26 Apr 2020 05:29:32 +0200 Subject: [PATCH 115/117] fix #6115: qt wallet>information was broken for imported wallets --- electrum/gui/qt/main_window.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/electrum/gui/qt/main_window.py b/electrum/gui/qt/main_window.py index 60472e2a40..f72196ecd0 100644 --- a/electrum/gui/qt/main_window.py +++ b/electrum/gui/qt/main_window.py @@ -2277,6 +2277,8 @@ def show_wallet_info(self): grid.addWidget(lightning_b, 5, 2) vbox.addLayout(grid) + labels_clayout = None + if self.wallet.is_deterministic(): mpk_text = ShowQRTextEdit() mpk_text.setMaximumHeight(150) @@ -2286,8 +2288,6 @@ def show_mpk(index): mpk_text.setText(mpk_list[index]) mpk_text.repaint() # macOS hack for #4777 - # declare this value such that the hooks can later figure out what to do - labels_clayout = None # only show the combobox in case multiple accounts are available if len(mpk_list) > 1: # only show the combobox if multiple master keys are defined From 100a216165baada3079dfe2d5f6501b39bb747a6 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Sun, 26 Apr 2020 05:49:34 +0200 Subject: [PATCH 116/117] qt wallet>info: add text if lightning is not available for wallet --- electrum/gui/qt/main_window.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/electrum/gui/qt/main_window.py b/electrum/gui/qt/main_window.py index f72196ecd0..fa5a529e93 100644 --- a/electrum/gui/qt/main_window.py +++ b/electrum/gui/qt/main_window.py @@ -2260,6 +2260,7 @@ def show_wallet_info(self): ks_type = str(keystore_types[0]) if keystore_types else _('No keystore') grid.addWidget(QLabel(ks_type), 4, 1) # lightning + grid.addWidget(QLabel(_('Lightning') + ':'), 5, 0) if self.wallet.can_have_lightning(): if self.wallet.has_lightning(): lightning_b = QPushButton(_('Disable')) @@ -2272,9 +2273,11 @@ def show_wallet_info(self): lightning_b.clicked.connect(dialog.close) lightning_b.clicked.connect(self.enable_lightning) lightning_label = QLabel(_('Disabled')) - grid.addWidget(QLabel(_('Lightning')), 5, 0) grid.addWidget(lightning_label, 5, 1) grid.addWidget(lightning_b, 5, 2) + else: + grid.addWidget(QLabel(_("Not available for this wallet.")), 5, 1) + grid.addWidget(HelpButton(_("Lightning is currently restricted to HD wallets with p2wpkh addresses.")), 5, 2) vbox.addLayout(grid) labels_clayout = None From 527e0b9b897e07ae671ca180ab8e5f89423b8aec Mon Sep 17 00:00:00 2001 From: SomberNight Date: Sun, 26 Apr 2020 05:51:02 +0200 Subject: [PATCH 117/117] qt main window: only show "Channels" tab if wallet has lightning --- electrum/gui/qt/main_window.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/electrum/gui/qt/main_window.py b/electrum/gui/qt/main_window.py index fa5a529e93..0e85be312f 100644 --- a/electrum/gui/qt/main_window.py +++ b/electrum/gui/qt/main_window.py @@ -224,7 +224,8 @@ def add_optional_tab(tabs, tab, icon, description, name): tabs.addTab(tab, icon, description.replace("&", "")) add_optional_tab(tabs, self.addresses_tab, read_QIcon("tab_addresses.png"), _("&Addresses"), "addresses") - add_optional_tab(tabs, self.channels_tab, read_QIcon("lightning.png"), _("Channels"), "channels") + if self.wallet.has_lightning(): + add_optional_tab(tabs, self.channels_tab, read_QIcon("lightning.png"), _("Channels"), "channels") add_optional_tab(tabs, self.utxo_tab, read_QIcon("tab_coins.png"), _("Co&ins"), "utxo") add_optional_tab(tabs, self.contacts_tab, read_QIcon("tab_contacts.png"), _("Con&tacts"), "contacts") add_optional_tab(tabs, self.console_tab, read_QIcon("tab_console.png"), _("Con&sole"), "console")