From 7488cc91cdbece4f93b0047e36cfba65ae8b01a2 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Thu, 26 Mar 2020 01:20:41 +0100 Subject: [PATCH 01/69] qt channels: expose long channel id (in ctx menu and details dlg) Also add separators to context menu to more visible separate close/delete actions from rest. --- electrum/gui/qt/channel_details.py | 2 +- electrum/gui/qt/channels_list.py | 8 +++++++- electrum/gui/qt/util.py | 6 ++++-- electrum/lnworker.py | 2 +- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/electrum/gui/qt/channel_details.py b/electrum/gui/qt/channel_details.py index d3d8151b0..fa75d43d3 100644 --- a/electrum/gui/qt/channel_details.py +++ b/electrum/gui/qt/channel_details.py @@ -153,7 +153,7 @@ def __init__(self, window: 'ElectrumWindow', chan_id: bytes): form_layout = QtWidgets.QFormLayout(None) # add form content - form_layout.addRow(_('Channel ID:'), SelectableLabel(chan.get_id_for_log())) + form_layout.addRow(_('Channel ID:'), SelectableLabel(f"{chan.channel_id.hex()} (Short: {chan.short_channel_id})")) form_layout.addRow(_('State:'), SelectableLabel(chan.get_state_for_GUI())) self.initiator = 'Local' if chan.constraints.is_initiator else 'Remote' form_layout.addRow(_('Initiator:'), SelectableLabel(self.initiator)) diff --git a/electrum/gui/qt/channels_list.py b/electrum/gui/qt/channels_list.py index 4eafd4a8c..73342d480 100644 --- a/electrum/gui/qt/channels_list.py +++ b/electrum/gui/qt/channels_list.py @@ -120,6 +120,7 @@ def remove_channel(self, channel_id): def create_menu(self, position): menu = QMenu() + menu.setSeparatorsCollapsible(True) # consecutive separators are merged together idx = self.selectionModel().currentIndex() item = self.model().itemFromIndex(idx) if not item: @@ -127,14 +128,18 @@ def create_menu(self, position): channel_id = idx.sibling(idx.row(), self.Columns.NODE_ID).data(ROLE_CHANNEL_ID) chan = self.lnworker.channels[channel_id] menu.addAction(_("Details..."), lambda: self.parent.show_channel(channel_id)) - self.add_copy_menu(menu, idx) + cc = self.add_copy_menu(menu, idx) + cc.addAction(_("Long Channel ID"), lambda: self.place_text_on_clipboard(channel_id.hex(), + title=_("Long Channel ID"))) funding_tx = self.parent.wallet.db.get_transaction(chan.funding_outpoint.txid) if funding_tx: menu.addAction(_("View funding transaction"), lambda: self.parent.show_transaction(funding_tx)) if not chan.is_closed(): + menu.addSeparator() if chan.peer_state == peer_states.GOOD: menu.addAction(_("Close channel"), lambda: self.close_channel(channel_id)) menu.addAction(_("Force-close channel"), lambda: self.force_close(channel_id)) + menu.addSeparator() else: item = chan.get_closing_height() if item: @@ -143,6 +148,7 @@ def create_menu(self, position): if closing_tx: menu.addAction(_("View closing transaction"), lambda: self.parent.show_transaction(closing_tx)) if chan.is_redeemed(): + menu.addSeparator() menu.addAction(_("Delete"), lambda: self.remove_channel(channel_id)) menu.exec_(self.viewport().mapToGlobal(position)) diff --git a/electrum/gui/qt/util.py b/electrum/gui/qt/util.py index 184d509dc..2229c0bca 100644 --- a/electrum/gui/qt/util.py +++ b/electrum/gui/qt/util.py @@ -20,7 +20,8 @@ QAbstractItemView, QVBoxLayout, QLineEdit, QStyle, QDialog, QGroupBox, QButtonGroup, QRadioButton, QFileDialog, QWidget, QToolButton, QTreeView, QPlainTextEdit, - QHeaderView, QApplication, QToolTip, QTreeWidget, QStyledItemDelegate) + QHeaderView, QApplication, QToolTip, QTreeWidget, QStyledItemDelegate, + QMenu) from electrum.i18n import _, languages from electrum.util import FileImportFailed, FileExportFailed, make_aiohttp_session, resource_path @@ -658,7 +659,7 @@ def show_toolbar(self, state, config=None): def toggle_toolbar(self, config=None): self.show_toolbar(not self.toolbar_shown, config) - def add_copy_menu(self, menu, idx): + def add_copy_menu(self, menu: QMenu, idx) -> QMenu: cc = menu.addMenu(_("Copy")) for column in self.Columns: column_title = self.model().horizontalHeaderItem(column).text() @@ -669,6 +670,7 @@ def add_copy_menu(self, menu, idx): cc.addAction(column_title, lambda text=clipboard_data, title=column_title: self.place_text_on_clipboard(text, title=title)) + return cc def place_text_on_clipboard(self, text: str, *, title: str = None) -> None: self.parent.do_copy(text, title=title) diff --git a/electrum/lnworker.py b/electrum/lnworker.py index 2d6dbe37f..5264606c4 100644 --- a/electrum/lnworker.py +++ b/electrum/lnworker.py @@ -203,7 +203,7 @@ async def _maintain_connectivity(self): if last_tried + PEER_RETRY_INTERVAL < now: await self._add_peer(peer.host, peer.port, peer.pubkey) - async def _add_peer(self, host, port, node_id): + async def _add_peer(self, host, port, node_id) -> Peer: if node_id in self.peers: return self.peers[node_id] port = int(port) From 95979ba58d93ae83ec519f7f8f8468105e3f3e9e Mon Sep 17 00:00:00 2001 From: SomberNight Date: Thu, 26 Mar 2020 02:54:21 +0100 Subject: [PATCH 02/69] qt channels list: make selection more in line with other tabs (allow selecting none, and allow multi-select) --- electrum/gui/qt/channels_list.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/electrum/gui/qt/channels_list.py b/electrum/gui/qt/channels_list.py index 73342d480..883015af9 100644 --- a/electrum/gui/qt/channels_list.py +++ b/electrum/gui/qt/channels_list.py @@ -4,7 +4,8 @@ from PyQt5 import QtCore, QtGui from PyQt5.QtCore import Qt -from PyQt5.QtWidgets import QMenu, QHBoxLayout, QLabel, QVBoxLayout, QGridLayout, QLineEdit, QPushButton +from PyQt5.QtWidgets import (QMenu, QHBoxLayout, QLabel, QVBoxLayout, QGridLayout, QLineEdit, + QPushButton, QAbstractItemView) from PyQt5.QtGui import QFont from electrum.util import bh2u, NotEnoughFunds, NoDynamicFeeEstimates @@ -46,6 +47,7 @@ def __init__(self, parent): super().__init__(parent, self.create_menu, stretch_column=self.Columns.NODE_ID, editable_columns=[]) self.setModel(QtGui.QStandardItemModel(self)) + self.setSelectionMode(QAbstractItemView.ExtendedSelection) self.main_window = parent self.update_rows.connect(self.do_update_rows) self.update_single_row.connect(self.do_update_single_row) @@ -121,7 +123,15 @@ def remove_channel(self, channel_id): def create_menu(self, position): menu = QMenu() menu.setSeparatorsCollapsible(True) # consecutive separators are merged together - idx = self.selectionModel().currentIndex() + selected = self.selected_in_column(self.Columns.NODE_ID) + if not selected: + return + multi_select = len(selected) > 1 + if multi_select: + return + idx = self.indexAt(position) + if not idx.isValid(): + return item = self.model().itemFromIndex(idx) if not item: return @@ -153,7 +163,7 @@ def create_menu(self, position): menu.exec_(self.viewport().mapToGlobal(position)) @QtCore.pyqtSlot(Channel) - def do_update_single_row(self, chan): + def do_update_single_row(self, chan: Channel): lnworker = self.parent.wallet.lnworker if not lnworker: return From 9c8d2be6389e8265d08e4c58fe5cc20b3d630911 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Thu, 26 Mar 2020 02:54:50 +0100 Subject: [PATCH 03/69] qt channels list: sort by short chan id by default --- electrum/gui/qt/channels_list.py | 1 + 1 file changed, 1 insertion(+) diff --git a/electrum/gui/qt/channels_list.py b/electrum/gui/qt/channels_list.py index 883015af9..d342011cb 100644 --- a/electrum/gui/qt/channels_list.py +++ b/electrum/gui/qt/channels_list.py @@ -192,6 +192,7 @@ def do_update_rows(self, wallet): items[self.Columns.LOCAL_BALANCE].setFont(QFont(MONOSPACE_FONT)) items[self.Columns.REMOTE_BALANCE].setFont(QFont(MONOSPACE_FONT)) self.model().insertRow(0, items) + self.sortByColumn(self.Columns.SHORT_CHANID, Qt.DescendingOrder) def update_can_send(self, lnworker): msg = _('Can send') + ' ' + self.parent.format_amount(lnworker.can_send())\ From deb50e7ec310e60133a141a7ca4eb71456de61b5 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Thu, 26 Mar 2020 03:32:44 +0100 Subject: [PATCH 04/69] lnchannel: implement "freezing" channels (for sending) and expose it in Qt GUI --- electrum/gui/qt/channels_list.py | 40 ++++++++++++++++++++++++++++---- electrum/lnchannel.py | 31 +++++++++++++++++++++---- 2 files changed, 61 insertions(+), 10 deletions(-) diff --git a/electrum/gui/qt/channels_list.py b/electrum/gui/qt/channels_list.py index d342011cb..12f519d16 100644 --- a/electrum/gui/qt/channels_list.py +++ b/electrum/gui/qt/channels_list.py @@ -1,12 +1,13 @@ # -*- coding: utf-8 -*- import traceback from enum import IntEnum +from typing import Sequence, Optional from PyQt5 import QtCore, QtGui from PyQt5.QtCore import Qt from PyQt5.QtWidgets import (QMenu, QHBoxLayout, QLabel, QVBoxLayout, QGridLayout, QLineEdit, QPushButton, QAbstractItemView) -from PyQt5.QtGui import QFont +from PyQt5.QtGui import QFont, QStandardItem, QBrush from electrum.util import bh2u, NotEnoughFunds, NoDynamicFeeEstimates from electrum.i18n import _ @@ -15,7 +16,7 @@ from electrum.lnutil import LOCAL, REMOTE, format_short_channel_id, LN_MAX_FUNDING_SAT from .util import (MyTreeView, WindowModalDialog, Buttons, OkButton, CancelButton, - EnterButton, WaitingDialog, MONOSPACE_FONT) + EnterButton, WaitingDialog, MONOSPACE_FONT, ColorScheme) from .amountedit import BTCAmountEdit, FreezableLineEdit @@ -43,6 +44,8 @@ class Columns(IntEnum): Columns.CHANNEL_STATUS: _('Status'), } + _default_item_bg_brush = None # type: Optional[QBrush] + def __init__(self, parent): super().__init__(parent, self.create_menu, stretch_column=self.Columns.NODE_ID, editable_columns=[]) @@ -141,6 +144,12 @@ def create_menu(self, position): cc = self.add_copy_menu(menu, idx) cc.addAction(_("Long Channel ID"), lambda: self.place_text_on_clipboard(channel_id.hex(), title=_("Long Channel ID"))) + + if not chan.is_frozen(): + menu.addAction(_("Freeze"), lambda: chan.set_frozen(True)) + else: + menu.addAction(_("Unfreeze"), lambda: chan.set_frozen(False)) + funding_tx = self.parent.wallet.db.get_transaction(chan.funding_outpoint.txid) if funding_tx: menu.addAction(_("View funding transaction"), lambda: self.parent.show_transaction(funding_tx)) @@ -169,9 +178,12 @@ def do_update_single_row(self, chan: Channel): return for row in range(self.model().rowCount()): item = self.model().item(row, self.Columns.NODE_ID) - if item.data(ROLE_CHANNEL_ID) == chan.channel_id: - for column, v in enumerate(self.format_fields(chan)): - self.model().item(row, column).setData(v, QtCore.Qt.DisplayRole) + if item.data(ROLE_CHANNEL_ID) != chan.channel_id: + continue + for column, v in enumerate(self.format_fields(chan)): + self.model().item(row, column).setData(v, QtCore.Qt.DisplayRole) + items = [self.model().item(row, column) for column in self.Columns] + self._update_chan_frozen_bg(chan=chan, items=items) self.update_can_send(lnworker) @QtCore.pyqtSlot(Abstract_Wallet) @@ -187,13 +199,31 @@ def do_update_rows(self, wallet): for chan in lnworker.channels.values(): items = [QtGui.QStandardItem(x) for x in self.format_fields(chan)] self.set_editability(items) + if self._default_item_bg_brush is None: + self._default_item_bg_brush = items[self.Columns.NODE_ID].background() items[self.Columns.NODE_ID].setData(chan.channel_id, ROLE_CHANNEL_ID) items[self.Columns.NODE_ID].setFont(QFont(MONOSPACE_FONT)) items[self.Columns.LOCAL_BALANCE].setFont(QFont(MONOSPACE_FONT)) items[self.Columns.REMOTE_BALANCE].setFont(QFont(MONOSPACE_FONT)) + self._update_chan_frozen_bg(chan=chan, items=items) self.model().insertRow(0, items) self.sortByColumn(self.Columns.SHORT_CHANID, Qt.DescendingOrder) + def _update_chan_frozen_bg(self, *, chan: Channel, items: Sequence[QStandardItem]): + assert self._default_item_bg_brush is not None + for col in [ + self.Columns.LOCAL_BALANCE, + self.Columns.REMOTE_BALANCE, + self.Columns.CHANNEL_STATUS, + ]: + item = items[col] + if chan.is_frozen(): + item.setBackground(ColorScheme.BLUE.as_color(True)) + item.setToolTip(_("This channel is frozen. Frozen channels will not be used for outgoing payments.")) + else: + item.setBackground(self._default_item_bg_brush) + item.setToolTip("") + def update_can_send(self, lnworker): msg = _('Can send') + ' ' + self.parent.format_amount(lnworker.can_send())\ + ' ' + self.parent.base_unit() + '; '\ diff --git a/electrum/lnchannel.py b/electrum/lnchannel.py index 432187d72..9e8ffeeba 100644 --- a/electrum/lnchannel.py +++ b/electrum/lnchannel.py @@ -390,7 +390,22 @@ def delete_closing_height(self): def is_redeemed(self): return self.get_state() == channel_states.REDEEMED - def _check_can_pay(self, amount_msat: int) -> None: + def is_frozen(self) -> bool: + """Whether the user has marked this channel as frozen. + Frozen channels are not supposed to be used for new outgoing payments. + (note that payment-forwarding ignores this option) + """ + return self.storage.get('frozen_for_sending', False) + + def set_frozen(self, b: bool) -> None: + self.storage['frozen_for_sending'] = bool(b) + if self.lnworker: + self.lnworker.network.trigger_callback('channel', self) + + def _assert_we_can_add_htlc(self, amount_msat: int) -> None: + """Raises PaymentFailure if the local party cannot add this new HTLC. + (this is relevant both for payments initiated by us and when forwarding) + """ # TODO check if this method uses correct ctns (should use "latest" + 1) if self.is_closed(): raise PaymentFailure('Channel closed') @@ -398,6 +413,8 @@ def _check_can_pay(self, amount_msat: int) -> None: raise PaymentFailure('Channel not open', self.get_state()) if not self.can_send_ctx_updates(): raise PaymentFailure('Channel cannot send ctx updates') + if not self.can_send_update_add_htlc(): + raise PaymentFailure('Channel cannot add htlc') if self.available_to_spend(LOCAL) < amount_msat: raise PaymentFailure(f'Not enough local balance. Have: {self.available_to_spend(LOCAL)}, Need: {amount_msat}') if len(self.hm.htlcs(LOCAL)) + 1 > self.config[REMOTE].max_accepted_htlcs: @@ -409,9 +426,14 @@ def _check_can_pay(self, amount_msat: int) -> None: if amount_msat < self.config[REMOTE].htlc_minimum_msat: raise PaymentFailure(f'HTLC value too small: {amount_msat} msat') - def can_pay(self, amount_msat): + def can_pay(self, amount_msat: int) -> bool: + """Returns whether we can initiate a new payment of given value. + (we are the payer, not just a forwarding node) + """ + if self.is_frozen(): + return False try: - self._check_can_pay(amount_msat) + self._assert_we_can_add_htlc(amount_msat) except PaymentFailure: return False return True @@ -430,11 +452,10 @@ def add_htlc(self, htlc: UpdateAddHtlc) -> UpdateAddHtlc: This docstring is from LND. """ - assert self.can_send_ctx_updates(), f"cannot update channel. {self.get_state()!r} {self.peer_state!r}" if isinstance(htlc, dict): # legacy conversion # FIXME remove htlc = UpdateAddHtlc(**htlc) assert isinstance(htlc, UpdateAddHtlc) - self._check_can_pay(htlc.amount_msat) + self._assert_we_can_add_htlc(htlc.amount_msat) if htlc.htlc_id is None: htlc = attr.evolve(htlc, htlc_id=self.hm.get_next_htlc_id(LOCAL)) with self.db_lock: From 777e350fae40afadb367914b7127a6ebe3f0241b Mon Sep 17 00:00:00 2001 From: SomberNight Date: Thu, 26 Mar 2020 05:43:26 +0100 Subject: [PATCH 05/69] lnchannel: partly fix available_to_spend we were looking at inconsistent ctns and we were looking at the wrong subject's ctx all the FIXMEs and TODOs here will still warrant some attention. (note that test_DesyncHTLCs was passing incorrectly: the "assertRaises" was catching a different exception) --- electrum/commands.py | 4 +-- electrum/lnchannel.py | 51 ++++++++++++++++++++------------ electrum/lnutil.py | 10 +++++-- electrum/tests/test_lnchannel.py | 11 ++++++- electrum/tests/test_lnutil.py | 8 ++--- 5 files changed, 56 insertions(+), 28 deletions(-) diff --git a/electrum/commands.py b/electrum/commands.py index 5a0caa075..2cf1ab00d 100644 --- a/electrum/commands.py +++ b/electrum/commands.py @@ -1004,8 +1004,8 @@ async def list_channels(self, wallet: Abstract_Wallet = None): 'remote_balance': chan.balance(REMOTE)//1000, 'local_reserve': chan.config[LOCAL].reserve_sat, 'remote_reserve': chan.config[REMOTE].reserve_sat, - 'local_unsettled_sent': chan.unsettled_sent_balance(LOCAL)//1000, - 'remote_unsettled_sent': chan.unsettled_sent_balance(REMOTE)//1000, + 'local_unsettled_sent': chan.balance_tied_up_in_htlcs_by_direction(LOCAL, direction=SENT) // 1000, + 'remote_unsettled_sent': chan.balance_tied_up_in_htlcs_by_direction(REMOTE, direction=SENT) // 1000, } for channel_id, chan in l ] diff --git a/electrum/lnchannel.py b/electrum/lnchannel.py index 9e8ffeeba..1f2744bc2 100644 --- a/electrum/lnchannel.py +++ b/electrum/lnchannel.py @@ -664,20 +664,28 @@ def balance(self, whose: HTLCOwner, *, ctx_owner=HTLCOwner.LOCAL, ctn: int = Non ctn=ctn, initial_balance_msat=initial) - def balance_minus_outgoing_htlcs(self, whose: HTLCOwner, *, ctx_owner: HTLCOwner = HTLCOwner.LOCAL): + def balance_minus_outgoing_htlcs(self, whose: HTLCOwner, *, ctx_owner: HTLCOwner = HTLCOwner.LOCAL, + ctn: int = None): """ This balance in mSAT, which includes the value of pending outgoing HTLCs, is used in the UI. """ assert type(whose) is HTLCOwner - ctn = self.get_next_ctn(ctx_owner) - return self.balance(whose, ctx_owner=ctx_owner, ctn=ctn) - self.unsettled_sent_balance(ctx_owner) - - def unsettled_sent_balance(self, subject: HTLCOwner = LOCAL): - ctn = self.get_next_ctn(subject) - return htlcsum(self.hm.htlcs_by_direction(subject, SENT, ctn).values()) + if ctn is None: + ctn = self.get_next_ctn(ctx_owner) + committed_balance = self.balance(whose, ctx_owner=ctx_owner, ctn=ctn) + direction = RECEIVED if whose != ctx_owner else SENT + balance_in_htlcs = self.balance_tied_up_in_htlcs_by_direction(ctx_owner, ctn=ctn, direction=direction) + return committed_balance - balance_in_htlcs + + def balance_tied_up_in_htlcs_by_direction(self, ctx_owner: HTLCOwner = LOCAL, *, ctn: int = None, + direction: Direction): + # in msat + if ctn is None: + ctn = self.get_next_ctn(ctx_owner) + return htlcsum(self.hm.htlcs_by_direction(ctx_owner, direction, ctn).values()) - def available_to_spend(self, subject): + def available_to_spend(self, subject: HTLCOwner) -> int: """ This balance in mSAT, while technically correct, can not be used in the UI cause it fluctuates (commit fee) @@ -685,14 +693,17 @@ def available_to_spend(self, subject): # FIXME whose balance? whose ctx? # FIXME confusing/mixing ctns (should probably use latest_ctn + 1; not oldest_unrevoked + 1) assert type(subject) is HTLCOwner - return self.balance_minus_outgoing_htlcs(subject, ctx_owner=subject)\ - - self.config[-subject].reserve_sat * 1000\ - - calc_onchain_fees( - # TODO should we include a potential new htlc, when we are called from receive_htlc? - len(self.included_htlcs(subject, SENT) + self.included_htlcs(subject, RECEIVED)), - self.get_latest_feerate(subject), - self.constraints.is_initiator, - )[subject] + ctx_owner = subject.inverted() + ctn = self.get_next_ctn(ctx_owner) + balance = self.balance_minus_outgoing_htlcs(whose=subject, ctx_owner=ctx_owner, ctn=ctn) + reserve = self.config[-subject].reserve_sat * 1000 + # TODO should we include a potential new htlc, when we are called from receive_htlc? + fees = calc_onchain_fees( + num_htlcs=len(self.included_htlcs(ctx_owner, SENT, ctn=ctn) + self.included_htlcs(ctx_owner, RECEIVED, ctn=ctn)), + feerate=self.get_feerate(ctx_owner, ctn=ctn), + is_local_initiator=self.constraints.is_initiator, + )[subject] + return balance - reserve - fees def included_htlcs(self, subject, direction, ctn=None): """ @@ -877,10 +888,12 @@ def make_commitment(self, subject, this_point, ctn) -> PartialTransaction: local_htlc_pubkey=this_htlc_pubkey, payment_hash=htlc.payment_hash, cltv_expiry=htlc.cltv_expiry), htlc)) + # note: maybe flip initiator here for fee purposes, we want LOCAL and REMOTE + # in the resulting dict to correspond to the to_local and to_remote *outputs* of the ctx onchain_fees = calc_onchain_fees( - len(htlcs), - feerate, - self.constraints.is_initiator == (subject == LOCAL), + num_htlcs=len(htlcs), + feerate=feerate, + is_local_initiator=self.constraints.is_initiator == (subject == LOCAL), ) if self.is_static_remotekey_enabled(): payment_pubkey = other_config.payment_basepoint.pubkey diff --git a/electrum/lnutil.py b/electrum/lnutil.py index 85e719954..e1f0c5c16 100644 --- a/electrum/lnutil.py +++ b/electrum/lnutil.py @@ -556,11 +556,17 @@ def make_commitment_outputs(*, fees_per_participant: Mapping[HTLCOwner, int], lo c_outputs_filtered = list(filter(lambda x: x.value >= dust_limit_sat, non_htlc_outputs + htlc_outputs)) return htlc_outputs, c_outputs_filtered -def calc_onchain_fees(num_htlcs, feerate, we_pay_fee): + +def calc_onchain_fees(*, num_htlcs: int, feerate: int, is_local_initiator: bool) -> Dict['HTLCOwner', int]: + # feerate is in sat/kw + # returns fees in msats overall_weight = 500 + 172 * num_htlcs + 224 fee = feerate * overall_weight fee = fee // 1000 * 1000 - return {LOCAL: fee if we_pay_fee else 0, REMOTE: fee if not we_pay_fee else 0} + return { + LOCAL: fee if is_local_initiator else 0, + REMOTE: fee if not is_local_initiator else 0, + } def make_commitment(ctn, local_funding_pubkey, remote_funding_pubkey, remote_payment_pubkey, funder_payment_basepoint, diff --git a/electrum/tests/test_lnchannel.py b/electrum/tests/test_lnchannel.py index ffe7e6ef1..ce824792b 100644 --- a/electrum/tests/test_lnchannel.py +++ b/electrum/tests/test_lnchannel.py @@ -605,21 +605,28 @@ def test_AddHTLCNegativeBalance(self): class TestAvailableToSpend(ElectrumTestCase): def test_DesyncHTLCs(self): alice_channel, bob_channel = create_test_channels() + self.assertEqual(499995656000, alice_channel.available_to_spend(LOCAL)) + self.assertEqual(500000000000, bob_channel.available_to_spend(LOCAL)) paymentPreimage = b"\x01" * 32 paymentHash = bitcoin.sha256(paymentPreimage) htlc_dict = { 'payment_hash' : paymentHash, - 'amount_msat' : int(4.1 * one_bitcoin_in_msat), + 'amount_msat' : one_bitcoin_in_msat * 41 // 10, 'cltv_expiry' : 5, 'timestamp' : 0, } alice_idx = alice_channel.add_htlc(htlc_dict).htlc_id bob_idx = bob_channel.receive_htlc(htlc_dict).htlc_id + self.assertEqual(89994624000, alice_channel.available_to_spend(LOCAL)) + self.assertEqual(500000000000, bob_channel.available_to_spend(LOCAL)) + force_state_transition(alice_channel, bob_channel) bob_channel.fail_htlc(bob_idx) alice_channel.receive_fail_htlc(alice_idx, error_bytes=None) + self.assertEqual(89994624000, alice_channel.available_to_spend(LOCAL)) + self.assertEqual(500000000000, bob_channel.available_to_spend(LOCAL)) # Alice now has gotten all her original balance (5 BTC) back, however, # adding a new HTLC at this point SHOULD fail, since if she adds the # HTLC and signs the next state, Bob cannot assume she received the @@ -638,6 +645,8 @@ def test_DesyncHTLCs(self): # Now do a state transition, which will ACK the FailHTLC, making Alice # able to add the new HTLC. force_state_transition(alice_channel, bob_channel) + self.assertEqual(499995656000, alice_channel.available_to_spend(LOCAL)) + self.assertEqual(500000000000, bob_channel.available_to_spend(LOCAL)) alice_channel.add_htlc(htlc_dict) class TestChanReserve(ElectrumTestCase): diff --git a/electrum/tests/test_lnutil.py b/electrum/tests/test_lnutil.py index c115d1817..c0c01cd06 100644 --- a/electrum/tests/test_lnutil.py +++ b/electrum/tests/test_lnutil.py @@ -516,7 +516,7 @@ def test_commitment_tx_with_all_five_HTLCs_untrimmed_minimum_feerate(self): local_revocation_pubkey, local_delayedpubkey, local_delay, funding_tx_id, funding_output_index, funding_amount_satoshi, to_local_msat, to_remote_msat, local_dust_limit_satoshi, - calc_onchain_fees(len(htlcs), local_feerate_per_kw, True), htlcs=htlcs) + calc_onchain_fees(num_htlcs=len(htlcs), feerate=local_feerate_per_kw, is_local_initiator=True), htlcs=htlcs) self.sign_and_insert_remote_sig(our_commit_tx, remote_funding_pubkey, remote_signature, local_funding_pubkey, local_funding_privkey) self.assertEqual(str(our_commit_tx), output_commit_tx) @@ -593,7 +593,7 @@ def test_commitment_tx_with_one_output(self): local_revocation_pubkey, local_delayedpubkey, local_delay, funding_tx_id, funding_output_index, funding_amount_satoshi, to_local_msat, to_remote_msat, local_dust_limit_satoshi, - calc_onchain_fees(0, local_feerate_per_kw, True), htlcs=[]) + calc_onchain_fees(num_htlcs=0, feerate=local_feerate_per_kw, is_local_initiator=True), htlcs=[]) self.sign_and_insert_remote_sig(our_commit_tx, remote_funding_pubkey, remote_signature, local_funding_pubkey, local_funding_privkey) self.assertEqual(str(our_commit_tx), output_commit_tx) @@ -612,7 +612,7 @@ def test_commitment_tx_with_fee_greater_than_funder_amount(self): local_revocation_pubkey, local_delayedpubkey, local_delay, funding_tx_id, funding_output_index, funding_amount_satoshi, to_local_msat, to_remote_msat, local_dust_limit_satoshi, - calc_onchain_fees(0, local_feerate_per_kw, True), htlcs=[]) + calc_onchain_fees(num_htlcs=0, feerate=local_feerate_per_kw, is_local_initiator=True), htlcs=[]) self.sign_and_insert_remote_sig(our_commit_tx, remote_funding_pubkey, remote_signature, local_funding_pubkey, local_funding_privkey) self.assertEqual(str(our_commit_tx), output_commit_tx) @@ -670,7 +670,7 @@ def test_simple_commitment_tx_with_no_HTLCs(self): local_revocation_pubkey, local_delayedpubkey, local_delay, funding_tx_id, funding_output_index, funding_amount_satoshi, to_local_msat, to_remote_msat, local_dust_limit_satoshi, - calc_onchain_fees(0, local_feerate_per_kw, True), htlcs=[]) + calc_onchain_fees(num_htlcs=0, feerate=local_feerate_per_kw, is_local_initiator=True), htlcs=[]) self.sign_and_insert_remote_sig(our_commit_tx, remote_funding_pubkey, remote_signature, local_funding_pubkey, local_funding_privkey) ref_commit_tx_str = '02000000000101bef67e4e2fb9ddeeb3461973cd4c62abb35050b1add772995b820b584a488489000000000038b02b8002c0c62d0000000000160014ccf1af2f2aabee14bb40fa3851ab2301de84311054a56a00000000002200204adb4e2f00643db396dd120d4e7dc17625f5f2c11a40d857accc862d6b7dd80e0400473044022051b75c73198c6deee1a875871c3961832909acd297c6b908d59e3319e5185a46022055c419379c5051a78d00dbbce11b5b664a0c22815fbcc6fcef6b1937c383693901483045022100f51d2e566a70ba740fc5d8c0f07b9b93d2ed741c3c0860c613173de7d39e7968022041376d520e9c0e1ad52248ddf4b22e12be8763007df977253ef45a4ca3bdb7c001475221023da092f6980e58d2c037173180e9a465476026ee50f96695963e8efe436f54eb21030e9f7b623d2ccc7c9bd44d66d5ce21ce504c0acf6385a132cec6d3c39fa711c152ae3e195220' self.assertEqual(str(our_commit_tx), ref_commit_tx_str) From 53c6fc8cf14a914e961bd2f137540684244b461c Mon Sep 17 00:00:00 2001 From: SomberNight Date: Thu, 26 Mar 2020 06:25:26 +0100 Subject: [PATCH 06/69] lnchannel: test for max htlc value (needs to be below protocol maximum) --- electrum/lnchannel.py | 18 ++++++++------ electrum/lnutil.py | 1 + electrum/tests/regtest/regtest.sh | 8 +++---- electrum/tests/test_lnchannel.py | 40 +++++++++++++++++++++++++++---- 4 files changed, 51 insertions(+), 16 deletions(-) diff --git a/electrum/lnchannel.py b/electrum/lnchannel.py index 1f2744bc2..ca735faeb 100644 --- a/electrum/lnchannel.py +++ b/electrum/lnchannel.py @@ -44,13 +44,14 @@ from .lnonion import decode_onion_error, OnionFailureCode, OnionRoutingFailureMessage from . import lnutil from .lnutil import (Outpoint, LocalConfig, RemoteConfig, Keypair, OnlyPubkeyKeypair, ChannelConstraints, - get_per_commitment_secret_from_seed, secret_to_pubkey, derive_privkey, make_closing_tx, - sign_and_get_sig_string, RevocationStore, derive_blinded_pubkey, Direction, derive_pubkey, - make_htlc_tx_with_open_channel, make_commitment, make_received_htlc, make_offered_htlc, - HTLC_TIMEOUT_WEIGHT, HTLC_SUCCESS_WEIGHT, extract_ctn_from_tx_and_chan, UpdateAddHtlc, - funding_output_script, SENT, RECEIVED, LOCAL, REMOTE, HTLCOwner, make_commitment_outputs, - ScriptHtlc, PaymentFailure, calc_onchain_fees, RemoteMisbehaving, make_htlc_output_witness_script, - ShortChannelID, map_htlcs_to_ctx_output_idxs, LNPeerAddr, BarePaymentAttemptLog) + get_per_commitment_secret_from_seed, secret_to_pubkey, derive_privkey, make_closing_tx, + sign_and_get_sig_string, RevocationStore, derive_blinded_pubkey, Direction, derive_pubkey, + make_htlc_tx_with_open_channel, make_commitment, make_received_htlc, make_offered_htlc, + HTLC_TIMEOUT_WEIGHT, HTLC_SUCCESS_WEIGHT, extract_ctn_from_tx_and_chan, UpdateAddHtlc, + funding_output_script, SENT, RECEIVED, LOCAL, REMOTE, HTLCOwner, make_commitment_outputs, + ScriptHtlc, PaymentFailure, calc_onchain_fees, RemoteMisbehaving, make_htlc_output_witness_script, + ShortChannelID, map_htlcs_to_ctx_output_idxs, LNPeerAddr, BarePaymentAttemptLog, + LN_MAX_HTLC_VALUE_MSAT) from .lnsweep import create_sweeptxs_for_our_ctx, create_sweeptxs_for_their_ctx from .lnsweep import create_sweeptx_for_their_revoked_htlc, SweepInfo from .lnhtlc import HTLCManager @@ -159,6 +160,7 @@ def __init__(self, state: 'StoredDict', *, sweep_address=None, name=None, lnwork self.revocation_store = RevocationStore(state["revocation_store"]) self._can_send_ctx_updates = True # type: bool self._receive_fail_reasons = {} # type: Dict[int, BarePaymentAttemptLog] + self._ignore_max_htlc_value = False # used in tests def get_id_for_log(self) -> str: scid = self.short_channel_id @@ -425,6 +427,8 @@ def _assert_we_can_add_htlc(self, amount_msat: int) -> None: raise PaymentFailure(f'HTLC value sum (sum of pending htlcs: {current_htlc_sum/1000} sat plus new htlc: {amount_msat/1000} sat) would exceed max allowed: {self.config[REMOTE].max_htlc_value_in_flight_msat/1000} sat') if amount_msat < self.config[REMOTE].htlc_minimum_msat: raise PaymentFailure(f'HTLC value too small: {amount_msat} msat') + if amount_msat > LN_MAX_HTLC_VALUE_MSAT and not self._ignore_max_htlc_value: + raise PaymentFailure(f"HTLC value over protocol maximum: {amount_msat} > {LN_MAX_HTLC_VALUE_MSAT} msat") def can_pay(self, amount_msat: int) -> bool: """Returns whether we can initiate a new payment of given value. diff --git a/electrum/lnutil.py b/electrum/lnutil.py index e1f0c5c16..22349a29f 100644 --- a/electrum/lnutil.py +++ b/electrum/lnutil.py @@ -34,6 +34,7 @@ HTLC_SUCCESS_WEIGHT = 703 LN_MAX_FUNDING_SAT = pow(2, 24) - 1 +LN_MAX_HTLC_VALUE_MSAT = pow(2, 32) - 1 # dummy address for fee estimation of funding tx def ln_dummy_address(): diff --git a/electrum/tests/regtest/regtest.sh b/electrum/tests/regtest/regtest.sh index 7bb204059..7c9d15305 100755 --- a/electrum/tests/regtest/regtest.sh +++ b/electrum/tests/regtest/regtest.sh @@ -171,7 +171,7 @@ if [[ $1 == "redeem_htlcs" ]]; then new_blocks 3 wait_until_channel_open alice # alice pays bob - invoice=$($bob add_lightning_request 0.05 -m "test") + invoice=$($bob add_lightning_request 0.04 -m "test") $alice lnpay $invoice --timeout=1 || true unsettled=$($alice list_channels | jq '.[] | .local_unsettled_sent') if [[ "$unsettled" == "0" ]]; then @@ -213,7 +213,7 @@ if [[ $1 == "breach_with_unspent_htlc" ]]; then new_blocks 3 wait_until_channel_open alice echo "alice pays bob" - invoice=$($bob add_lightning_request 0.05 -m "test") + invoice=$($bob add_lightning_request 0.04 -m "test") $alice lnpay $invoice --timeout=1 || true unsettled=$($alice list_channels | jq '.[] | .local_unsettled_sent') if [[ "$unsettled" == "0" ]]; then @@ -242,7 +242,7 @@ if [[ $1 == "breach_with_spent_htlc" ]]; then new_blocks 3 wait_until_channel_open alice echo "alice pays bob" - invoice=$($bob add_lightning_request 0.05 -m "test") + invoice=$($bob add_lightning_request 0.04 -m "test") $alice lnpay $invoice --timeout=1 || true ctx=$($alice get_channel_ctx $channel --iknowwhatimdoing) unsettled=$($alice list_channels | jq '.[] | .local_unsettled_sent') @@ -284,7 +284,7 @@ if [[ $1 == "breach_with_spent_htlc" ]]; then $bob daemon -d sleep 1 $bob load_wallet - wait_for_balance bob 0.049 + wait_for_balance bob 0.039 $bob getbalance fi diff --git a/electrum/tests/test_lnchannel.py b/electrum/tests/test_lnchannel.py index ce824792b..46b68a4ff 100644 --- a/electrum/tests/test_lnchannel.py +++ b/electrum/tests/test_lnchannel.py @@ -105,12 +105,12 @@ def bip32(sequence): assert type(k) is bytes return k -def create_test_channels(feerate=6000, local=None, remote=None): +def create_test_channels(*, feerate=6000, local_msat=None, remote_msat=None): funding_txid = binascii.hexlify(b"\x01"*32).decode("ascii") funding_index = 0 - funding_sat = ((local + remote) // 1000) if local is not None and remote is not None else (bitcoin.COIN * 10) - local_amount = local if local is not None else (funding_sat * 1000 // 2) - remote_amount = remote if remote is not None else (funding_sat * 1000 // 2) + funding_sat = ((local_msat + remote_msat) // 1000) if local_msat is not None and remote_msat is not None else (bitcoin.COIN * 10) + local_amount = local_msat if local_msat is not None else (funding_sat * 1000 // 2) + remote_amount = remote_msat if remote_msat is not None else (funding_sat * 1000 // 2) alice_raw = [ bip32("m/" + str(i)) for i in range(5) ] bob_raw = [ bip32("m/" + str(i)) for i in range(5,11) ] alice_privkeys = [lnutil.Keypair(lnutil.privkey_to_pubkey(x), x) for x in alice_raw] @@ -164,6 +164,10 @@ def create_test_channels(feerate=6000, local=None, remote=None): # TODO: sweep_address in lnchannel.py should use static_remotekey alice.sweep_address = bitcoin.pubkey_to_address('p2wpkh', alice.config[LOCAL].payment_basepoint.pubkey.hex()) bob.sweep_address = bitcoin.pubkey_to_address('p2wpkh', bob.config[LOCAL].payment_basepoint.pubkey.hex()) + + alice._ignore_max_htlc_value = True + bob._ignore_max_htlc_value = True + return alice, bob class TestFee(ElectrumTestCase): @@ -172,7 +176,9 @@ class TestFee(ElectrumTestCase): https://github.com/lightningnetwork/lightning-rfc/blob/e0c436bd7a3ed6a028e1cb472908224658a14eca/03-transactions.md#requirements-2 """ def test_fee(self): - alice_channel, bob_channel = create_test_channels(253, 10000000000, 5000000000) + alice_channel, bob_channel = create_test_channels(feerate=253, + local_msat=10000000000, + remote_msat=5000000000) self.assertIn(9999817, [x.value for x in alice_channel.get_latest_commitment(LOCAL).outputs()]) class TestChannel(ElectrumTestCase): @@ -649,6 +655,30 @@ def test_DesyncHTLCs(self): self.assertEqual(500000000000, bob_channel.available_to_spend(LOCAL)) alice_channel.add_htlc(htlc_dict) + def test_max_htlc_value(self): + alice_channel, bob_channel = create_test_channels() + paymentPreimage = b"\x01" * 32 + paymentHash = bitcoin.sha256(paymentPreimage) + htlc_dict = { + 'payment_hash' : paymentHash, + 'amount_msat' : one_bitcoin_in_msat * 41 // 10, + 'cltv_expiry' : 5, + 'timestamp' : 0, + } + + alice_channel._ignore_max_htlc_value = False + bob_channel._ignore_max_htlc_value = False + with self.assertRaises(lnutil.PaymentFailure): + alice_channel.add_htlc(htlc_dict) + with self.assertRaises(lnutil.PaymentFailure): + bob_channel.receive_htlc(htlc_dict) + + alice_channel._ignore_max_htlc_value = True + bob_channel._ignore_max_htlc_value = True + alice_channel.add_htlc(htlc_dict) + bob_channel.receive_htlc(htlc_dict) + + class TestChanReserve(ElectrumTestCase): def setUp(self): alice_channel, bob_channel = create_test_channels() From 01207316aa078264d79dcad0ff88b3fd1417b613 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Thu, 26 Mar 2020 06:32:12 +0100 Subject: [PATCH 07/69] storage upgrade: move "htlc_minimum_msat" to base channel config --- electrum/lnpeer.py | 5 +++-- electrum/lnutil.py | 2 +- electrum/tests/test_lnchannel.py | 1 + electrum/wallet_db.py | 11 ++++++++++- 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/electrum/lnpeer.py b/electrum/lnpeer.py index f7979a350..2c79099e2 100644 --- a/electrum/lnpeer.py +++ b/electrum/lnpeer.py @@ -504,11 +504,12 @@ def make_local_config(self, funding_sat: int, push_msat: int, initiator: HTLCOwn was_announced=False, current_commitment_signature=None, current_htlc_signatures=b'', + htlc_minimum_msat=1, ) return local_config @log_exceptions - async def channel_establishment_flow(self, password: Optional[str], funding_tx: 'PartialTransaction', funding_sat: int, + async def channel_establishment_flow(self, password: Optional[str], funding_tx: 'PartialTransaction', funding_sat: int, push_msat: int, temp_channel_id: bytes) -> Tuple[Channel, 'PartialTransaction']: await asyncio.wait_for(self.initialized, LN_P2P_NETWORK_TIMEOUT) feerate = self.lnworker.current_feerate_per_kw() @@ -536,7 +537,7 @@ async def channel_establishment_flow(self, password: Optional[str], funding_tx: max_htlc_value_in_flight_msat=local_config.max_htlc_value_in_flight_msat, channel_flags=0x00, # not willing to announce channel channel_reserve_satoshis=local_config.reserve_sat, - htlc_minimum_msat=1, + htlc_minimum_msat=local_config.htlc_minimum_msat, ) payload = await self.wait_for_message('accept_channel', temp_channel_id) remote_per_commitment_point = payload['first_per_commitment_point'] diff --git a/electrum/lnutil.py b/electrum/lnutil.py index 22349a29f..03d6cfa6e 100644 --- a/electrum/lnutil.py +++ b/electrum/lnutil.py @@ -69,6 +69,7 @@ class Config(StoredObject): max_accepted_htlcs = attr.ib(type=int) initial_msat = attr.ib(type=int) reserve_sat = attr.ib(type=int) + htlc_minimum_msat = attr.ib(type=int) @attr.s class LocalConfig(Config): @@ -80,7 +81,6 @@ class LocalConfig(Config): @attr.s class RemoteConfig(Config): - htlc_minimum_msat = attr.ib(type=int) next_per_commitment_point = attr.ib(type=bytes, converter=hex_to_bytes) current_per_commitment_point = attr.ib(default=None, type=bytes, converter=hex_to_bytes) diff --git a/electrum/tests/test_lnchannel.py b/electrum/tests/test_lnchannel.py index 46b68a4ff..14baabf87 100644 --- a/electrum/tests/test_lnchannel.py +++ b/electrum/tests/test_lnchannel.py @@ -83,6 +83,7 @@ def create_channel_state(funding_txid, funding_index, funding_sat, is_initiator, was_announced=False, current_commitment_signature=None, current_htlc_signatures=None, + htlc_minimum_msat=1, ), "constraints":lnpeer.ChannelConstraints( capacity=funding_sat, diff --git a/electrum/wallet_db.py b/electrum/wallet_db.py index 952c97018..0f10ebf19 100644 --- a/electrum/wallet_db.py +++ b/electrum/wallet_db.py @@ -50,7 +50,7 @@ OLD_SEED_VERSION = 4 # electrum versions < 2.0 NEW_SEED_VERSION = 11 # electrum versions >= 2.0 -FINAL_SEED_VERSION = 26 # electrum >= 2.7 will set this to prevent +FINAL_SEED_VERSION = 27 # electrum >= 2.7 will set this to prevent # old versions from overwriting new format @@ -172,6 +172,7 @@ def upgrade(self): self._convert_version_24() self._convert_version_25() self._convert_version_26() + self._convert_version_27() self.put('seed_version', FINAL_SEED_VERSION) # just to be sure self._after_upgrade_tasks() @@ -587,6 +588,14 @@ def _convert_version_26(self): c['closing_height'] = closing_txid, closing_height, closing_timestamp self.data['seed_version'] = 26 + def _convert_version_27(self): + if not self._is_upgrade_method_needed(26, 26): + return + channels = self.data.get('channels', {}) + for channel_id, c in channels.items(): + c['local_config']['htlc_minimum_msat'] = 1 + self.data['seed_version'] = 27 + def _convert_imported(self): if not self._is_upgrade_method_needed(0, 13): return From 5c8455d00b367da8d37cc54a15d1a655779cf687 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Thu, 26 Mar 2020 06:42:08 +0100 Subject: [PATCH 08/69] lnchannel: when adding HTLCs, run checks for both directions --- electrum/lnchannel.py | 49 ++++++++++++++++++-------------- electrum/tests/test_lnchannel.py | 2 +- 2 files changed, 29 insertions(+), 22 deletions(-) diff --git a/electrum/lnchannel.py b/electrum/lnchannel.py index ca735faeb..2a9178072 100644 --- a/electrum/lnchannel.py +++ b/electrum/lnchannel.py @@ -404,28 +404,35 @@ def set_frozen(self, b: bool) -> None: if self.lnworker: self.lnworker.network.trigger_callback('channel', self) - def _assert_we_can_add_htlc(self, amount_msat: int) -> None: - """Raises PaymentFailure if the local party cannot add this new HTLC. - (this is relevant both for payments initiated by us and when forwarding) + def _assert_can_add_htlc(self, *, htlc_proposer: HTLCOwner, amount_msat: int) -> None: + """Raises PaymentFailure if the htlc_proposer cannot add this new HTLC. + (this is relevant both for forwarding and endpoint) """ # TODO check if this method uses correct ctns (should use "latest" + 1) + # TODO review all these checks... e.g. shouldn't we check both parties' ctx sometimes? + htlc_receiver = htlc_proposer.inverted() if self.is_closed(): raise PaymentFailure('Channel closed') if self.get_state() != channel_states.OPEN: raise PaymentFailure('Channel not open', self.get_state()) - if not self.can_send_ctx_updates(): - raise PaymentFailure('Channel cannot send ctx updates') - if not self.can_send_update_add_htlc(): - raise PaymentFailure('Channel cannot add htlc') - if self.available_to_spend(LOCAL) < amount_msat: - raise PaymentFailure(f'Not enough local balance. Have: {self.available_to_spend(LOCAL)}, Need: {amount_msat}') - if len(self.hm.htlcs(LOCAL)) + 1 > self.config[REMOTE].max_accepted_htlcs: + if htlc_proposer == LOCAL: + if not self.can_send_ctx_updates(): + raise PaymentFailure('Channel cannot send ctx updates') + if not self.can_send_update_add_htlc(): + raise PaymentFailure('Channel cannot add htlc') + if amount_msat <= 0: + raise PaymentFailure("HTLC value cannot must be >= 0") + if self.available_to_spend(htlc_proposer) < amount_msat: + raise PaymentFailure(f'Not enough local balance. Have: {self.available_to_spend(htlc_proposer)}, Need: {amount_msat}') + if len(self.hm.htlcs(htlc_proposer)) + 1 > self.config[htlc_receiver].max_accepted_htlcs: raise PaymentFailure('Too many HTLCs already in channel') - current_htlc_sum = (htlcsum(self.hm.htlcs_by_direction(LOCAL, SENT).values()) - + htlcsum(self.hm.htlcs_by_direction(LOCAL, RECEIVED).values())) - if current_htlc_sum + amount_msat > self.config[REMOTE].max_htlc_value_in_flight_msat: - raise PaymentFailure(f'HTLC value sum (sum of pending htlcs: {current_htlc_sum/1000} sat plus new htlc: {amount_msat/1000} sat) would exceed max allowed: {self.config[REMOTE].max_htlc_value_in_flight_msat/1000} sat') - if amount_msat < self.config[REMOTE].htlc_minimum_msat: + current_htlc_sum = (htlcsum(self.hm.htlcs_by_direction(htlc_proposer, SENT).values()) + + htlcsum(self.hm.htlcs_by_direction(htlc_proposer, RECEIVED).values())) + if current_htlc_sum + amount_msat > self.config[htlc_receiver].max_htlc_value_in_flight_msat: + raise PaymentFailure(f'HTLC value sum (sum of pending htlcs: {current_htlc_sum/1000} sat ' + f'plus new htlc: {amount_msat/1000} sat) ' + f'would exceed max allowed: {self.config[htlc_receiver].max_htlc_value_in_flight_msat/1000} sat') + if amount_msat < self.config[htlc_receiver].htlc_minimum_msat: raise PaymentFailure(f'HTLC value too small: {amount_msat} msat') if amount_msat > LN_MAX_HTLC_VALUE_MSAT and not self._ignore_max_htlc_value: raise PaymentFailure(f"HTLC value over protocol maximum: {amount_msat} > {LN_MAX_HTLC_VALUE_MSAT} msat") @@ -437,7 +444,7 @@ def can_pay(self, amount_msat: int) -> bool: if self.is_frozen(): return False try: - self._assert_we_can_add_htlc(amount_msat) + self._assert_can_add_htlc(htlc_proposer=LOCAL, amount_msat=amount_msat) except PaymentFailure: return False return True @@ -459,7 +466,7 @@ def add_htlc(self, htlc: UpdateAddHtlc) -> UpdateAddHtlc: if isinstance(htlc, dict): # legacy conversion # FIXME remove htlc = UpdateAddHtlc(**htlc) assert isinstance(htlc, UpdateAddHtlc) - self._assert_we_can_add_htlc(htlc.amount_msat) + self._assert_can_add_htlc(htlc_proposer=LOCAL, amount_msat=htlc.amount_msat) if htlc.htlc_id is None: htlc = attr.evolve(htlc, htlc_id=self.hm.get_next_htlc_id(LOCAL)) with self.db_lock: @@ -478,12 +485,12 @@ def receive_htlc(self, htlc: UpdateAddHtlc, onion_packet:bytes = None) -> Update if isinstance(htlc, dict): # legacy conversion # FIXME remove htlc = UpdateAddHtlc(**htlc) assert isinstance(htlc, UpdateAddHtlc) + try: + self._assert_can_add_htlc(htlc_proposer=REMOTE, amount_msat=htlc.amount_msat) + except PaymentFailure as e: + raise RemoteMisbehaving(e) from e if htlc.htlc_id is None: # used in unit tests htlc = attr.evolve(htlc, htlc_id=self.hm.get_next_htlc_id(REMOTE)) - if 0 <= self.available_to_spend(REMOTE) < htlc.amount_msat: - raise RemoteMisbehaving('Remote dipped below channel reserve.' +\ - f' Available at remote: {self.available_to_spend(REMOTE)},' +\ - f' HTLC amount: {htlc.amount_msat}') with self.db_lock: self.hm.recv_htlc(htlc) local_ctn = self.get_latest_ctn(LOCAL) diff --git a/electrum/tests/test_lnchannel.py b/electrum/tests/test_lnchannel.py index 14baabf87..3c3a9a323 100644 --- a/electrum/tests/test_lnchannel.py +++ b/electrum/tests/test_lnchannel.py @@ -671,7 +671,7 @@ def test_max_htlc_value(self): bob_channel._ignore_max_htlc_value = False with self.assertRaises(lnutil.PaymentFailure): alice_channel.add_htlc(htlc_dict) - with self.assertRaises(lnutil.PaymentFailure): + with self.assertRaises(lnutil.RemoteMisbehaving): bob_channel.receive_htlc(htlc_dict) alice_channel._ignore_max_htlc_value = True From 79d202485e1747ad5d16cecb7c35db64ce502177 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Thu, 26 Mar 2020 07:00:15 +0100 Subject: [PATCH 09/69] lnworker: rename can_send to num_sats_can_send --- electrum/gui/kivy/uix/dialogs/invoice_dialog.py | 2 +- electrum/gui/kivy/uix/dialogs/lightning_channels.py | 13 ++++++++++--- electrum/gui/kivy/uix/dialogs/request_dialog.py | 9 +++++++-- electrum/gui/qt/channels_list.py | 7 ++++--- electrum/lnworker.py | 6 +++--- 5 files changed, 25 insertions(+), 12 deletions(-) diff --git a/electrum/gui/kivy/uix/dialogs/invoice_dialog.py b/electrum/gui/kivy/uix/dialogs/invoice_dialog.py index a84432cd6..249e7f638 100644 --- a/electrum/gui/kivy/uix/dialogs/invoice_dialog.py +++ b/electrum/gui/kivy/uix/dialogs/invoice_dialog.py @@ -105,7 +105,7 @@ def update_status(self): self.status_color = pr_color[self.status] self.can_pay = self.status in [PR_UNPAID, PR_FAILED] if self.can_pay and self.is_lightning and self.app.wallet.lnworker: - if self.amount and self.amount > self.app.wallet.lnworker.can_send(): + if self.amount and self.amount > self.app.wallet.lnworker.num_sats_can_send(): self.warning = _('Warning') + ': ' + _('This amount exceeds the maximum you can currently send with your channels') def on_dismiss(self): diff --git a/electrum/gui/kivy/uix/dialogs/lightning_channels.py b/electrum/gui/kivy/uix/dialogs/lightning_channels.py index 5453d1e79..47d85e0d1 100644 --- a/electrum/gui/kivy/uix/dialogs/lightning_channels.py +++ b/electrum/gui/kivy/uix/dialogs/lightning_channels.py @@ -1,14 +1,21 @@ import asyncio import binascii +from typing import TYPE_CHECKING + from kivy.lang import Builder from kivy.factory import Factory from kivy.uix.popup import Popup from kivy.clock import Clock + from electrum.util import bh2u from electrum.lnutil import LOCAL, REMOTE, format_short_channel_id from electrum.gui.kivy.i18n import _ from .question import Question +if TYPE_CHECKING: + from ...main_window import ElectrumWindow + + Builder.load_string(r''' details: {} @@ -267,7 +274,7 @@ def _force_close(self, b): class LightningChannelsDialog(Factory.Popup): - def __init__(self, app): + def __init__(self, app: 'ElectrumWindow'): super(LightningChannelsDialog, self).__init__() self.clocks = [] self.app = app @@ -321,5 +328,5 @@ def update(self): def update_can_send(self): lnworker = self.app.wallet.lnworker - self.can_send = self.app.format_amount_and_units(lnworker.can_send()) - self.can_receive = self.app.format_amount_and_units(lnworker.can_receive()) + self.can_send = self.app.format_amount_and_units(lnworker.num_sats_can_send()) + self.can_receive = self.app.format_amount_and_units(lnworker.num_sats_can_receive()) diff --git a/electrum/gui/kivy/uix/dialogs/request_dialog.py b/electrum/gui/kivy/uix/dialogs/request_dialog.py index a6a6a388e..d319022d9 100644 --- a/electrum/gui/kivy/uix/dialogs/request_dialog.py +++ b/electrum/gui/kivy/uix/dialogs/request_dialog.py @@ -1,3 +1,5 @@ +from typing import TYPE_CHECKING + from kivy.factory import Factory from kivy.lang import Builder from kivy.core.clipboard import Clipboard @@ -8,6 +10,9 @@ from electrum.util import pr_tooltips, pr_color, get_request_status from electrum.util import PR_UNKNOWN, PR_UNPAID, PR_FAILED, PR_TYPE_LN +if TYPE_CHECKING: + from ...main_window import ElectrumWindow + Builder.load_string(''' @@ -84,7 +89,7 @@ class RequestDialog(Factory.Popup): def __init__(self, title, data, key, *, is_lightning=False): self.status = PR_UNKNOWN Factory.Popup.__init__(self) - self.app = App.get_running_app() + self.app = App.get_running_app() # type: ElectrumWindow self.title = title self.data = data self.key = key @@ -107,7 +112,7 @@ def update_status(self): self.status, self.status_str = get_request_status(req) self.status_color = pr_color[self.status] if self.status == PR_UNPAID and self.is_lightning and self.app.wallet.lnworker: - if self.amount and self.amount > self.app.wallet.lnworker.can_receive(): + if self.amount and self.amount > self.app.wallet.lnworker.num_sats_can_receive(): self.warning = _('Warning') + ': ' + _('This amount exceeds the maximum you can currently receive with your channels') def on_dismiss(self): diff --git a/electrum/gui/qt/channels_list.py b/electrum/gui/qt/channels_list.py index 12f519d16..63292ca20 100644 --- a/electrum/gui/qt/channels_list.py +++ b/electrum/gui/qt/channels_list.py @@ -14,6 +14,7 @@ from electrum.lnchannel import Channel, peer_states from electrum.wallet import Abstract_Wallet from electrum.lnutil import LOCAL, REMOTE, format_short_channel_id, LN_MAX_FUNDING_SAT +from electrum.lnworker import LNWallet from .util import (MyTreeView, WindowModalDialog, Buttons, OkButton, CancelButton, EnterButton, WaitingDialog, MONOSPACE_FONT, ColorScheme) @@ -224,10 +225,10 @@ def _update_chan_frozen_bg(self, *, chan: Channel, items: Sequence[QStandardItem item.setBackground(self._default_item_bg_brush) item.setToolTip("") - def update_can_send(self, lnworker): - msg = _('Can send') + ' ' + self.parent.format_amount(lnworker.can_send())\ + def update_can_send(self, lnworker: LNWallet): + msg = _('Can send') + ' ' + self.parent.format_amount(lnworker.num_sats_can_send())\ + ' ' + self.parent.base_unit() + '; '\ - + _('can receive') + ' ' + self.parent.format_amount(lnworker.can_receive())\ + + _('can receive') + ' ' + self.parent.format_amount(lnworker.num_sats_can_receive())\ + ' ' + self.parent.base_unit() self.can_send_label.setText(msg) diff --git a/electrum/lnworker.py b/electrum/lnworker.py index 5264606c4..f826d544c 100644 --- a/electrum/lnworker.py +++ b/electrum/lnworker.py @@ -7,7 +7,7 @@ from decimal import Decimal import random import time -from typing import Optional, Sequence, Tuple, List, Dict, TYPE_CHECKING, NamedTuple +from typing import Optional, Sequence, Tuple, List, Dict, TYPE_CHECKING, NamedTuple, Union import threading import socket import json @@ -1314,11 +1314,11 @@ def get_balance(self): with self.lock: return Decimal(sum(chan.balance(LOCAL) if not chan.is_closed() else 0 for chan in self.channels.values()))/1000 - def can_send(self): + def num_sats_can_send(self) -> Union[Decimal, int]: with self.lock: return Decimal(max(chan.available_to_spend(LOCAL) if chan.is_open() else 0 for chan in self.channels.values()))/1000 if self.channels else 0 - def can_receive(self): + def num_sats_can_receive(self) -> Union[Decimal, int]: with self.lock: return Decimal(max(chan.available_to_spend(REMOTE) if chan.is_open() else 0 for chan in self.channels.values()))/1000 if self.channels else 0 From 3ed6afce64b44431be87409e6ef64de8d29c979f Mon Sep 17 00:00:00 2001 From: SomberNight Date: Thu, 26 Mar 2020 07:09:38 +0100 Subject: [PATCH 10/69] lnchannel: implement freezing channels (for receiving) A bit weird, I know... :) It allows for rebalancing our own channels! :P --- electrum/gui/qt/channels_list.py | 39 ++++++++++++++++++++------------ electrum/lnchannel.py | 36 ++++++++++++++++++++++------- electrum/lnrouter.py | 5 ++-- electrum/lnworker.py | 7 +----- 4 files changed, 56 insertions(+), 31 deletions(-) diff --git a/electrum/gui/qt/channels_list.py b/electrum/gui/qt/channels_list.py index 63292ca20..6b45b677e 100644 --- a/electrum/gui/qt/channels_list.py +++ b/electrum/gui/qt/channels_list.py @@ -146,10 +146,15 @@ def create_menu(self, position): cc.addAction(_("Long Channel ID"), lambda: self.place_text_on_clipboard(channel_id.hex(), title=_("Long Channel ID"))) - if not chan.is_frozen(): - menu.addAction(_("Freeze"), lambda: chan.set_frozen(True)) + if not chan.is_frozen_for_sending(): + menu.addAction(_("Freeze (for sending)"), lambda: chan.set_frozen_for_sending(True)) else: - menu.addAction(_("Unfreeze"), lambda: chan.set_frozen(False)) + menu.addAction(_("Unfreeze (for sending)"), lambda: chan.set_frozen_for_sending(False)) + if not chan.is_frozen_for_receiving(): + menu.addAction(_("Freeze (for receiving)"), lambda: chan.set_frozen_for_receiving(True)) + else: + menu.addAction(_("Unfreeze (for receiving)"), lambda: chan.set_frozen_for_receiving(False)) + funding_tx = self.parent.wallet.db.get_transaction(chan.funding_outpoint.txid) if funding_tx: @@ -212,18 +217,22 @@ def do_update_rows(self, wallet): def _update_chan_frozen_bg(self, *, chan: Channel, items: Sequence[QStandardItem]): assert self._default_item_bg_brush is not None - for col in [ - self.Columns.LOCAL_BALANCE, - self.Columns.REMOTE_BALANCE, - self.Columns.CHANNEL_STATUS, - ]: - item = items[col] - if chan.is_frozen(): - item.setBackground(ColorScheme.BLUE.as_color(True)) - item.setToolTip(_("This channel is frozen. Frozen channels will not be used for outgoing payments.")) - else: - item.setBackground(self._default_item_bg_brush) - item.setToolTip("") + # frozen for sending + item = items[self.Columns.LOCAL_BALANCE] + if chan.is_frozen_for_sending(): + item.setBackground(ColorScheme.BLUE.as_color(True)) + item.setToolTip(_("This channel is frozen for sending. It will not be used for outgoing payments.")) + else: + item.setBackground(self._default_item_bg_brush) + item.setToolTip("") + # frozen for receiving + item = items[self.Columns.REMOTE_BALANCE] + if chan.is_frozen_for_receiving(): + item.setBackground(ColorScheme.BLUE.as_color(True)) + item.setToolTip(_("This channel is frozen for receiving. It will not be included in invoices.")) + else: + item.setBackground(self._default_item_bg_brush) + item.setToolTip("") def update_can_send(self, lnworker: LNWallet): msg = _('Can send') + ' ' + self.parent.format_amount(lnworker.num_sats_can_send())\ diff --git a/electrum/lnchannel.py b/electrum/lnchannel.py index 2a9178072..4c89b9a81 100644 --- a/electrum/lnchannel.py +++ b/electrum/lnchannel.py @@ -392,18 +392,30 @@ def delete_closing_height(self): def is_redeemed(self): return self.get_state() == channel_states.REDEEMED - def is_frozen(self) -> bool: - """Whether the user has marked this channel as frozen. + def is_frozen_for_sending(self) -> bool: + """Whether the user has marked this channel as frozen for sending. Frozen channels are not supposed to be used for new outgoing payments. (note that payment-forwarding ignores this option) """ return self.storage.get('frozen_for_sending', False) - def set_frozen(self, b: bool) -> None: + def set_frozen_for_sending(self, b: bool) -> None: self.storage['frozen_for_sending'] = bool(b) if self.lnworker: self.lnworker.network.trigger_callback('channel', self) + def is_frozen_for_receiving(self) -> bool: + """Whether the user has marked this channel as frozen for receiving. + Frozen channels are not supposed to be used for new incoming payments. + (note that payment-forwarding ignores this option) + """ + return self.storage.get('frozen_for_receiving', False) + + def set_frozen_for_receiving(self, b: bool) -> None: + self.storage['frozen_for_receiving'] = bool(b) + if self.lnworker: + self.lnworker.network.trigger_callback('channel', self) + def _assert_can_add_htlc(self, *, htlc_proposer: HTLCOwner, amount_msat: int) -> None: """Raises PaymentFailure if the htlc_proposer cannot add this new HTLC. (this is relevant both for forwarding and endpoint) @@ -437,11 +449,9 @@ def _assert_can_add_htlc(self, *, htlc_proposer: HTLCOwner, amount_msat: int) -> if amount_msat > LN_MAX_HTLC_VALUE_MSAT and not self._ignore_max_htlc_value: raise PaymentFailure(f"HTLC value over protocol maximum: {amount_msat} > {LN_MAX_HTLC_VALUE_MSAT} msat") - def can_pay(self, amount_msat: int) -> bool: - """Returns whether we can initiate a new payment of given value. - (we are the payer, not just a forwarding node) - """ - if self.is_frozen(): + def can_pay(self, amount_msat: int, *, check_frozen=False) -> bool: + """Returns whether we can add an HTLC of given value.""" + if check_frozen and self.is_frozen_for_sending(): return False try: self._assert_can_add_htlc(htlc_proposer=LOCAL, amount_msat=amount_msat) @@ -449,6 +459,16 @@ def can_pay(self, amount_msat: int) -> bool: return False return True + def can_receive(self, amount_msat: int, *, check_frozen=False) -> bool: + """Returns whether the remote can add an HTLC of given value.""" + if check_frozen and self.is_frozen_for_receiving(): + return False + try: + self._assert_can_add_htlc(htlc_proposer=REMOTE, amount_msat=amount_msat) + except PaymentFailure: + return False + return True + def should_try_to_reestablish_peer(self) -> bool: return channel_states.PREOPENING < self._state < channel_states.FORCE_CLOSING and self.peer_state == peer_states.DISCONNECTED diff --git a/electrum/lnrouter.py b/electrum/lnrouter.py index 09dbdc0a5..e830970f7 100644 --- a/electrum/lnrouter.py +++ b/electrum/lnrouter.py @@ -205,11 +205,12 @@ def inspect_edge(): is_mine = edge_channel_id in my_channels if is_mine: if edge_startnode == nodeA: # payment outgoing, on our channel - if not my_channels[edge_channel_id].can_pay(amount_msat): + if not my_channels[edge_channel_id].can_pay(amount_msat, check_frozen=True): return else: # payment incoming, on our channel. (funny business, cycle weirdness) assert edge_endnode == nodeA, (bh2u(edge_startnode), bh2u(edge_endnode)) - pass # TODO? + if not my_channels[edge_channel_id].can_receive(amount_msat, check_frozen=True): + return edge_cost, fee_for_edge_msat = self._edge_cost( edge_channel_id, start_node=edge_startnode, diff --git a/electrum/lnworker.py b/electrum/lnworker.py index f826d544c..bafe2a5a3 100644 --- a/electrum/lnworker.py +++ b/electrum/lnworker.py @@ -1266,12 +1266,7 @@ async def _calc_routing_hints_for_invoice(self, amount_sat): if chan.short_channel_id is not None} # note: currently we add *all* our channels; but this might be a privacy leak? for chan in channels: - # check channel is open - if chan.get_state() != channel_states.OPEN: - continue - # check channel has sufficient balance - # FIXME because of on-chain fees of ctx, this check is insufficient - if amount_sat and chan.balance(REMOTE) // 1000 < amount_sat: + if not chan.can_receive(amount_sat, check_frozen=True): continue chan_id = chan.short_channel_id assert isinstance(chan_id, bytes), chan_id From 7ac1cace7a7caaa0d0cc32a1cb0a60c0ddc4c132 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Fri, 27 Mar 2020 02:28:43 +0100 Subject: [PATCH 11/69] wallet_db.clear_history: now clears prevouts_by_scripthash too (which is the logical thing to do, as it too will be rebuilt as part of the history, and the parts of it that might not be present after the rebuild is exactly what a call to "clear_history" is supposed to get rid of) --- electrum/wallet_db.py | 1 + 1 file changed, 1 insertion(+) diff --git a/electrum/wallet_db.py b/electrum/wallet_db.py index 0f10ebf19..44065400f 100644 --- a/electrum/wallet_db.py +++ b/electrum/wallet_db.py @@ -1082,6 +1082,7 @@ def clear_history(self): self.history.clear() self.verified_tx.clear() self.tx_fees.clear() + self._prevouts_by_scripthash.clear() def _convert_dict(self, path, key, v): if key == 'transactions': From bb35e330fb47cf3ec46d4aadb7ff50167539faa9 Mon Sep 17 00:00:00 2001 From: ThomasV Date: Fri, 27 Mar 2020 11:19:27 +0100 Subject: [PATCH 12/69] do not show freeze/unfreeze channel options if channel is closed --- electrum/gui/qt/channels_list.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/electrum/gui/qt/channels_list.py b/electrum/gui/qt/channels_list.py index 6b45b677e..609f24bf3 100644 --- a/electrum/gui/qt/channels_list.py +++ b/electrum/gui/qt/channels_list.py @@ -145,16 +145,15 @@ def create_menu(self, position): cc = self.add_copy_menu(menu, idx) cc.addAction(_("Long Channel ID"), lambda: self.place_text_on_clipboard(channel_id.hex(), title=_("Long Channel ID"))) - - if not chan.is_frozen_for_sending(): - menu.addAction(_("Freeze (for sending)"), lambda: chan.set_frozen_for_sending(True)) - else: - menu.addAction(_("Unfreeze (for sending)"), lambda: chan.set_frozen_for_sending(False)) - if not chan.is_frozen_for_receiving(): - menu.addAction(_("Freeze (for receiving)"), lambda: chan.set_frozen_for_receiving(True)) - else: - menu.addAction(_("Unfreeze (for receiving)"), lambda: chan.set_frozen_for_receiving(False)) - + if not chan.is_closed(): + if not chan.is_frozen_for_sending(): + menu.addAction(_("Freeze (for sending)"), lambda: chan.set_frozen_for_sending(True)) + else: + menu.addAction(_("Unfreeze (for sending)"), lambda: chan.set_frozen_for_sending(False)) + if not chan.is_frozen_for_receiving(): + menu.addAction(_("Freeze (for receiving)"), lambda: chan.set_frozen_for_receiving(True)) + else: + menu.addAction(_("Unfreeze (for receiving)"), lambda: chan.set_frozen_for_receiving(False)) funding_tx = self.parent.wallet.db.get_transaction(chan.funding_outpoint.txid) if funding_tx: From 5b7ce98ab2d97b43fa90f9bbfc6faa3820d8e512 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Fri, 27 Mar 2020 19:06:30 +0100 Subject: [PATCH 13/69] lnchannel: fix included_htlcs --- electrum/lnchannel.py | 2 +- electrum/lnutil.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/electrum/lnchannel.py b/electrum/lnchannel.py index 4c89b9a81..b0e45cd18 100644 --- a/electrum/lnchannel.py +++ b/electrum/lnchannel.py @@ -746,7 +746,7 @@ def included_htlcs(self, subject, direction, ctn=None): ctn = self.get_oldest_unrevoked_ctn(subject) feerate = self.get_feerate(subject, ctn) conf = self.config[subject] - if (subject, direction) in [(REMOTE, RECEIVED), (LOCAL, SENT)]: + if direction == RECEIVED: weight = HTLC_SUCCESS_WEIGHT else: weight = HTLC_TIMEOUT_WEIGHT diff --git a/electrum/lnutil.py b/electrum/lnutil.py index 03d6cfa6e..a7aa113af 100644 --- a/electrum/lnutil.py +++ b/electrum/lnutil.py @@ -531,8 +531,8 @@ def inverted(self): return HTLCOwner(-self) class Direction(IntFlag): - SENT = -1 - RECEIVED = 1 + SENT = -1 # in the context of HTLCs: "offered" HTLCs + RECEIVED = 1 # in the context of HTLCs: "received" HTLCs SENT = Direction.SENT RECEIVED = Direction.RECEIVED From 74982719274c063655c150717bbb22d35b719b51 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Sat, 28 Mar 2020 16:29:39 +0100 Subject: [PATCH 14/69] follow-up prev: htlc direction madness Sometimes direction was relative sometimes absolute... ?! No. Make it always relative (to subject). --- electrum/lnchannel.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/electrum/lnchannel.py b/electrum/lnchannel.py index b0e45cd18..cb74b468a 100644 --- a/electrum/lnchannel.py +++ b/electrum/lnchannel.py @@ -890,19 +890,15 @@ def make_commitment(self, subject, this_point, ctn) -> PartialTransaction: other = subject.inverted() local_msat = self.balance(subject, ctx_owner=subject, ctn=ctn) remote_msat = self.balance(other, ctx_owner=subject, ctn=ctn) - received_htlcs = self.hm.htlcs_by_direction(subject, SENT if subject == LOCAL else RECEIVED, ctn).values() - sent_htlcs = self.hm.htlcs_by_direction(subject, RECEIVED if subject == LOCAL else SENT, ctn).values() - if subject != LOCAL: - remote_msat -= htlcsum(received_htlcs) - local_msat -= htlcsum(sent_htlcs) - else: - remote_msat -= htlcsum(sent_htlcs) - local_msat -= htlcsum(received_htlcs) + received_htlcs = self.hm.htlcs_by_direction(subject, RECEIVED, ctn).values() + sent_htlcs = self.hm.htlcs_by_direction(subject, SENT, ctn).values() + remote_msat -= htlcsum(received_htlcs) + local_msat -= htlcsum(sent_htlcs) assert remote_msat >= 0 assert local_msat >= 0 # same htlcs as before, but now without dust. - received_htlcs = self.included_htlcs(subject, SENT if subject == LOCAL else RECEIVED, ctn) - sent_htlcs = self.included_htlcs(subject, RECEIVED if subject == LOCAL else SENT, ctn) + received_htlcs = self.included_htlcs(subject, RECEIVED, ctn) + sent_htlcs = self.included_htlcs(subject, SENT, ctn) this_config = self.config[subject] other_config = self.config[-subject] @@ -910,7 +906,7 @@ def make_commitment(self, subject, this_point, ctn) -> PartialTransaction: this_htlc_pubkey = derive_pubkey(this_config.htlc_basepoint.pubkey, this_point) other_revocation_pubkey = derive_blinded_pubkey(other_config.revocation_basepoint.pubkey, this_point) htlcs = [] # type: List[ScriptHtlc] - for is_received_htlc, htlc_list in zip((subject != LOCAL, subject == LOCAL), (received_htlcs, sent_htlcs)): + for is_received_htlc, htlc_list in zip((True, False), (received_htlcs, sent_htlcs)): for htlc in htlc_list: htlcs.append(ScriptHtlc(make_htlc_output_witness_script( is_received_htlc=is_received_htlc, From d520dc9fae08425dc13ff0302bf1e91b071bcf7c Mon Sep 17 00:00:00 2001 From: JeremyRand Date: Sun, 29 Mar 2020 04:48:39 +0000 Subject: [PATCH 15/69] Readme: Clarify dependencies of make_libsecp256k1.sh --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index 32395f74e..a6ff0596b 100644 --- a/README.rst +++ b/README.rst @@ -52,6 +52,7 @@ For elliptic curve operations, `libsecp256k1`_ is a required dependency:: Alternatively, when running from a cloned repository, a script is provided to build libsecp256k1 yourself:: + sudo apt-get install automake libtool ./contrib/make_libsecp256k1.sh Due to the need for fast symmetric ciphers, either one of `pycryptodomex`_ From 875e6b31b1d4fa0e133a7176e7592361c45beee7 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Sun, 29 Mar 2020 07:51:48 +0200 Subject: [PATCH 16/69] make_libsecp256k1.sh: add comment how to cross-compile to Windows related: #5976, #6054 --- contrib/make_libsecp256k1.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/contrib/make_libsecp256k1.sh b/contrib/make_libsecp256k1.sh index 8b602f34f..6981ffbf5 100755 --- a/contrib/make_libsecp256k1.sh +++ b/contrib/make_libsecp256k1.sh @@ -1,5 +1,15 @@ #!/bin/bash +# This script was tested on Linux and MacOS hosts, where it can be used +# to build native libsecp256k1 binaries. +# +# It can also be used to cross-compile to Windows: +# $ sudo apt-get install mingw-w64 +# For a Windows x86 (32-bit) target, run: +# $ GCC_TRIPLET_HOST="i686-w64-mingw32" ./contrib/make_libsecp256k1.sh +# Or for a Windows x86_64 (64-bit) target, run: +# $ GCC_TRIPLET_HOST="x86_64-w64-mingw32" ./contrib/make_libsecp256k1.sh + LIBSECP_VERSION="b408c6a8b287003d1ade5709e6f7bc3c7f1d5be7" set -e From 001ee25604c37c41037b220dae2b98056f329032 Mon Sep 17 00:00:00 2001 From: JeremyRand Date: Sun, 29 Mar 2020 05:53:31 +0000 Subject: [PATCH 17/69] UTXOList: Split stretch_column out of __init__ Makes it easier to subclass UTXOList without code duplication. --- electrum/gui/qt/utxo_list.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/electrum/gui/qt/utxo_list.py b/electrum/gui/qt/utxo_list.py index 257abf571..db7222645 100644 --- a/electrum/gui/qt/utxo_list.py +++ b/electrum/gui/qt/utxo_list.py @@ -56,10 +56,11 @@ class Columns(IntEnum): Columns.OUTPOINT: _('Output point'), } filter_columns = [Columns.ADDRESS, Columns.LABEL, Columns.OUTPOINT] + stretch_column = Columns.LABEL def __init__(self, parent): super().__init__(parent, self.create_menu, - stretch_column=self.Columns.LABEL, + stretch_column=self.stretch_column, editable_columns=[]) self._spend_set = None self._utxo_dict = {} From 90f3b667aad43b21adbbbe29204e9b317130b49d Mon Sep 17 00:00:00 2001 From: SomberNight Date: Mon, 30 Mar 2020 01:42:14 +0200 Subject: [PATCH 18/69] small clean-up re max CLTV delta for LN --- electrum/lnpeer.py | 5 ++++- electrum/lnrouter.py | 4 ++-- electrum/lnutil.py | 2 +- electrum/lnworker.py | 2 +- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/electrum/lnpeer.py b/electrum/lnpeer.py index 2c79099e2..eb8f3f6f3 100644 --- a/electrum/lnpeer.py +++ b/electrum/lnpeer.py @@ -1027,13 +1027,16 @@ def pay(self, route: 'LNPaymentRoute', chan: Channel, amount_msat: int, assert amount_msat > 0, "amount_msat is not greater zero" if not chan.can_send_update_add_htlc(): raise PaymentFailure("Channel cannot send update_add_htlc") + local_height = self.network.get_local_height() # create onion packet - final_cltv = self.network.get_local_height() + min_final_cltv_expiry + final_cltv = local_height + min_final_cltv_expiry hops_data, amount_msat, cltv = calc_hops_data_for_payment(route, amount_msat, final_cltv) assert final_cltv <= cltv, (final_cltv, cltv) secret_key = os.urandom(32) onion = new_onion_packet([x.node_id for x in route], secret_key, hops_data, associated_data=payment_hash) # create htlc + if cltv > local_height + lnutil.NBLOCK_CLTV_EXPIRY_TOO_FAR_INTO_FUTURE: + raise PaymentFailure(f"htlc expiry too far into future. (in {cltv-local_height} blocks)") htlc = UpdateAddHtlc(amount_msat=amount_msat, payment_hash=payment_hash, cltv_expiry=cltv, timestamp=int(time.time())) htlc = chan.add_htlc(htlc) chan.set_onion_key(htlc.htlc_id, secret_key) diff --git a/electrum/lnrouter.py b/electrum/lnrouter.py index e830970f7..eb38e46ec 100644 --- a/electrum/lnrouter.py +++ b/electrum/lnrouter.py @@ -31,6 +31,7 @@ from .logging import Logger from .lnutil import NUM_MAX_EDGES_IN_PAYMENT_PATH, ShortChannelID from .channel_db import ChannelDB, Policy +from .lnutil import NBLOCK_CLTV_EXPIRY_TOO_FAR_INTO_FUTURE if TYPE_CHECKING: from .lnchannel import Channel @@ -99,8 +100,7 @@ def is_route_sane_to_use(route: LNPaymentRoute, invoice_amount_msat: int, min_fi cltv += route_edge.cltv_expiry_delta total_fee = amt - invoice_amount_msat # TODO revise ad-hoc heuristics - # cltv cannot be more than 2 months - if cltv > 60 * 144: + if cltv > NBLOCK_CLTV_EXPIRY_TOO_FAR_INTO_FUTURE: return False if not is_fee_sane(total_fee, payment_amount_msat=invoice_amount_msat): return False diff --git a/electrum/lnutil.py b/electrum/lnutil.py index a7aa113af..3c33f9f2d 100644 --- a/electrum/lnutil.py +++ b/electrum/lnutil.py @@ -193,7 +193,7 @@ class PaymentFailure(UserFacingException): pass OUR_FEE_BASE_MSAT = 1000 OUR_FEE_PROPORTIONAL_MILLIONTHS = 1 -NBLOCK_CLTV_EXPIRY_TOO_FAR_INTO_FUTURE = 4032 +NBLOCK_CLTV_EXPIRY_TOO_FAR_INTO_FUTURE = 28 * 144 # When we open a channel, the remote peer has to support at least this diff --git a/electrum/lnworker.py b/electrum/lnworker.py index bafe2a5a3..96930517c 100644 --- a/electrum/lnworker.py +++ b/electrum/lnworker.py @@ -1040,7 +1040,7 @@ def _check_invoice(invoice, amount_sat=None): addr.amount = Decimal(amount_sat) / COIN if addr.amount is None: raise InvoiceError(_("Missing amount")) - if addr.get_min_final_cltv_expiry() > 60 * 144: + if addr.get_min_final_cltv_expiry() > lnutil.NBLOCK_CLTV_EXPIRY_TOO_FAR_INTO_FUTURE: raise InvoiceError("{}\n{}".format( _("Invoice wants us to risk locking funds for unreasonably long."), f"min_final_cltv_expiry: {addr.get_min_final_cltv_expiry()}")) From acb0d7ebac81a36ed6ce1fd01a392302aa675791 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Mon, 30 Mar 2020 01:53:34 +0200 Subject: [PATCH 19/69] lnchannel: better checks for "update_add_htlc" I believe this now implements all the checks listed in BOLT-02 for update_add_htlc, however, the BOLT is sometimes ambiguous, and actually the checks listed there IMO are insufficient. There are still some TODOs, in part because of the above. --- electrum/lnchannel.py | 156 +++++++++++++++++++++---------- electrum/lnutil.py | 38 +++++++- electrum/tests/test_lnchannel.py | 8 +- electrum/tests/test_lnutil.py | 10 +- 4 files changed, 150 insertions(+), 62 deletions(-) diff --git a/electrum/lnchannel.py b/electrum/lnchannel.py index cb74b468a..6a0046f00 100644 --- a/electrum/lnchannel.py +++ b/electrum/lnchannel.py @@ -49,9 +49,10 @@ make_htlc_tx_with_open_channel, make_commitment, make_received_htlc, make_offered_htlc, HTLC_TIMEOUT_WEIGHT, HTLC_SUCCESS_WEIGHT, extract_ctn_from_tx_and_chan, UpdateAddHtlc, funding_output_script, SENT, RECEIVED, LOCAL, REMOTE, HTLCOwner, make_commitment_outputs, - ScriptHtlc, PaymentFailure, calc_onchain_fees, RemoteMisbehaving, make_htlc_output_witness_script, + ScriptHtlc, PaymentFailure, calc_fees_for_commitment_tx, RemoteMisbehaving, make_htlc_output_witness_script, ShortChannelID, map_htlcs_to_ctx_output_idxs, LNPeerAddr, BarePaymentAttemptLog, - LN_MAX_HTLC_VALUE_MSAT) + LN_MAX_HTLC_VALUE_MSAT, fee_for_htlc_output, offered_htlc_trim_threshold_sat, + received_htlc_trim_threshold_sat) from .lnsweep import create_sweeptxs_for_our_ctx, create_sweeptxs_for_their_ctx from .lnsweep import create_sweeptx_for_their_revoked_htlc, SweepInfo from .lnhtlc import HTLCManager @@ -141,7 +142,7 @@ def __init__(self, state: 'StoredDict', *, sweep_address=None, name=None, lnwork self.sweep_address = sweep_address self.storage = state self.db_lock = self.storage.db.lock if self.storage.db else threading.RLock() - self.config = {} + self.config = {} # type: Dict[HTLCOwner, lnutil.Config] self.config[LOCAL] = state["local_config"] self.config[REMOTE] = state["remote_config"] self.channel_id = bfh(state["channel_id"]) @@ -420,9 +421,11 @@ def _assert_can_add_htlc(self, *, htlc_proposer: HTLCOwner, amount_msat: int) -> """Raises PaymentFailure if the htlc_proposer cannot add this new HTLC. (this is relevant both for forwarding and endpoint) """ - # TODO check if this method uses correct ctns (should use "latest" + 1) - # TODO review all these checks... e.g. shouldn't we check both parties' ctx sometimes? htlc_receiver = htlc_proposer.inverted() + # note: all these tests are about the *receiver's* *next* commitment transaction, + # and the constraints are the ones imposed by their config + ctn = self.get_next_ctn(htlc_receiver) + chan_config = self.config[htlc_receiver] if self.is_closed(): raise PaymentFailure('Channel closed') if self.get_state() != channel_states.OPEN: @@ -432,23 +435,42 @@ def _assert_can_add_htlc(self, *, htlc_proposer: HTLCOwner, amount_msat: int) -> raise PaymentFailure('Channel cannot send ctx updates') if not self.can_send_update_add_htlc(): raise PaymentFailure('Channel cannot add htlc') + + # If proposer is LOCAL we apply stricter checks as that is behaviour we can control. + # This should lead to fewer disagreements (i.e. channels failing). + strict = (htlc_proposer == LOCAL) + + # check htlc raw value if amount_msat <= 0: - raise PaymentFailure("HTLC value cannot must be >= 0") - if self.available_to_spend(htlc_proposer) < amount_msat: - raise PaymentFailure(f'Not enough local balance. Have: {self.available_to_spend(htlc_proposer)}, Need: {amount_msat}') - if len(self.hm.htlcs(htlc_proposer)) + 1 > self.config[htlc_receiver].max_accepted_htlcs: - raise PaymentFailure('Too many HTLCs already in channel') - current_htlc_sum = (htlcsum(self.hm.htlcs_by_direction(htlc_proposer, SENT).values()) - + htlcsum(self.hm.htlcs_by_direction(htlc_proposer, RECEIVED).values())) - if current_htlc_sum + amount_msat > self.config[htlc_receiver].max_htlc_value_in_flight_msat: - raise PaymentFailure(f'HTLC value sum (sum of pending htlcs: {current_htlc_sum/1000} sat ' - f'plus new htlc: {amount_msat/1000} sat) ' - f'would exceed max allowed: {self.config[htlc_receiver].max_htlc_value_in_flight_msat/1000} sat') - if amount_msat < self.config[htlc_receiver].htlc_minimum_msat: + raise PaymentFailure("HTLC value must be positive") + if amount_msat < chan_config.htlc_minimum_msat: raise PaymentFailure(f'HTLC value too small: {amount_msat} msat') if amount_msat > LN_MAX_HTLC_VALUE_MSAT and not self._ignore_max_htlc_value: raise PaymentFailure(f"HTLC value over protocol maximum: {amount_msat} > {LN_MAX_HTLC_VALUE_MSAT} msat") + # check proposer can afford htlc + max_can_send_msat = self.available_to_spend(htlc_proposer, strict=strict) + if max_can_send_msat < amount_msat: + raise PaymentFailure(f'Not enough balance. can send: {max_can_send_msat}, tried: {amount_msat}') + + # check "max_accepted_htlcs" + # this is the loose check BOLT-02 specifies: + if len(self.hm.htlcs_by_direction(htlc_receiver, direction=RECEIVED, ctn=ctn)) + 1 > chan_config.max_accepted_htlcs: + raise PaymentFailure('Too many HTLCs already in channel') + # however, c-lightning is a lot stricter, so extra checks: + if strict: + max_concurrent_htlcs = min(self.config[htlc_proposer].max_accepted_htlcs, + self.config[htlc_receiver].max_accepted_htlcs) + if len(self.hm.htlcs(htlc_receiver, ctn=ctn)) + 1 > max_concurrent_htlcs: + raise PaymentFailure('Too many HTLCs already in channel') + + # check "max_htlc_value_in_flight_msat" + current_htlc_sum = htlcsum(self.hm.htlcs_by_direction(htlc_receiver, direction=RECEIVED, ctn=ctn).values()) + if current_htlc_sum + amount_msat > chan_config.max_htlc_value_in_flight_msat: + raise PaymentFailure(f'HTLC value sum (sum of pending htlcs: {current_htlc_sum/1000} sat ' + f'plus new htlc: {amount_msat/1000} sat) ' + f'would exceed max allowed: {chan_config.max_htlc_value_in_flight_msat/1000} sat') + def can_pay(self, amount_msat: int, *, check_frozen=False) -> bool: """Returns whether we can add an HTLC of given value.""" if check_frozen and self.is_frozen_for_sending(): @@ -678,15 +700,8 @@ def receive_revocation(self, revocation: RevokeAndAck): self.lnworker.payment_failed(self, htlc.payment_hash, payment_attempt) def balance(self, whose: HTLCOwner, *, ctx_owner=HTLCOwner.LOCAL, ctn: int = None) -> int: - """ - This balance in mSAT is not including reserve and fees. - So a node cannot actually use its whole balance. - But this number is simple, since it is derived simply - from the initial balance, and the value of settled HTLCs. - Note that it does not decrease once an HTLC is added, - failed or fulfilled, since the balance change is only - committed to later when the respective commitment - transaction has been revoked. + """This balance (in msat) only considers HTLCs that have been settled by ctn. + It disregards reserve, fees, and pending HTLCs (in both directions). """ assert type(whose) is HTLCOwner initial = self.config[whose].initial_msat @@ -697,8 +712,7 @@ def balance(self, whose: HTLCOwner, *, ctx_owner=HTLCOwner.LOCAL, ctn: int = Non def balance_minus_outgoing_htlcs(self, whose: HTLCOwner, *, ctx_owner: HTLCOwner = HTLCOwner.LOCAL, ctn: int = None): - """ - This balance in mSAT, which includes the value of + """This balance (in msat), which includes the value of pending outgoing HTLCs, is used in the UI. """ assert type(whose) is HTLCOwner @@ -716,27 +730,62 @@ def balance_tied_up_in_htlcs_by_direction(self, ctx_owner: HTLCOwner = LOCAL, *, ctn = self.get_next_ctn(ctx_owner) return htlcsum(self.hm.htlcs_by_direction(ctx_owner, direction, ctn).values()) - def available_to_spend(self, subject: HTLCOwner) -> int: + def available_to_spend(self, subject: HTLCOwner, *, strict: bool = True) -> int: + """The usable balance of 'subject' in msat, after taking reserve and fees into + consideration. Note that fees (and hence the result) fluctuate even without user interaction. """ - This balance in mSAT, while technically correct, can - not be used in the UI cause it fluctuates (commit fee) - """ - # FIXME whose balance? whose ctx? - # FIXME confusing/mixing ctns (should probably use latest_ctn + 1; not oldest_unrevoked + 1) assert type(subject) is HTLCOwner - ctx_owner = subject.inverted() + sender = subject + receiver = subject.inverted() + ctx_owner = receiver + # TODO but what about the other ctx? BOLT-02 only talks about checking the receiver's ctx, + # however the channel reserve is only meaningful if we also check the sender's ctx! + # in particular, note that dust limits can be different between the parties! + # but due to the racy nature of this, we cannot be sure exactly what the sender's + # next ctx will look like (e.g. what feerate it will use). hmmm :/ ctn = self.get_next_ctn(ctx_owner) - balance = self.balance_minus_outgoing_htlcs(whose=subject, ctx_owner=ctx_owner, ctn=ctn) - reserve = self.config[-subject].reserve_sat * 1000 - # TODO should we include a potential new htlc, when we are called from receive_htlc? - fees = calc_onchain_fees( - num_htlcs=len(self.included_htlcs(ctx_owner, SENT, ctn=ctn) + self.included_htlcs(ctx_owner, RECEIVED, ctn=ctn)), - feerate=self.get_feerate(ctx_owner, ctn=ctn), + sender_balance_msat = self.balance_minus_outgoing_htlcs(whose=sender, ctx_owner=ctx_owner, ctn=ctn) + receiver_balance_msat = self.balance_minus_outgoing_htlcs(whose=receiver, ctx_owner=ctx_owner, ctn=ctn) + sender_reserve_msat = self.config[receiver].reserve_sat * 1000 + receiver_reserve_msat = self.config[sender].reserve_sat * 1000 + initiator = LOCAL if self.constraints.is_initiator else REMOTE + # the initiator/funder pays on-chain fees + num_htlcs_in_ctx = len(self.included_htlcs(ctx_owner, SENT, ctn=ctn) + self.included_htlcs(ctx_owner, RECEIVED, ctn=ctn)) + feerate = self.get_feerate(ctx_owner, ctn=ctn) + ctx_fees_msat = calc_fees_for_commitment_tx( + num_htlcs=num_htlcs_in_ctx, + feerate=feerate, is_local_initiator=self.constraints.is_initiator, - )[subject] - return balance - reserve - fees + round_to_sat=False, + ) + # note: if this supposed new HTLC is large enough to create an output, the initiator needs to pay for that too + # note: if sender != initiator, both the sender and the receiver need to "afford" the payment + htlc_fee_msat = fee_for_htlc_output(feerate=feerate) + # TODO stuck channels. extra funder reserve? "fee spike buffer" (maybe only if "strict") + # see https://github.com/lightningnetwork/lightning-rfc/issues/728 + # note: in terms of on-chain outputs, as we are considering the htlc_receiver's ctx, this is a "received" HTLC + htlc_trim_threshold_msat = received_htlc_trim_threshold_sat(dust_limit_sat=self.config[receiver].dust_limit_sat, feerate=feerate) * 1000 + if strict: + # also consider the other ctx, where the trim threshold is different + # note: the 'feerate' we use is not technically correct but we have no way + # of knowing the actual future feerate ahead of time (this is a protocol bug) + htlc_trim_threshold_msat = min(htlc_trim_threshold_msat, + offered_htlc_trim_threshold_sat(dust_limit_sat=self.config[sender].dust_limit_sat, feerate=feerate) * 1000) + max_send_msat = sender_balance_msat - sender_reserve_msat - ctx_fees_msat[sender] + if max_send_msat < htlc_trim_threshold_msat: + # there will be no corresponding HTLC output + return max_send_msat + if sender == initiator: + max_send_after_htlc_fee_msat = max_send_msat - htlc_fee_msat + max_send_msat = max(htlc_trim_threshold_msat - 1, max_send_after_htlc_fee_msat) + return max_send_msat + else: + # the receiver is the initiator, so they need to be able to pay tx fees + if receiver_balance_msat - receiver_reserve_msat - ctx_fees_msat[receiver] - htlc_fee_msat < 0: + max_send_msat = htlc_trim_threshold_msat - 1 + return max_send_msat - def included_htlcs(self, subject, direction, ctn=None): + def included_htlcs(self, subject: HTLCOwner, direction: Direction, ctn: int = None) -> Sequence[UpdateAddHtlc]: """ return filter of non-dust htlcs for subjects commitment transaction, initiated by given party """ @@ -747,12 +796,11 @@ def included_htlcs(self, subject, direction, ctn=None): feerate = self.get_feerate(subject, ctn) conf = self.config[subject] if direction == RECEIVED: - weight = HTLC_SUCCESS_WEIGHT + threshold_sat = received_htlc_trim_threshold_sat(dust_limit_sat=conf.dust_limit_sat, feerate=feerate) else: - weight = HTLC_TIMEOUT_WEIGHT + threshold_sat = offered_htlc_trim_threshold_sat(dust_limit_sat=conf.dust_limit_sat, feerate=feerate) htlcs = self.hm.htlcs_by_direction(subject, direction, ctn=ctn).values() - htlc_value_after_fees = lambda htlc: htlc.amount_msat // 1000 - (weight * feerate // 1000) - return list(filter(lambda htlc: htlc_value_after_fees(htlc) >= conf.dust_limit_sat, htlcs)) + return list(filter(lambda htlc: htlc.amount_msat // 1000 >= threshold_sat, htlcs)) def get_secret_and_point(self, subject: HTLCOwner, ctn: int) -> Tuple[Optional[bytes], bytes]: assert type(subject) is HTLCOwner @@ -877,6 +925,8 @@ def update_fee(self, feerate: int, from_us: bool): # feerate uses sat/kw if self.constraints.is_initiator != from_us: raise Exception(f"Cannot update_fee: wrong initiator. us: {from_us}") + # TODO check that funder can afford the new on-chain fees (+ channel reserve) + # (maybe check both ctxs, at least if from_us is True??) with self.db_lock: if from_us: assert self.can_send_ctx_updates(), f"cannot update channel. {self.get_state()!r} {self.peer_state!r}" @@ -917,11 +967,19 @@ def make_commitment(self, subject, this_point, ctn) -> PartialTransaction: cltv_expiry=htlc.cltv_expiry), htlc)) # note: maybe flip initiator here for fee purposes, we want LOCAL and REMOTE # in the resulting dict to correspond to the to_local and to_remote *outputs* of the ctx - onchain_fees = calc_onchain_fees( + onchain_fees = calc_fees_for_commitment_tx( num_htlcs=len(htlcs), feerate=feerate, is_local_initiator=self.constraints.is_initiator == (subject == LOCAL), ) + + # TODO: we need to also include the respective channel reserves here, but not at the + # beginning of the channel lifecycle when the reserve might not be met yet + if remote_msat - onchain_fees[REMOTE] < 0: + raise Exception(f"negative remote_msat in make_commitment: {remote_msat}") + if local_msat - onchain_fees[LOCAL] < 0: + raise Exception(f"negative local_msat in make_commitment: {local_msat}") + if self.is_static_remotekey_enabled(): payment_pubkey = other_config.payment_basepoint.pubkey else: diff --git a/electrum/lnutil.py b/electrum/lnutil.py index 3c33f9f2d..1590a1558 100644 --- a/electrum/lnutil.py +++ b/electrum/lnutil.py @@ -30,8 +30,11 @@ from .lnonion import OnionRoutingFailureMessage +# defined in BOLT-03: HTLC_TIMEOUT_WEIGHT = 663 HTLC_SUCCESS_WEIGHT = 703 +COMMITMENT_TX_WEIGHT = 724 +HTLC_OUTPUT_WEIGHT = 172 LN_MAX_FUNDING_SAT = pow(2, 24) - 1 LN_MAX_HTLC_VALUE_MSAT = pow(2, 32) - 1 @@ -93,7 +96,7 @@ class FeeUpdate(StoredObject): @attr.s class ChannelConstraints(StoredObject): capacity = attr.ib(type=int) - is_initiator = attr.ib(type=bool) + is_initiator = attr.ib(type=bool) # note: sometimes also called "funder" funding_txn_minimum_depth = attr.ib(type=int) @@ -558,12 +561,39 @@ def make_commitment_outputs(*, fees_per_participant: Mapping[HTLCOwner, int], lo return htlc_outputs, c_outputs_filtered -def calc_onchain_fees(*, num_htlcs: int, feerate: int, is_local_initiator: bool) -> Dict['HTLCOwner', int]: +def offered_htlc_trim_threshold_sat(*, dust_limit_sat: int, feerate: int) -> int: + # offered htlcs strictly below this amount will be trimmed (from ctx). + # feerate is in sat/kw + # returns value in sat + weight = HTLC_TIMEOUT_WEIGHT + return dust_limit_sat + weight * feerate // 1000 + + +def received_htlc_trim_threshold_sat(*, dust_limit_sat: int, feerate: int) -> int: + # received htlcs strictly below this amount will be trimmed (from ctx). + # feerate is in sat/kw + # returns value in sat + weight = HTLC_SUCCESS_WEIGHT + return dust_limit_sat + weight * feerate // 1000 + + +def fee_for_htlc_output(*, feerate: int) -> int: + # feerate is in sat/kw + # returns fee in msat + return feerate * HTLC_OUTPUT_WEIGHT + + +def calc_fees_for_commitment_tx(*, num_htlcs: int, feerate: int, + is_local_initiator: bool, round_to_sat: bool = True) -> Dict['HTLCOwner', int]: # feerate is in sat/kw # returns fees in msats - overall_weight = 500 + 172 * num_htlcs + 224 + # note: BOLT-02 specifies that msat fees need to be rounded down to sat. + # However, the rounding needs to happen for the total fees, so if the return value + # is to be used as part of additional fee calculation then rounding should be done after that. + overall_weight = COMMITMENT_TX_WEIGHT + num_htlcs * HTLC_OUTPUT_WEIGHT fee = feerate * overall_weight - fee = fee // 1000 * 1000 + if round_to_sat: + fee = fee // 1000 * 1000 return { LOCAL: fee if is_local_initiator else 0, REMOTE: fee if not is_local_initiator else 0, diff --git a/electrum/tests/test_lnchannel.py b/electrum/tests/test_lnchannel.py index 3c3a9a323..29fa283af 100644 --- a/electrum/tests/test_lnchannel.py +++ b/electrum/tests/test_lnchannel.py @@ -612,7 +612,7 @@ def test_AddHTLCNegativeBalance(self): class TestAvailableToSpend(ElectrumTestCase): def test_DesyncHTLCs(self): alice_channel, bob_channel = create_test_channels() - self.assertEqual(499995656000, alice_channel.available_to_spend(LOCAL)) + self.assertEqual(499994624000, alice_channel.available_to_spend(LOCAL)) self.assertEqual(500000000000, bob_channel.available_to_spend(LOCAL)) paymentPreimage = b"\x01" * 32 @@ -626,13 +626,13 @@ def test_DesyncHTLCs(self): alice_idx = alice_channel.add_htlc(htlc_dict).htlc_id bob_idx = bob_channel.receive_htlc(htlc_dict).htlc_id - self.assertEqual(89994624000, alice_channel.available_to_spend(LOCAL)) + self.assertEqual(89993592000, alice_channel.available_to_spend(LOCAL)) self.assertEqual(500000000000, bob_channel.available_to_spend(LOCAL)) force_state_transition(alice_channel, bob_channel) bob_channel.fail_htlc(bob_idx) alice_channel.receive_fail_htlc(alice_idx, error_bytes=None) - self.assertEqual(89994624000, alice_channel.available_to_spend(LOCAL)) + self.assertEqual(89993592000, alice_channel.available_to_spend(LOCAL)) self.assertEqual(500000000000, bob_channel.available_to_spend(LOCAL)) # Alice now has gotten all her original balance (5 BTC) back, however, # adding a new HTLC at this point SHOULD fail, since if she adds the @@ -652,7 +652,7 @@ def test_DesyncHTLCs(self): # Now do a state transition, which will ACK the FailHTLC, making Alice # able to add the new HTLC. force_state_transition(alice_channel, bob_channel) - self.assertEqual(499995656000, alice_channel.available_to_spend(LOCAL)) + self.assertEqual(499994624000, alice_channel.available_to_spend(LOCAL)) self.assertEqual(500000000000, bob_channel.available_to_spend(LOCAL)) alice_channel.add_htlc(htlc_dict) diff --git a/electrum/tests/test_lnutil.py b/electrum/tests/test_lnutil.py index c0c01cd06..e5297a798 100644 --- a/electrum/tests/test_lnutil.py +++ b/electrum/tests/test_lnutil.py @@ -8,7 +8,7 @@ make_htlc_tx_inputs, secret_to_pubkey, derive_blinded_pubkey, derive_privkey, derive_pubkey, make_htlc_tx, extract_ctn_from_tx, UnableToDeriveSecret, get_compressed_pubkey_from_bech32, split_host_port, ConnStringFormatError, - ScriptHtlc, extract_nodeid, calc_onchain_fees, UpdateAddHtlc) + ScriptHtlc, extract_nodeid, calc_fees_for_commitment_tx, UpdateAddHtlc) from electrum.util import bh2u, bfh, MyEncoder from electrum.transaction import Transaction, PartialTransaction @@ -516,7 +516,7 @@ def test_commitment_tx_with_all_five_HTLCs_untrimmed_minimum_feerate(self): local_revocation_pubkey, local_delayedpubkey, local_delay, funding_tx_id, funding_output_index, funding_amount_satoshi, to_local_msat, to_remote_msat, local_dust_limit_satoshi, - calc_onchain_fees(num_htlcs=len(htlcs), feerate=local_feerate_per_kw, is_local_initiator=True), htlcs=htlcs) + calc_fees_for_commitment_tx(num_htlcs=len(htlcs), feerate=local_feerate_per_kw, is_local_initiator=True), htlcs=htlcs) self.sign_and_insert_remote_sig(our_commit_tx, remote_funding_pubkey, remote_signature, local_funding_pubkey, local_funding_privkey) self.assertEqual(str(our_commit_tx), output_commit_tx) @@ -593,7 +593,7 @@ def test_commitment_tx_with_one_output(self): local_revocation_pubkey, local_delayedpubkey, local_delay, funding_tx_id, funding_output_index, funding_amount_satoshi, to_local_msat, to_remote_msat, local_dust_limit_satoshi, - calc_onchain_fees(num_htlcs=0, feerate=local_feerate_per_kw, is_local_initiator=True), htlcs=[]) + calc_fees_for_commitment_tx(num_htlcs=0, feerate=local_feerate_per_kw, is_local_initiator=True), htlcs=[]) self.sign_and_insert_remote_sig(our_commit_tx, remote_funding_pubkey, remote_signature, local_funding_pubkey, local_funding_privkey) self.assertEqual(str(our_commit_tx), output_commit_tx) @@ -612,7 +612,7 @@ def test_commitment_tx_with_fee_greater_than_funder_amount(self): local_revocation_pubkey, local_delayedpubkey, local_delay, funding_tx_id, funding_output_index, funding_amount_satoshi, to_local_msat, to_remote_msat, local_dust_limit_satoshi, - calc_onchain_fees(num_htlcs=0, feerate=local_feerate_per_kw, is_local_initiator=True), htlcs=[]) + calc_fees_for_commitment_tx(num_htlcs=0, feerate=local_feerate_per_kw, is_local_initiator=True), htlcs=[]) self.sign_and_insert_remote_sig(our_commit_tx, remote_funding_pubkey, remote_signature, local_funding_pubkey, local_funding_privkey) self.assertEqual(str(our_commit_tx), output_commit_tx) @@ -670,7 +670,7 @@ def test_simple_commitment_tx_with_no_HTLCs(self): local_revocation_pubkey, local_delayedpubkey, local_delay, funding_tx_id, funding_output_index, funding_amount_satoshi, to_local_msat, to_remote_msat, local_dust_limit_satoshi, - calc_onchain_fees(num_htlcs=0, feerate=local_feerate_per_kw, is_local_initiator=True), htlcs=[]) + calc_fees_for_commitment_tx(num_htlcs=0, feerate=local_feerate_per_kw, is_local_initiator=True), htlcs=[]) self.sign_and_insert_remote_sig(our_commit_tx, remote_funding_pubkey, remote_signature, local_funding_pubkey, local_funding_privkey) ref_commit_tx_str = '02000000000101bef67e4e2fb9ddeeb3461973cd4c62abb35050b1add772995b820b584a488489000000000038b02b8002c0c62d0000000000160014ccf1af2f2aabee14bb40fa3851ab2301de84311054a56a00000000002200204adb4e2f00643db396dd120d4e7dc17625f5f2c11a40d857accc862d6b7dd80e0400473044022051b75c73198c6deee1a875871c3961832909acd297c6b908d59e3319e5185a46022055c419379c5051a78d00dbbce11b5b664a0c22815fbcc6fcef6b1937c383693901483045022100f51d2e566a70ba740fc5d8c0f07b9b93d2ed741c3c0860c613173de7d39e7968022041376d520e9c0e1ad52248ddf4b22e12be8763007df977253ef45a4ca3bdb7c001475221023da092f6980e58d2c037173180e9a465476026ee50f96695963e8efe436f54eb21030e9f7b623d2ccc7c9bd44d66d5ce21ce504c0acf6385a132cec6d3c39fa711c152ae3e195220' self.assertEqual(str(our_commit_tx), ref_commit_tx_str) From 8ad6d5dddadda3df05da06a6a5634a8ebc5a8493 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Mon, 30 Mar 2020 02:28:50 +0200 Subject: [PATCH 20/69] lnchannel: clean-up docstrings a bit Removed lnd copyright as by now everything covered in this file has been rewritten. --- electrum/lnchannel.py | 93 +++++++++++++------------------- electrum/lnutil.py | 2 +- electrum/tests/test_lnchannel.py | 3 ++ 3 files changed, 42 insertions(+), 56 deletions(-) diff --git a/electrum/lnchannel.py b/electrum/lnchannel.py index 6a0046f00..35f174292 100644 --- a/electrum/lnchannel.py +++ b/electrum/lnchannel.py @@ -1,5 +1,4 @@ # Copyright (C) 2018 The Electrum developers -# Copyright (C) 2015-2018 The Lightning Network Developers # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -19,9 +18,6 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. -# API (method signatures and docstrings) partially copied from lnd -# 42de4400bff5105352d0552155f73589166d162b - import os from collections import namedtuple, defaultdict import binascii @@ -499,11 +495,8 @@ def get_funding_address(self): return redeem_script_to_address('p2wsh', script) def add_htlc(self, htlc: UpdateAddHtlc) -> UpdateAddHtlc: - """ - AddHTLC adds an HTLC to the state machine's local update log. This method - should be called when preparing to send an outgoing HTLC. - - This docstring is from LND. + """Adds a new LOCAL HTLC to the channel. + Action must be initiated by LOCAL. """ if isinstance(htlc, dict): # legacy conversion # FIXME remove htlc = UpdateAddHtlc(**htlc) @@ -517,12 +510,8 @@ def add_htlc(self, htlc: UpdateAddHtlc) -> UpdateAddHtlc: return htlc def receive_htlc(self, htlc: UpdateAddHtlc, onion_packet:bytes = None) -> UpdateAddHtlc: - """ - ReceiveHTLC adds an HTLC to the state machine's remote update log. This - method should be called in response to receiving a new HTLC from the remote - party. - - This docstring is from LND. + """Adds a new REMOTE HTLC to the channel. + Action must be initiated by REMOTE. """ if isinstance(htlc, dict): # legacy conversion # FIXME remove htlc = UpdateAddHtlc(**htlc) @@ -543,17 +532,10 @@ def receive_htlc(self, htlc: UpdateAddHtlc, onion_packet:bytes = None) -> Update self.logger.info("receive_htlc") return htlc - def sign_next_commitment(self): - """ - SignNextCommitment signs a new commitment which includes any previous - unsettled HTLCs, any new HTLCs, and any modifications to prior HTLCs - committed in previous commitment updates. - The first return parameter is the signature for the commitment transaction - itself, while the second parameter is are all HTLC signatures concatenated. - any). The HTLC signatures are sorted according to the BIP 69 order of the - HTLC's on the commitment transaction. - - This docstring was adapted from LND. + def sign_next_commitment(self) -> Tuple[bytes, Sequence[bytes]]: + """Returns signatures for our next remote commitment tx. + Action must be initiated by LOCAL. + Finally, the next remote ctx becomes the latest remote ctx. """ next_remote_ctn = self.get_next_ctn(REMOTE) self.logger.info(f"sign_next_commitment {next_remote_ctn}") @@ -589,18 +571,10 @@ def sign_next_commitment(self): self.hm.send_ctx() return sig_64, htlcsigs - def receive_new_commitment(self, sig, htlc_sigs): - """ - ReceiveNewCommitment process a signature for a new commitment state sent by - the remote party. This method should be called in response to the - remote party initiating a new change, or when the remote party sends a - signature fully accepting a new state we've initiated. If we are able to - successfully validate the signature, then the generated commitment is added - to our local commitment chain. Once we send a revocation for our prior - state, then this newly added commitment becomes our current accepted channel - state. - - This docstring is from LND. + def receive_new_commitment(self, sig: bytes, htlc_sigs: Sequence[bytes]) -> None: + """Processes signatures for our next local commitment tx, sent by the REMOTE. + Action must be initiated by REMOTE. + If all checks pass, the next local ctx becomes the latest local ctx. """ # TODO in many failure cases below, we should "fail" the channel (force-close) next_local_ctn = self.get_next_ctn(LOCAL) @@ -627,19 +601,19 @@ def receive_new_commitment(self, sig, htlc_sigs): raise Exception(f'htlc sigs failure. recv {len(htlc_sigs)} sigs, expected {len(htlc_to_ctx_output_idx_map)}') for (direction, htlc), (ctx_output_idx, htlc_relative_idx) in htlc_to_ctx_output_idx_map.items(): htlc_sig = htlc_sigs[htlc_relative_idx] - self.verify_htlc(htlc=htlc, - htlc_sig=htlc_sig, - htlc_direction=direction, - pcp=pcp, - ctx=pending_local_commitment, - ctx_output_idx=ctx_output_idx) + self._verify_htlc_sig(htlc=htlc, + htlc_sig=htlc_sig, + htlc_direction=direction, + pcp=pcp, + ctx=pending_local_commitment, + ctx_output_idx=ctx_output_idx) with self.db_lock: self.hm.recv_ctx() self.config[LOCAL].current_commitment_signature=sig self.config[LOCAL].current_htlc_signatures=htlc_sigs_string - def verify_htlc(self, *, htlc: UpdateAddHtlc, htlc_sig: bytes, htlc_direction: Direction, - pcp: bytes, ctx: Transaction, ctx_output_idx: int) -> None: + def _verify_htlc_sig(self, *, htlc: UpdateAddHtlc, htlc_sig: bytes, htlc_direction: Direction, + pcp: bytes, ctx: Transaction, ctx_output_idx: int) -> None: _script, htlc_tx = make_htlc_tx_with_open_channel(chan=self, pcp=pcp, subject=LOCAL, @@ -786,8 +760,8 @@ def available_to_spend(self, subject: HTLCOwner, *, strict: bool = True) -> int: return max_send_msat def included_htlcs(self, subject: HTLCOwner, direction: Direction, ctn: int = None) -> Sequence[UpdateAddHtlc]: - """ - return filter of non-dust htlcs for subjects commitment transaction, initiated by given party + """Returns list of non-dust HTLCs for subject's commitment tx at ctn, + filtered by direction (of HTLCs). """ assert type(subject) is HTLCOwner assert type(direction) is Direction @@ -859,14 +833,14 @@ def get_latest_ctn(self, subject: HTLCOwner) -> int: def get_next_ctn(self, subject: HTLCOwner) -> int: return self.hm.ctn_latest(subject) + 1 - def total_msat(self, direction): + def total_msat(self, direction: Direction) -> int: """Return the cumulative total msat amount received/sent so far.""" assert type(direction) is Direction return htlcsum(self.hm.all_settled_htlcs_ever_by_direction(LOCAL, direction)) - def settle_htlc(self, preimage, htlc_id): - """ - SettleHTLC attempts to settle an existing outstanding received HTLC. + def settle_htlc(self, preimage: bytes, htlc_id: int) -> None: + """Settle/fulfill a pending received HTLC. + Action must be initiated by LOCAL. """ self.logger.info("settle_htlc") assert self.can_send_ctx_updates(), f"cannot update channel. {self.get_state()!r} {self.peer_state!r}" @@ -889,7 +863,10 @@ def decode_onion_error(self, reason: bytes, route: Sequence['RouteEdge'], self.onion_keys[htlc_id]) return failure_msg, sender_idx - def receive_htlc_settle(self, preimage, htlc_id): + def receive_htlc_settle(self, preimage: bytes, htlc_id: int) -> None: + """Settle/fulfill a pending offered HTLC. + Action must be initiated by REMOTE. + """ self.logger.info("receive_htlc_settle") log = self.hm.log[LOCAL] htlc = log['adds'][htlc_id] @@ -898,7 +875,10 @@ def receive_htlc_settle(self, preimage, htlc_id): with self.db_lock: self.hm.recv_settle(htlc_id) - def fail_htlc(self, htlc_id): + def fail_htlc(self, htlc_id: int) -> None: + """Fail a pending received HTLC. + Action must be initiated by LOCAL. + """ self.logger.info("fail_htlc") assert self.can_send_ctx_updates(), f"cannot update channel. {self.get_state()!r} {self.peer_state!r}" with self.db_lock: @@ -906,7 +886,10 @@ def fail_htlc(self, htlc_id): def receive_fail_htlc(self, htlc_id: int, *, error_bytes: Optional[bytes], - reason: Optional[OnionRoutingFailureMessage] = None): + reason: Optional[OnionRoutingFailureMessage] = None) -> None: + """Fail a pending offered HTLC. + Action must be initiated by REMOTE. + """ self.logger.info("receive_fail_htlc") with self.db_lock: self.hm.recv_fail(htlc_id) diff --git a/electrum/lnutil.py b/electrum/lnutil.py index 1590a1558..16e4584c0 100644 --- a/electrum/lnutil.py +++ b/electrum/lnutil.py @@ -209,7 +209,7 @@ class PaymentFailure(UserFacingException): pass MAXIMUM_REMOTE_TO_SELF_DELAY_ACCEPTED = 2016 class RevocationStore: - """ Taken from LND, see license in lnchannel.py. """ + # closely based on code in lightningnetwork/lnd START_INDEX = 2 ** 48 - 1 diff --git a/electrum/tests/test_lnchannel.py b/electrum/tests/test_lnchannel.py index 29fa283af..ecadbf59c 100644 --- a/electrum/tests/test_lnchannel.py +++ b/electrum/tests/test_lnchannel.py @@ -18,6 +18,9 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. +# +# Many of these unit tests are heavily based on unit tests in lnd +# (around commit 42de4400bff5105352d0552155f73589166d162b). import unittest import os From db84de5493604a956c6d6492e7098e367b4b573f Mon Sep 17 00:00:00 2001 From: SomberNight Date: Mon, 30 Mar 2020 02:42:07 +0200 Subject: [PATCH 21/69] trivial: use "chunks()" for htlc_sigs in lnchannel --- electrum/lnchannel.py | 4 ++-- electrum/lnpeer.py | 2 +- electrum/tests/test_util.py | 3 +++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/electrum/lnchannel.py b/electrum/lnchannel.py index 35f174292..e4fa11176 100644 --- a/electrum/lnchannel.py +++ b/electrum/lnchannel.py @@ -32,7 +32,7 @@ from . import ecc from . import constants -from .util import bfh, bh2u +from .util import bfh, bh2u, chunks from .bitcoin import redeem_script_to_address from .crypto import sha256, sha256d from .transaction import Transaction, PartialTransaction @@ -628,7 +628,7 @@ def _verify_htlc_sig(self, *, htlc: UpdateAddHtlc, htlc_sig: bytes, htlc_directi def get_remote_htlc_sig_for_htlc(self, *, htlc_relative_idx: int) -> bytes: data = self.config[LOCAL].current_htlc_signatures - htlc_sigs = [data[i:i + 64] for i in range(0, len(data), 64)] + htlc_sigs = list(chunks(data, 64)) htlc_sig = htlc_sigs[htlc_relative_idx] remote_htlc_sig = ecc.der_sig_from_sig_string(htlc_sig) + b'\x01' return remote_htlc_sig diff --git a/electrum/lnpeer.py b/electrum/lnpeer.py index eb8f3f6f3..91853884d 100644 --- a/electrum/lnpeer.py +++ b/electrum/lnpeer.py @@ -1076,7 +1076,7 @@ def on_commitment_signed(self, chan: Channel, payload): if chan.hm.is_revack_pending(LOCAL): raise RemoteMisbehaving('received commitment_signed before we revoked previous ctx') data = payload["htlc_signature"] - htlc_sigs = [data[i:i+64] for i in range(0, len(data), 64)] + htlc_sigs = list(chunks(data, 64)) chan.receive_new_commitment(payload["signature"], htlc_sigs) self.send_revoke_and_ack(chan) diff --git a/electrum/tests/test_util.py b/electrum/tests/test_util.py index 618904af1..dd978a9a0 100644 --- a/electrum/tests/test_util.py +++ b/electrum/tests/test_util.py @@ -108,6 +108,9 @@ def test_is_hash256_str(self): def test_chunks(self): self.assertEqual([[1, 2], [3, 4], [5]], list(chunks([1, 2, 3, 4, 5], 2))) + self.assertEqual([], list(chunks(b'', 64))) + self.assertEqual([b'12', b'34', b'56'], + list(chunks(b'123456', 2))) with self.assertRaises(ValueError): list(chunks([1, 2, 3], 0)) From 79d57784c15dc4debf13c36c871df515db48ada0 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Mon, 30 Mar 2020 03:49:50 +0200 Subject: [PATCH 22/69] lnchannel: add more type hints --- electrum/lnchannel.py | 113 ++++++++++++++++--------------- electrum/lnutil.py | 30 +++++--- electrum/lnwatcher.py | 3 +- electrum/tests/test_lnchannel.py | 20 +++--- electrum/tests/test_lnutil.py | 97 ++++++++++++++++++-------- 5 files changed, 160 insertions(+), 103 deletions(-) diff --git a/electrum/lnchannel.py b/electrum/lnchannel.py index e4fa11176..d38bd4824 100644 --- a/electrum/lnchannel.py +++ b/electrum/lnchannel.py @@ -23,7 +23,8 @@ import binascii import json from enum import IntEnum -from typing import Optional, Dict, List, Tuple, NamedTuple, Set, Callable, Iterable, Sequence, TYPE_CHECKING, Iterator +from typing import (Optional, Dict, List, Tuple, NamedTuple, Set, Callable, + Iterable, Sequence, TYPE_CHECKING, Iterator, Union) import time import threading @@ -63,7 +64,7 @@ # lightning channel states # Note: these states are persisted by name (for a given channel) in the wallet file, # so consider doing a wallet db upgrade when changing them. -class channel_states(IntEnum): +class channel_states(IntEnum): # TODO rename to use CamelCase PREOPENING = 0 # Initial negotiation. Channel will not be reestablished OPENING = 1 # Channel will be reestablished. (per BOLT2) # - Funding node: has received funding_signed (can broadcast the funding tx) @@ -75,7 +76,7 @@ class channel_states(IntEnum): CLOSED = 6 # closing tx has been mined REDEEMED = 7 # we can stop watching -class peer_states(IntEnum): +class peer_states(IntEnum): # TODO rename to use CamelCase DISCONNECTED = 0 REESTABLISHING = 1 GOOD = 2 @@ -138,15 +139,15 @@ def __init__(self, state: 'StoredDict', *, sweep_address=None, name=None, lnwork self.sweep_address = sweep_address self.storage = state self.db_lock = self.storage.db.lock if self.storage.db else threading.RLock() - self.config = {} # type: Dict[HTLCOwner, lnutil.Config] + self.config = {} # type: Dict[HTLCOwner, Union[LocalConfig, RemoteConfig]] self.config[LOCAL] = state["local_config"] self.config[REMOTE] = state["remote_config"] self.channel_id = bfh(state["channel_id"]) - self.constraints = state["constraints"] - self.funding_outpoint = state["funding_outpoint"] + self.constraints = state["constraints"] # type: ChannelConstraints + self.funding_outpoint = state["funding_outpoint"] # type: Outpoint self.node_id = bfh(state["node_id"]) self.short_channel_id = ShortChannelID.normalize(state["short_channel_id"]) - self.onion_keys = state['onion_keys'] + self.onion_keys = state['onion_keys'] # type: Dict[int, bytes] self.data_loss_protect_remote_pcp = state['data_loss_protect_remote_pcp'] self.hm = HTLCManager(log=state['log'], initial_feerate=initial_feerate) self._state = channel_states[state['state']] @@ -165,10 +166,10 @@ def get_id_for_log(self) -> str: return str(scid) return self.channel_id.hex() - def set_onion_key(self, key, value): + def set_onion_key(self, key: int, value: bytes): self.onion_keys[key] = value - def get_onion_key(self, key): + def get_onion_key(self, key: int) -> bytes: return self.onion_keys.get(key) def set_data_loss_protect_remote_pcp(self, key, value): @@ -262,23 +263,24 @@ def construct_channel_announcement_without_sigs(self) -> bytes: self._chan_ann_without_sigs = chan_ann return chan_ann - def is_static_remotekey_enabled(self): - return self.storage.get('static_remotekey_enabled') + def is_static_remotekey_enabled(self) -> bool: + return bool(self.storage.get('static_remotekey_enabled')) - def set_short_channel_id(self, short_id): + def set_short_channel_id(self, short_id: ShortChannelID) -> None: self.short_channel_id = short_id self.storage["short_channel_id"] = short_id - def get_feerate(self, subject, ctn): + def get_feerate(self, subject: HTLCOwner, *, ctn: int) -> int: + # returns feerate in sat/kw return self.hm.get_feerate(subject, ctn) - def get_oldest_unrevoked_feerate(self, subject): + def get_oldest_unrevoked_feerate(self, subject: HTLCOwner) -> int: return self.hm.get_feerate_in_oldest_unrevoked_ctx(subject) - def get_latest_feerate(self, subject): + def get_latest_feerate(self, subject: HTLCOwner) -> int: return self.hm.get_feerate_in_latest_ctx(subject) - def get_next_feerate(self, subject): + def get_next_feerate(self, subject: HTLCOwner) -> int: return self.hm.get_feerate_in_next_ctx(subject) def get_payments(self): @@ -316,7 +318,7 @@ def open_with_first_pcp(self, remote_pcp: bytes, remote_sig: bytes) -> None: self.hm.channel_open_finished() self.peer_state = peer_states.GOOD - def set_state(self, state): + def set_state(self, state: channel_states) -> None: """ set on-chain state """ old_state = self._state if (old_state, state) not in state_transitions: @@ -329,7 +331,7 @@ def set_state(self, state): self.lnworker.save_channel(self) self.lnworker.network.trigger_callback('channel', self) - def get_state(self): + def get_state(self) -> channel_states: return self._state def get_state_for_GUI(self): @@ -767,7 +769,7 @@ def included_htlcs(self, subject: HTLCOwner, direction: Direction, ctn: int = No assert type(direction) is Direction if ctn is None: ctn = self.get_oldest_unrevoked_ctn(subject) - feerate = self.get_feerate(subject, ctn) + feerate = self.get_feerate(subject, ctn=ctn) conf = self.config[subject] if direction == RECEIVED: threshold_sat = received_htlc_trim_threshold_sat(dust_limit_sat=conf.dust_limit_sat, feerate=feerate) @@ -798,30 +800,30 @@ def get_secret_and_point(self, subject: HTLCOwner, ctn: int) -> Tuple[Optional[b point = secret_to_pubkey(int.from_bytes(secret, 'big')) return secret, point - def get_secret_and_commitment(self, subject, ctn): + def get_secret_and_commitment(self, subject: HTLCOwner, *, ctn: int) -> Tuple[Optional[bytes], PartialTransaction]: secret, point = self.get_secret_and_point(subject, ctn) ctx = self.make_commitment(subject, point, ctn) return secret, ctx - def get_commitment(self, subject, ctn) -> PartialTransaction: - secret, ctx = self.get_secret_and_commitment(subject, ctn) + def get_commitment(self, subject: HTLCOwner, *, ctn: int) -> PartialTransaction: + secret, ctx = self.get_secret_and_commitment(subject, ctn=ctn) return ctx def get_next_commitment(self, subject: HTLCOwner) -> PartialTransaction: ctn = self.get_next_ctn(subject) - return self.get_commitment(subject, ctn) + return self.get_commitment(subject, ctn=ctn) def get_latest_commitment(self, subject: HTLCOwner) -> PartialTransaction: ctn = self.get_latest_ctn(subject) - return self.get_commitment(subject, ctn) + return self.get_commitment(subject, ctn=ctn) def get_oldest_unrevoked_commitment(self, subject: HTLCOwner) -> PartialTransaction: ctn = self.get_oldest_unrevoked_ctn(subject) - return self.get_commitment(subject, ctn) + return self.get_commitment(subject, ctn=ctn) def create_sweeptxs(self, ctn: int) -> List[Transaction]: from .lnsweep import create_sweeptxs_for_watchtower - secret, ctx = self.get_secret_and_commitment(REMOTE, ctn) + secret, ctx = self.get_secret_and_commitment(REMOTE, ctn=ctn) return create_sweeptxs_for_watchtower(self, ctx, secret, self.sweep_address) def get_oldest_unrevoked_ctn(self, subject: HTLCOwner) -> int: @@ -850,9 +852,9 @@ def settle_htlc(self, preimage: bytes, htlc_id: int) -> None: assert htlc_id not in log['settles'] self.hm.send_settle(htlc_id) - def get_payment_hash(self, htlc_id): + def get_payment_hash(self, htlc_id: int) -> bytes: log = self.hm.log[LOCAL] - htlc = log['adds'][htlc_id] + htlc = log['adds'][htlc_id] # type: UpdateAddHtlc return htlc.payment_hash def decode_onion_error(self, reason: bytes, route: Sequence['RouteEdge'], @@ -898,13 +900,13 @@ def receive_fail_htlc(self, htlc_id: int, *, error_bytes=error_bytes, error_reason=reason) - def pending_local_fee(self): - return self.constraints.capacity - sum(x.value for x in self.get_next_commitment(LOCAL).outputs()) + def get_next_fee(self, subject: HTLCOwner) -> int: + return self.constraints.capacity - sum(x.value for x in self.get_next_commitment(subject).outputs()) - def get_latest_fee(self, subject): + def get_latest_fee(self, subject: HTLCOwner) -> int: return self.constraints.capacity - sum(x.value for x in self.get_latest_commitment(subject).outputs()) - def update_fee(self, feerate: int, from_us: bool): + def update_fee(self, feerate: int, from_us: bool) -> None: # feerate uses sat/kw if self.constraints.is_initiator != from_us: raise Exception(f"Cannot update_fee: wrong initiator. us: {from_us}") @@ -917,9 +919,9 @@ def update_fee(self, feerate: int, from_us: bool): else: self.hm.recv_update_fee(feerate) - def make_commitment(self, subject, this_point, ctn) -> PartialTransaction: + def make_commitment(self, subject: HTLCOwner, this_point: bytes, ctn: int) -> PartialTransaction: assert type(subject) is HTLCOwner - feerate = self.get_feerate(subject, ctn) + feerate = self.get_feerate(subject, ctn=ctn) other = subject.inverted() local_msat = self.balance(subject, ctx_owner=subject, ctn=ctn) remote_msat = self.balance(other, ctx_owner=subject, ctn=ctn) @@ -969,23 +971,24 @@ def make_commitment(self, subject, this_point, ctn) -> PartialTransaction: payment_pubkey = derive_pubkey(other_config.payment_basepoint.pubkey, this_point) return make_commitment( - ctn, - this_config.multisig_key.pubkey, - other_config.multisig_key.pubkey, - payment_pubkey, - self.config[LOCAL if self.constraints.is_initiator else REMOTE].payment_basepoint.pubkey, - self.config[LOCAL if not self.constraints.is_initiator else REMOTE].payment_basepoint.pubkey, - other_revocation_pubkey, - derive_pubkey(this_config.delayed_basepoint.pubkey, this_point), - other_config.to_self_delay, - self.funding_outpoint.txid, - self.funding_outpoint.output_index, - self.constraints.capacity, - local_msat, - remote_msat, - this_config.dust_limit_sat, - onchain_fees, - htlcs=htlcs) + ctn=ctn, + local_funding_pubkey=this_config.multisig_key.pubkey, + remote_funding_pubkey=other_config.multisig_key.pubkey, + remote_payment_pubkey=payment_pubkey, + funder_payment_basepoint=self.config[LOCAL if self.constraints.is_initiator else REMOTE].payment_basepoint.pubkey, + fundee_payment_basepoint=self.config[LOCAL if not self.constraints.is_initiator else REMOTE].payment_basepoint.pubkey, + revocation_pubkey=other_revocation_pubkey, + delayed_pubkey=derive_pubkey(this_config.delayed_basepoint.pubkey, this_point), + to_self_delay=other_config.to_self_delay, + funding_txid=self.funding_outpoint.txid, + funding_pos=self.funding_outpoint.output_index, + funding_sat=self.constraints.capacity, + local_amount=local_msat, + remote_amount=remote_msat, + dust_limit_sat=this_config.dust_limit_sat, + fees_per_participant=onchain_fees, + htlcs=htlcs, + ) def make_closing_tx(self, local_script: bytes, remote_script: bytes, fee_sat: int, *, drop_remote = False) -> Tuple[bytes, PartialTransaction]: @@ -1013,7 +1016,7 @@ def make_closing_tx(self, local_script: bytes, remote_script: bytes, sig = ecc.sig_string_from_der_sig(der_sig[:-1]) return sig, closing_tx - def signature_fits(self, tx: PartialTransaction): + def signature_fits(self, tx: PartialTransaction) -> bool: remote_sig = self.config[LOCAL].current_commitment_signature preimage_hex = tx.serialize_preimage(0) msg_hash = sha256d(bfh(preimage_hex)) @@ -1021,7 +1024,7 @@ def signature_fits(self, tx: PartialTransaction): res = ecc.verify_signature(self.config[REMOTE].multisig_key.pubkey, remote_sig, msg_hash) return res - def force_close_tx(self): + def force_close_tx(self) -> PartialTransaction: tx = self.get_latest_commitment(LOCAL) assert self.signature_fits(tx) tx.sign({bh2u(self.config[LOCAL].multisig_key.pubkey): (self.config[LOCAL].multisig_key.privkey, True)}) @@ -1048,11 +1051,11 @@ def sweep_ctx(self, ctx: Transaction) -> Dict[str, SweepInfo]: self.sweep_info[txid] = {} return self.sweep_info[txid] - def sweep_htlc(self, ctx:Transaction, htlc_tx: Transaction): + def sweep_htlc(self, ctx: Transaction, htlc_tx: Transaction) -> Optional[SweepInfo]: # look at the output address, check if it matches return create_sweeptx_for_their_revoked_htlc(self, ctx, htlc_tx, self.sweep_address) - def has_pending_changes(self, subject): + def has_pending_changes(self, subject: HTLCOwner) -> bool: next_htlcs = self.hm.get_htlcs_in_next_ctx(subject) latest_htlcs = self.hm.get_htlcs_in_latest_ctx(subject) return not (next_htlcs == latest_htlcs and self.get_next_feerate(subject) == self.get_latest_feerate(subject)) diff --git a/electrum/lnutil.py b/electrum/lnutil.py index 16e4584c0..1ae216919 100644 --- a/electrum/lnutil.py +++ b/electrum/lnutil.py @@ -599,13 +599,27 @@ def calc_fees_for_commitment_tx(*, num_htlcs: int, feerate: int, REMOTE: fee if not is_local_initiator else 0, } -def make_commitment(ctn, local_funding_pubkey, remote_funding_pubkey, - remote_payment_pubkey, funder_payment_basepoint, - fundee_payment_basepoint, revocation_pubkey, - delayed_pubkey, to_self_delay, funding_txid, - funding_pos, funding_sat, local_amount, remote_amount, - dust_limit_sat, fees_per_participant, - htlcs: List[ScriptHtlc]) -> PartialTransaction: + +def make_commitment( + *, + ctn: int, + local_funding_pubkey: bytes, + remote_funding_pubkey: bytes, + remote_payment_pubkey: bytes, + funder_payment_basepoint: bytes, + fundee_payment_basepoint: bytes, + revocation_pubkey: bytes, + delayed_pubkey: bytes, + to_self_delay: int, + funding_txid: str, + funding_pos: int, + funding_sat: int, + local_amount: int, + remote_amount: int, + dust_limit_sat: int, + fees_per_participant: Mapping[HTLCOwner, int], + htlcs: List[ScriptHtlc] +) -> PartialTransaction: c_input = make_funding_input(local_funding_pubkey, remote_funding_pubkey, funding_pos, funding_txid, funding_sat) obs = get_obscured_ctn(ctn, funder_payment_basepoint, fundee_payment_basepoint) @@ -618,7 +632,7 @@ def make_commitment(ctn, local_funding_pubkey, remote_funding_pubkey, # commitment tx outputs local_address = make_commitment_output_to_local_address(revocation_pubkey, to_self_delay, delayed_pubkey) remote_address = make_commitment_output_to_remote_address(remote_payment_pubkey) - # TODO trim htlc outputs here while also considering 2nd stage htlc transactions + # note: it is assumed that the given 'htlcs' are all non-dust (dust htlcs already trimmed) # BOLT-03: "Transaction Input and Output Ordering # Lexicographic ordering: see BIP69. In the case of identical HTLC outputs, diff --git a/electrum/lnwatcher.py b/electrum/lnwatcher.py index faf27a8c9..c7c0f0f44 100644 --- a/electrum/lnwatcher.py +++ b/electrum/lnwatcher.py @@ -23,6 +23,7 @@ if TYPE_CHECKING: from .network import Network from .lnsweep import SweepInfo + from .lnworker import LNWallet class ListenerItem(NamedTuple): # this is triggered when the lnwatcher is all done with the outpoint used as index in LNWatcher.tx_progress @@ -332,7 +333,7 @@ async def update_channel_state(self, *args): class LNWalletWatcher(LNWatcher): - def __init__(self, lnworker, network): + def __init__(self, lnworker: 'LNWallet', network: 'Network'): LNWatcher.__init__(self, network) self.network = network self.lnworker = lnworker diff --git a/electrum/tests/test_lnchannel.py b/electrum/tests/test_lnchannel.py index ecadbf59c..90f0704cc 100644 --- a/electrum/tests/test_lnchannel.py +++ b/electrum/tests/test_lnchannel.py @@ -786,15 +786,15 @@ def part3(self): alice_idx = self.alice_channel.add_htlc(htlc_dict).htlc_id bob_idx = self.bob_channel.receive_htlc(htlc_dict).htlc_id force_state_transition(self.alice_channel, self.bob_channel) - self.check_bals(one_bitcoin_in_msat*3\ - - self.alice_channel.pending_local_fee(), - one_bitcoin_in_msat*5) + self.check_bals(one_bitcoin_in_msat * 3 + - self.alice_channel.get_next_fee(LOCAL), + one_bitcoin_in_msat * 5) self.bob_channel.settle_htlc(paymentPreimage, bob_idx) self.alice_channel.receive_htlc_settle(paymentPreimage, alice_idx) force_state_transition(self.alice_channel, self.bob_channel) - self.check_bals(one_bitcoin_in_msat*3\ - - self.alice_channel.pending_local_fee(), - one_bitcoin_in_msat*7) + self.check_bals(one_bitcoin_in_msat * 3 + - self.alice_channel.get_next_fee(LOCAL), + one_bitcoin_in_msat * 7) # And now let Bob add an HTLC of 1 BTC. This will take Bob's balance # all the way down to his channel reserve, but since he is not paying # the fee this is okay. @@ -802,9 +802,9 @@ def part3(self): self.bob_channel.add_htlc(htlc_dict) self.alice_channel.receive_htlc(htlc_dict) force_state_transition(self.alice_channel, self.bob_channel) - self.check_bals(one_bitcoin_in_msat*3\ - - self.alice_channel.pending_local_fee(), - one_bitcoin_in_msat*6) + self.check_bals(one_bitcoin_in_msat * 3 \ + - self.alice_channel.get_next_fee(LOCAL), + one_bitcoin_in_msat * 6) def check_bals(self, amt1, amt2): self.assertEqual(self.alice_channel.available_to_spend(LOCAL), amt1) @@ -840,7 +840,7 @@ def test_DustLimit(self): self.assertEqual(len(alice_ctx.outputs()), 3) self.assertEqual(len(bob_ctx.outputs()), 2) default_fee = calc_static_fee(0) - self.assertEqual(bob_channel.pending_local_fee(), default_fee + htlcAmt) + self.assertEqual(bob_channel.get_next_fee(LOCAL), default_fee + htlcAmt) bob_channel.settle_htlc(paymentPreimage, bobHtlcIndex) alice_channel.receive_htlc_settle(paymentPreimage, aliceHtlcIndex) force_state_transition(bob_channel, alice_channel) diff --git a/electrum/tests/test_lnutil.py b/electrum/tests/test_lnutil.py index e5297a798..885386684 100644 --- a/electrum/tests/test_lnutil.py +++ b/electrum/tests/test_lnutil.py @@ -510,13 +510,23 @@ def test_commitment_tx_with_all_five_HTLCs_untrimmed_minimum_feerate(self): htlcs = [ScriptHtlc(htlc[x], htlc_obj[x]) for x in range(5)] our_commit_tx = make_commitment( - commitment_number, - local_funding_pubkey, remote_funding_pubkey, remotepubkey, - local_payment_basepoint, remote_payment_basepoint, - local_revocation_pubkey, local_delayedpubkey, local_delay, - funding_tx_id, funding_output_index, funding_amount_satoshi, - to_local_msat, to_remote_msat, local_dust_limit_satoshi, - calc_fees_for_commitment_tx(num_htlcs=len(htlcs), feerate=local_feerate_per_kw, is_local_initiator=True), htlcs=htlcs) + ctn=commitment_number, + local_funding_pubkey=local_funding_pubkey, + remote_funding_pubkey=remote_funding_pubkey, + remote_payment_pubkey=remotepubkey, + funder_payment_basepoint=local_payment_basepoint, + fundee_payment_basepoint=remote_payment_basepoint, + revocation_pubkey=local_revocation_pubkey, + delayed_pubkey=local_delayedpubkey, + to_self_delay=local_delay, + funding_txid=funding_tx_id, + funding_pos=funding_output_index, + funding_sat=funding_amount_satoshi, + local_amount=to_local_msat, + remote_amount=to_remote_msat, + dust_limit_sat=local_dust_limit_satoshi, + fees_per_participant=calc_fees_for_commitment_tx(num_htlcs=len(htlcs), feerate=local_feerate_per_kw, is_local_initiator=True), + htlcs=htlcs) self.sign_and_insert_remote_sig(our_commit_tx, remote_funding_pubkey, remote_signature, local_funding_pubkey, local_funding_privkey) self.assertEqual(str(our_commit_tx), output_commit_tx) @@ -587,13 +597,23 @@ def test_commitment_tx_with_one_output(self): output_commit_tx= "02000000000101bef67e4e2fb9ddeeb3461973cd4c62abb35050b1add772995b820b584a488489000000000038b02b8001c0c62d0000000000160014ccf1af2f2aabee14bb40fa3851ab2301de8431100400473044022031a82b51bd014915fe68928d1abf4b9885353fb896cac10c3fdd88d7f9c7f2e00220716bda819641d2c63e65d3549b6120112e1aeaf1742eed94a471488e79e206b101473044022064901950be922e62cbe3f2ab93de2b99f37cff9fc473e73e394b27f88ef0731d02206d1dfa227527b4df44a07599289e207d6fd9cca60c0365682dcd3deaf739567e01475221023da092f6980e58d2c037173180e9a465476026ee50f96695963e8efe436f54eb21030e9f7b623d2ccc7c9bd44d66d5ce21ce504c0acf6385a132cec6d3c39fa711c152ae3e195220" our_commit_tx = make_commitment( - commitment_number, - local_funding_pubkey, remote_funding_pubkey, remotepubkey, - local_payment_basepoint, remote_payment_basepoint, - local_revocation_pubkey, local_delayedpubkey, local_delay, - funding_tx_id, funding_output_index, funding_amount_satoshi, - to_local_msat, to_remote_msat, local_dust_limit_satoshi, - calc_fees_for_commitment_tx(num_htlcs=0, feerate=local_feerate_per_kw, is_local_initiator=True), htlcs=[]) + ctn=commitment_number, + local_funding_pubkey=local_funding_pubkey, + remote_funding_pubkey=remote_funding_pubkey, + remote_payment_pubkey=remotepubkey, + funder_payment_basepoint=local_payment_basepoint, + fundee_payment_basepoint=remote_payment_basepoint, + revocation_pubkey=local_revocation_pubkey, + delayed_pubkey=local_delayedpubkey, + to_self_delay=local_delay, + funding_txid=funding_tx_id, + funding_pos=funding_output_index, + funding_sat=funding_amount_satoshi, + local_amount=to_local_msat, + remote_amount=to_remote_msat, + dust_limit_sat=local_dust_limit_satoshi, + fees_per_participant=calc_fees_for_commitment_tx(num_htlcs=0, feerate=local_feerate_per_kw, is_local_initiator=True), + htlcs=[]) self.sign_and_insert_remote_sig(our_commit_tx, remote_funding_pubkey, remote_signature, local_funding_pubkey, local_funding_privkey) self.assertEqual(str(our_commit_tx), output_commit_tx) @@ -606,13 +626,23 @@ def test_commitment_tx_with_fee_greater_than_funder_amount(self): output_commit_tx= "02000000000101bef67e4e2fb9ddeeb3461973cd4c62abb35050b1add772995b820b584a488489000000000038b02b8001c0c62d0000000000160014ccf1af2f2aabee14bb40fa3851ab2301de8431100400473044022031a82b51bd014915fe68928d1abf4b9885353fb896cac10c3fdd88d7f9c7f2e00220716bda819641d2c63e65d3549b6120112e1aeaf1742eed94a471488e79e206b101473044022064901950be922e62cbe3f2ab93de2b99f37cff9fc473e73e394b27f88ef0731d02206d1dfa227527b4df44a07599289e207d6fd9cca60c0365682dcd3deaf739567e01475221023da092f6980e58d2c037173180e9a465476026ee50f96695963e8efe436f54eb21030e9f7b623d2ccc7c9bd44d66d5ce21ce504c0acf6385a132cec6d3c39fa711c152ae3e195220" our_commit_tx = make_commitment( - commitment_number, - local_funding_pubkey, remote_funding_pubkey, remotepubkey, - local_payment_basepoint, remote_payment_basepoint, - local_revocation_pubkey, local_delayedpubkey, local_delay, - funding_tx_id, funding_output_index, funding_amount_satoshi, - to_local_msat, to_remote_msat, local_dust_limit_satoshi, - calc_fees_for_commitment_tx(num_htlcs=0, feerate=local_feerate_per_kw, is_local_initiator=True), htlcs=[]) + ctn=commitment_number, + local_funding_pubkey=local_funding_pubkey, + remote_funding_pubkey=remote_funding_pubkey, + remote_payment_pubkey=remotepubkey, + funder_payment_basepoint=local_payment_basepoint, + fundee_payment_basepoint=remote_payment_basepoint, + revocation_pubkey=local_revocation_pubkey, + delayed_pubkey=local_delayedpubkey, + to_self_delay=local_delay, + funding_txid=funding_tx_id, + funding_pos=funding_output_index, + funding_sat=funding_amount_satoshi, + local_amount=to_local_msat, + remote_amount=to_remote_msat, + dust_limit_sat=local_dust_limit_satoshi, + fees_per_participant=calc_fees_for_commitment_tx(num_htlcs=0, feerate=local_feerate_per_kw, is_local_initiator=True), + htlcs=[]) self.sign_and_insert_remote_sig(our_commit_tx, remote_funding_pubkey, remote_signature, local_funding_pubkey, local_funding_privkey) self.assertEqual(str(our_commit_tx), output_commit_tx) @@ -662,15 +692,24 @@ def test_simple_commitment_tx_with_no_HTLCs(self): # to_remote amount 3000000 P2WPKH(0394854aa6eab5b2a8122cc726e9dded053a2184d88256816826d6231c068d4a5b) remote_signature = "3045022100f51d2e566a70ba740fc5d8c0f07b9b93d2ed741c3c0860c613173de7d39e7968022041376d520e9c0e1ad52248ddf4b22e12be8763007df977253ef45a4ca3bdb7c0" # local_signature = 3044022051b75c73198c6deee1a875871c3961832909acd297c6b908d59e3319e5185a46022055c419379c5051a78d00dbbce11b5b664a0c22815fbcc6fcef6b1937c3836939 - htlcs=[] our_commit_tx = make_commitment( - commitment_number, - local_funding_pubkey, remote_funding_pubkey, remotepubkey, - local_payment_basepoint, remote_payment_basepoint, - local_revocation_pubkey, local_delayedpubkey, local_delay, - funding_tx_id, funding_output_index, funding_amount_satoshi, - to_local_msat, to_remote_msat, local_dust_limit_satoshi, - calc_fees_for_commitment_tx(num_htlcs=0, feerate=local_feerate_per_kw, is_local_initiator=True), htlcs=[]) + ctn=commitment_number, + local_funding_pubkey=local_funding_pubkey, + remote_funding_pubkey=remote_funding_pubkey, + remote_payment_pubkey=remotepubkey, + funder_payment_basepoint=local_payment_basepoint, + fundee_payment_basepoint=remote_payment_basepoint, + revocation_pubkey=local_revocation_pubkey, + delayed_pubkey=local_delayedpubkey, + to_self_delay=local_delay, + funding_txid=funding_tx_id, + funding_pos=funding_output_index, + funding_sat=funding_amount_satoshi, + local_amount=to_local_msat, + remote_amount=to_remote_msat, + dust_limit_sat=local_dust_limit_satoshi, + fees_per_participant=calc_fees_for_commitment_tx(num_htlcs=0, feerate=local_feerate_per_kw, is_local_initiator=True), + htlcs=[]) self.sign_and_insert_remote_sig(our_commit_tx, remote_funding_pubkey, remote_signature, local_funding_pubkey, local_funding_privkey) ref_commit_tx_str = '02000000000101bef67e4e2fb9ddeeb3461973cd4c62abb35050b1add772995b820b584a488489000000000038b02b8002c0c62d0000000000160014ccf1af2f2aabee14bb40fa3851ab2301de84311054a56a00000000002200204adb4e2f00643db396dd120d4e7dc17625f5f2c11a40d857accc862d6b7dd80e0400473044022051b75c73198c6deee1a875871c3961832909acd297c6b908d59e3319e5185a46022055c419379c5051a78d00dbbce11b5b664a0c22815fbcc6fcef6b1937c383693901483045022100f51d2e566a70ba740fc5d8c0f07b9b93d2ed741c3c0860c613173de7d39e7968022041376d520e9c0e1ad52248ddf4b22e12be8763007df977253ef45a4ca3bdb7c001475221023da092f6980e58d2c037173180e9a465476026ee50f96695963e8efe436f54eb21030e9f7b623d2ccc7c9bd44d66d5ce21ce504c0acf6385a132cec6d3c39fa711c152ae3e195220' self.assertEqual(str(our_commit_tx), ref_commit_tx_str) From 72de433f5ca24f8e16f960fe1e0d6a0ece7c292c Mon Sep 17 00:00:00 2001 From: JeremyRand Date: Mon, 30 Mar 2020 22:50:25 +0000 Subject: [PATCH 23/69] Commands: clarify description of getservers The previous description made it sound like it returned the list of currently connected servers; this clarifies that it's only a list of candidate servers to connect to (no guarantee that they are all currently connected). --- electrum/commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/electrum/commands.py b/electrum/commands.py index 2cf1ab00d..d2b836296 100644 --- a/electrum/commands.py +++ b/electrum/commands.py @@ -467,7 +467,7 @@ async def getmerkle(self, txid, height): @command('n') async def getservers(self): - """Return the list of available servers""" + """Return the list of known servers (candidates for connecting).""" return self.network.get_servers() @command('') From 900a7631cf3588147857688cb5f8efe11c4b2924 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Tue, 31 Mar 2020 05:50:18 +0200 Subject: [PATCH 24/69] commands: add new cmd "getprivatekeyforpath" to export a WIF for a path related: #6061 --- electrum/address_synchronizer.py | 2 +- electrum/bitcoin.py | 4 ++-- electrum/commands.py | 7 +++++++ electrum/tests/test_commands.py | 14 ++++++++++++++ electrum/wallet.py | 16 +++++++++++++--- 5 files changed, 37 insertions(+), 6 deletions(-) diff --git a/electrum/address_synchronizer.py b/electrum/address_synchronizer.py index a8634b189..e0575d606 100644 --- a/electrum/address_synchronizer.py +++ b/electrum/address_synchronizer.py @@ -449,7 +449,7 @@ def get_history(self, *, domain=None) -> Sequence[HistoryItem]: domain = set(domain) # 1. Get the history of each address in the domain, maintain the # delta of a tx as the sum of its deltas on domain addresses - tx_deltas = defaultdict(int) + tx_deltas = defaultdict(int) # type: Dict[str, Optional[int]] for addr in domain: h = self.get_address_history(addr) for tx_hash, height in h: diff --git a/electrum/bitcoin.py b/electrum/bitcoin.py index 6316dcef1..16d2fda9c 100644 --- a/electrum/bitcoin.py +++ b/electrum/bitcoin.py @@ -565,8 +565,8 @@ def is_segwit_script_type(txin_type: str) -> bool: return txin_type in ('p2wpkh', 'p2wpkh-p2sh', 'p2wsh', 'p2wsh-p2sh') -def serialize_privkey(secret: bytes, compressed: bool, txin_type: str, - internal_use: bool=False) -> str: +def serialize_privkey(secret: bytes, compressed: bool, txin_type: str, *, + internal_use: bool = False) -> str: # we only export secrets inside curve range secret = ecc.ECPrivkey.normalize_secret_bytes(secret) if internal_use: diff --git a/electrum/commands.py b/electrum/commands.py index d2b836296..39b25f470 100644 --- a/electrum/commands.py +++ b/electrum/commands.py @@ -414,6 +414,13 @@ async def getprivatekeys(self, address, password=None, wallet: Abstract_Wallet = domain = address return [wallet.export_private_key(address, password) for address in domain] + @command('wp') + async def getprivatekeyforpath(self, path, password=None, wallet: Abstract_Wallet = None): + """Get private key corresponding to derivation path (address index). + 'path' can be either a str such as "m/0/50", or a list of ints such as [0, 50]. + """ + return wallet.export_private_key_for_path(path, password) + @command('w') async def ismine(self, address, wallet: Abstract_Wallet = None): """Check if address is in wallet. Return true if and only address is in wallet""" diff --git a/electrum/tests/test_commands.py b/electrum/tests/test_commands.py index f57b4a579..9f4330afb 100644 --- a/electrum/tests/test_commands.py +++ b/electrum/tests/test_commands.py @@ -180,3 +180,17 @@ def test_serialize(self): } self.assertEqual("0200000000010139c5375fe9da7bd377c1783002b129f8c57d3e724d62f5eacb9739ca691a229d0100000000feffffff01301b0f0000000000160014ac0e2d229200bffb2167ed6fd196aef9d687d8bb0247304402206367fb2ddd723985f5f51e0f2435084c0a66f5c26f4403a75d3dd417b71a20450220545dc3637bcb49beedbbdf5063e05cad63be91af4f839886451c30ecd6edf1d20121021f110909ded653828a254515b58498a6bafc96799fb0851554463ed44ca7d9da00000000", cmds._run('serialize', (jsontx,))) + + @mock.patch.object(wallet.Abstract_Wallet, 'save_db') + def test_getprivatekeyforpath(self, mock_save_db): + wallet = restore_wallet_from_text('north rent dawn bunker hamster invest wagon market romance pig either squeeze', + gap_limit=2, + path='if_this_exists_mocking_failed_648151893', + config=self.config)['wallet'] + cmds = Commands(config=self.config) + self.assertEqual("p2wpkh:cUzm7zPpWgLYeURgff4EsoMjhskCpsviBH4Y3aZcrBX8UJSRPjC2", + cmds._run('getprivatekeyforpath', ([0, 10000],), wallet=wallet)) + self.assertEqual("p2wpkh:cUzm7zPpWgLYeURgff4EsoMjhskCpsviBH4Y3aZcrBX8UJSRPjC2", + cmds._run('getprivatekeyforpath', ("m/0/10000",), wallet=wallet)) + self.assertEqual("p2wpkh:cQAj4WGf1socCPCJNMjXYCJ8Bs5JUAk5pbDr4ris44QdgAXcV24S", + cmds._run('getprivatekeyforpath', ("m/5h/100000/88h/7",), wallet=wallet)) diff --git a/electrum/wallet.py b/electrum/wallet.py index 497ce3d42..9a5d71d30 100644 --- a/electrum/wallet.py +++ b/electrum/wallet.py @@ -44,7 +44,7 @@ import itertools from .i18n import _ -from .bip32 import BIP32Node, convert_bip32_intpath_to_strpath +from .bip32 import BIP32Node, convert_bip32_intpath_to_strpath, convert_bip32_path_to_list_of_uint32 from .crypto import sha256 from .util import (NotEnoughFunds, UserCancelled, profiler, format_satoshis, format_fee_satoshis, NoDynamicFeeEstimates, @@ -462,7 +462,7 @@ def get_txin_type(self, address: str) -> str: """Return script type of wallet address.""" pass - def export_private_key(self, address, password) -> str: + def export_private_key(self, address: str, password: Optional[str]) -> str: if self.is_watching_only(): raise Exception(_("This is a watching-only wallet")) if not is_address(address): @@ -475,6 +475,9 @@ def export_private_key(self, address, password) -> str: serialized_privkey = bitcoin.serialize_privkey(pk, compressed, txin_type) return serialized_privkey + def export_private_key_for_path(self, path: Union[Sequence[int], str], password: Optional[str]) -> str: + raise Exception("this wallet is not deterministic") + @abstractmethod def get_public_keys(self, address: str) -> Sequence[str]: pass @@ -2201,6 +2204,13 @@ def derive_address(self, for_change: int, n: int) -> str: pubkeys = self.derive_pubkeys(for_change, n) return self.pubkeys_to_address(pubkeys) + def export_private_key_for_path(self, path: Union[Sequence[int], str], password: Optional[str]) -> str: + if isinstance(path, str): + path = convert_bip32_path_to_list_of_uint32(path) + pk, compressed = self.keystore.get_private_key(path, password) + txin_type = self.get_txin_type() # assumes no mixed-scripts in wallet + return bitcoin.serialize_privkey(pk, compressed, txin_type) + def get_public_keys_with_deriv_info(self, address: str): der_suffix = self.get_address_index(address) der_suffix = [int(x) for x in der_suffix] @@ -2301,7 +2311,7 @@ def get_master_public_keys(self): def get_fingerprint(self): return self.get_master_public_key() - def get_txin_type(self, address): + def get_txin_type(self, address=None): return self.txin_type From 8be94076b58e8d03c5a36dc2b3dc36afe76aedf5 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Tue, 31 Mar 2020 07:08:31 +0200 Subject: [PATCH 25/69] network: update tx broadcast error msgs whitelist fixes #6052 --- electrum/network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/electrum/network.py b/electrum/network.py index 2a2d3e181..956c4f361 100644 --- a/electrum/network.py +++ b/electrum/network.py @@ -916,7 +916,7 @@ def sanitize_tx_broadcast_response(server_msg) -> str: r"Signature hash type missing or not understood", r"Non-canonical DER signature", r"Data push larger than necessary", - r"Only non-push operators allowed in signatures", + r"Only push operators allowed in signatures", r"Non-canonical signature: S value is unnecessarily high", r"Dummy CHECKMULTISIG argument must be zero", r"OP_IF/NOTIF argument must be minimal", From 8e9b401c88f25340ba102eaece0f9eb7ee7eb39e Mon Sep 17 00:00:00 2001 From: SomberNight Date: Tue, 31 Mar 2020 07:44:23 +0200 Subject: [PATCH 26/69] wizard: add a warning to multisig wallet creation to backup xpubs --- electrum/gui/kivy/uix/dialogs/installwizard.py | 12 ++++++++++++ electrum/gui/qt/installwizard.py | 11 +++++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/electrum/gui/kivy/uix/dialogs/installwizard.py b/electrum/gui/kivy/uix/dialogs/installwizard.py index 39357897c..7b3fb6b89 100644 --- a/electrum/gui/kivy/uix/dialogs/installwizard.py +++ b/electrum/gui/kivy/uix/dialogs/installwizard.py @@ -149,6 +149,18 @@ range: 1, n.value step: 1 value: 2 + Widget + size_hint: 1, 1 + Label: + id: backup_warning_label + color: root.text_color + size_hint: 1, None + text_size: self.width, None + height: self.texture_size[1] + opacity: int(m.value != n.value) + text: _("Warning: to be able to restore a multisig wallet, " \ + "you should include the master public key for each cosigner " \ + "in all of your backups.") diff --git a/electrum/gui/qt/installwizard.py b/electrum/gui/qt/installwizard.py index 54e91c016..ed470d571 100644 --- a/electrum/gui/qt/installwizard.py +++ b/electrum/gui/qt/installwizard.py @@ -664,18 +664,25 @@ def multisig_dialog(self, run_next): def on_m(m): m_label.setText(_('Require {0} signatures').format(m)) cw.set_m(m) + backup_warning_label.setVisible(cw.m != cw.n) def on_n(n): n_label.setText(_('From {0} cosigners').format(n)) cw.set_n(n) m_edit.setMaximum(n) + backup_warning_label.setVisible(cw.m != cw.n) n_edit.valueChanged.connect(on_n) m_edit.valueChanged.connect(on_m) - on_n(2) - on_m(2) vbox = QVBoxLayout() vbox.addWidget(cw) vbox.addWidget(WWLabel(_("Choose the number of signatures needed to unlock funds in your wallet:"))) vbox.addLayout(grid) + vbox.addSpacing(2 * char_width_in_lineedit()) + backup_warning_label = WWLabel(_("Warning: to be able to restore a multisig wallet, " + "you should include the master public key for each cosigner " + "in all of your backups.")) + vbox.addWidget(backup_warning_label) + on_n(2) + on_m(2) self.exec_layout(vbox, _("Multi-Signature Wallet")) m = int(m_edit.value()) n = int(n_edit.value()) From 6760c3f252797d137ebceee5a71a7a86025f12ad Mon Sep 17 00:00:00 2001 From: SomberNight Date: Tue, 31 Mar 2020 14:40:25 +0200 Subject: [PATCH 27/69] hw wallets: introduce HardwareHandlerBase previously, client.handler was sometimes - an InstallWizard - a QtHandlerBase where win was an ElectrumWindow - a QtHandlerBase where win was an InstallWizard - a CmdLineHandler That's just too much dynamic untyped undocumented polymorphism... Now it will never be an InstallWizard (replaced with QtHandlerBase where win is an InstallWizard), and now in all cases client.handler is an instance of HardwareHandlerBase, yay. related: #6063 --- electrum/base_wizard.py | 1 + electrum/gui/qt/installwizard.py | 4 --- electrum/keystore.py | 4 +-- electrum/plugin.py | 22 +++++++----- electrum/plugins/coldcard/cmdline.py | 4 ++- electrum/plugins/hw_wallet/__init__.py | 2 +- electrum/plugins/hw_wallet/cmdline.py | 4 ++- electrum/plugins/hw_wallet/plugin.py | 50 ++++++++++++++++++++++++-- electrum/plugins/hw_wallet/qt.py | 9 ++--- electrum/plugins/keepkey/keepkey.py | 3 +- electrum/plugins/keepkey/qt.py | 6 ++-- electrum/plugins/safe_t/qt.py | 6 ++-- electrum/plugins/safe_t/safe_t.py | 3 +- electrum/plugins/trezor/qt.py | 6 ++-- electrum/plugins/trezor/trezor.py | 3 +- 15 files changed, 87 insertions(+), 40 deletions(-) diff --git a/electrum/base_wizard.py b/electrum/base_wizard.py index 8cd7d4a11..f1ff514e5 100644 --- a/electrum/base_wizard.py +++ b/electrum/base_wizard.py @@ -535,6 +535,7 @@ def create_wallet(self): if self.wallet_type == 'standard' and isinstance(self.keystores[0], Hardware_KeyStore): # offer encrypting with a pw derived from the hw device k = self.keystores[0] # type: Hardware_KeyStore + assert isinstance(self.plugin, HW_PluginBase) try: k.handler = self.plugin.create_handler(self) password = k.get_password_for_storage_encryption() diff --git a/electrum/gui/qt/installwizard.py b/electrum/gui/qt/installwizard.py index ed470d571..88056d8ae 100644 --- a/electrum/gui/qt/installwizard.py +++ b/electrum/gui/qt/installwizard.py @@ -358,10 +358,6 @@ def run_upgrades(self, storage, db): return db - def finished(self): - """Called in hardware client wrapper, in order to close popups.""" - return - def on_error(self, exc_info): if not isinstance(exc_info[1], UserCancelled): self.logger.error("on_error", exc_info=exc_info) diff --git a/electrum/keystore.py b/electrum/keystore.py index 7620edb2a..c0ec81c44 100644 --- a/electrum/keystore.py +++ b/electrum/keystore.py @@ -48,7 +48,7 @@ if TYPE_CHECKING: from .gui.qt.util import TaskThread - from .plugins.hw_wallet import HW_PluginBase, HardwareClientBase + from .plugins.hw_wallet import HW_PluginBase, HardwareClientBase, HardwareHandlerBase class KeyStore(Logger, ABC): @@ -723,7 +723,7 @@ def __init__(self, d): # device reconnects self.xpub = d.get('xpub') self.label = d.get('label') - self.handler = None + self.handler = None # type: Optional[HardwareHandlerBase] run_hook('init_keystore', self) def set_label(self, label): diff --git a/electrum/plugin.py b/electrum/plugin.py index 0bcc5d20e..c84cd9aea 100644 --- a/electrum/plugin.py +++ b/electrum/plugin.py @@ -39,7 +39,7 @@ from .logging import get_logger, Logger if TYPE_CHECKING: - from .plugins.hw_wallet import HW_PluginBase, HardwareClientBase + from .plugins.hw_wallet import HW_PluginBase, HardwareClientBase, HardwareHandlerBase from .keystore import Hardware_KeyStore @@ -386,7 +386,8 @@ def register_devices(self, device_pairs): def register_enumerate_func(self, func): self.enumerate_func.add(func) - def create_client(self, device: 'Device', handler, plugin: 'HW_PluginBase') -> Optional['HardwareClientBase']: + def create_client(self, device: 'Device', handler: Optional['HardwareHandlerBase'], + plugin: 'HW_PluginBase') -> Optional['HardwareClientBase']: # Get from cache first client = self.client_lookup(device.id_) if client: @@ -447,7 +448,8 @@ def client_by_id(self, id_) -> Optional['HardwareClientBase']: self.scan_devices() return self.client_lookup(id_) - def client_for_keystore(self, plugin: 'HW_PluginBase', handler, keystore: 'Hardware_KeyStore', + def client_for_keystore(self, plugin: 'HW_PluginBase', handler: Optional['HardwareHandlerBase'], + keystore: 'Hardware_KeyStore', force_pair: bool) -> Optional['HardwareClientBase']: self.logger.info("getting client for keystore") if handler is None: @@ -468,7 +470,7 @@ def client_for_keystore(self, plugin: 'HW_PluginBase', handler, keystore: 'Hardw self.logger.info("end client for keystore") return client - def client_by_xpub(self, plugin: 'HW_PluginBase', xpub, handler, + def client_by_xpub(self, plugin: 'HW_PluginBase', xpub, handler: 'HardwareHandlerBase', devices: Iterable['Device']) -> Optional['HardwareClientBase']: _id = self.xpub_id(xpub) client = self.client_lookup(_id) @@ -482,7 +484,7 @@ def client_by_xpub(self, plugin: 'HW_PluginBase', xpub, handler, if device.id_ == _id: return self.create_client(device, handler, plugin) - def force_pair_xpub(self, plugin: 'HW_PluginBase', handler, + def force_pair_xpub(self, plugin: 'HW_PluginBase', handler: 'HardwareHandlerBase', info: 'DeviceInfo', xpub, derivation) -> Optional['HardwareClientBase']: # The wallet has not been previously paired, so let the user # choose an unpaired device and compare its first address. @@ -510,7 +512,8 @@ def force_pair_xpub(self, plugin: 'HW_PluginBase', handler, 'its seed (and passphrase, if any). Otherwise all bitcoins you ' 'receive will be unspendable.').format(plugin.device)) - def unpaired_device_infos(self, handler, plugin: 'HW_PluginBase', devices: List['Device'] = None, + def unpaired_device_infos(self, handler: Optional['HardwareHandlerBase'], plugin: 'HW_PluginBase', + devices: List['Device'] = None, include_failing_clients=False) -> List['DeviceInfo']: '''Returns a list of DeviceInfo objects: one for each connected, unpaired device accepted by the plugin.''' @@ -539,7 +542,7 @@ def unpaired_device_infos(self, handler, plugin: 'HW_PluginBase', devices: List[ return infos - def select_device(self, plugin: 'HW_PluginBase', handler, + def select_device(self, plugin: 'HW_PluginBase', handler: 'HardwareHandlerBase', keystore: 'Hardware_KeyStore', devices: List['Device'] = None) -> 'DeviceInfo': '''Ask the user to select a device to use if there is more than one, and return the DeviceInfo for the device.''' @@ -581,8 +584,9 @@ def select_device(self, plugin: 'HW_PluginBase', handler, info = infos[c] # save new label keystore.set_label(info.label) - if handler.win.wallet is not None: - handler.win.wallet.save_keystore() + wallet = handler.get_wallet() + if wallet is not None: + wallet.save_keystore() return info def _scan_devices_with_hid(self) -> List['Device']: diff --git a/electrum/plugins/coldcard/cmdline.py b/electrum/plugins/coldcard/cmdline.py index 6e6e69a21..7df86f1f2 100644 --- a/electrum/plugins/coldcard/cmdline.py +++ b/electrum/plugins/coldcard/cmdline.py @@ -2,13 +2,15 @@ from electrum.util import print_msg, raw_input, print_stderr from electrum.logging import get_logger +from ..hw_wallet.cmdline import CmdLineHandler + from .coldcard import ColdcardPlugin _logger = get_logger(__name__) -class ColdcardCmdLineHandler: +class ColdcardCmdLineHandler(CmdLineHandler): def get_passphrase(self, msg, confirm): raise NotImplementedError diff --git a/electrum/plugins/hw_wallet/__init__.py b/electrum/plugins/hw_wallet/__init__.py index 6e3ce1a00..8fd806783 100644 --- a/electrum/plugins/hw_wallet/__init__.py +++ b/electrum/plugins/hw_wallet/__init__.py @@ -1,2 +1,2 @@ -from .plugin import HW_PluginBase, HardwareClientBase +from .plugin import HW_PluginBase, HardwareClientBase, HardwareHandlerBase from .cmdline import CmdLineHandler diff --git a/electrum/plugins/hw_wallet/cmdline.py b/electrum/plugins/hw_wallet/cmdline.py index b91e094b8..5210267f1 100644 --- a/electrum/plugins/hw_wallet/cmdline.py +++ b/electrum/plugins/hw_wallet/cmdline.py @@ -1,11 +1,13 @@ from electrum.util import print_stderr, raw_input from electrum.logging import get_logger +from .plugin import HardwareHandlerBase + _logger = get_logger(__name__) -class CmdLineHandler: +class CmdLineHandler(HardwareHandlerBase): def get_passphrase(self, msg, confirm): import getpass diff --git a/electrum/plugins/hw_wallet/plugin.py b/electrum/plugins/hw_wallet/plugin.py index 97201f40e..61129601d 100644 --- a/electrum/plugins/hw_wallet/plugin.py +++ b/electrum/plugins/hw_wallet/plugin.py @@ -37,6 +37,7 @@ if TYPE_CHECKING: from electrum.wallet import Abstract_Wallet + from electrum.base_wizard import BaseWizard class HW_PluginBase(BasePlugin): @@ -63,7 +64,7 @@ def close_wallet(self, wallet: 'Abstract_Wallet'): if isinstance(keystore, self.keystore_class): self.device_manager().unpair_xpub(keystore.xpub) - def setup_device(self, device_info, wizard, purpose): + def setup_device(self, device_info, wizard: 'BaseWizard', purpose): """Called when creating a new wallet or when using the device to decrypt an existing wallet. Select the device to use. If the device is uninitialized, go through the initialization process. @@ -139,15 +140,23 @@ def set_ignore_outdated_fw(self): def is_outdated_fw_ignored(self) -> bool: return self._ignore_outdated_fw - def create_client(self, device: 'Device', handler) -> Optional['HardwareClientBase']: + def create_client(self, device: 'Device', + handler: Optional['HardwareHandlerBase']) -> Optional['HardwareClientBase']: raise NotImplementedError() - def get_xpub(self, device_id, derivation: str, xtype, wizard) -> str: + def get_xpub(self, device_id, derivation: str, xtype, wizard: 'BaseWizard') -> str: + raise NotImplementedError() + + def create_handler(self, window) -> 'HardwareHandlerBase': + # note: in Qt GUI, 'window' is either an ElectrumWindow or an InstallWizard raise NotImplementedError() class HardwareClientBase: + plugin: 'HW_PluginBase' + handler: Optional['HardwareHandlerBase'] + def is_pairable(self) -> bool: raise NotImplementedError() @@ -191,6 +200,41 @@ def get_password_for_storage_encryption(self) -> str: return password +class HardwareHandlerBase: + """An interface between the GUI and the device handling logic for handling I/O.""" + win = None + device: str + + def get_wallet(self) -> Optional['Abstract_Wallet']: + if self.win is not None: + if hasattr(self.win, 'wallet'): + return self.win.wallet + + def update_status(self, paired: bool) -> None: + pass + + def query_choice(self, msg: str, labels: Sequence[str]) -> Optional[int]: + raise NotImplementedError() + + def yes_no_question(self, msg: str) -> bool: + raise NotImplementedError() + + def show_message(self, msg: str, on_cancel=None) -> None: + raise NotImplementedError() + + def show_error(self, msg: str, blocking: bool = False) -> None: + raise NotImplementedError() + + def finished(self) -> None: + pass + + def get_word(self, msg: str) -> str: + raise NotImplementedError() + + def get_passphrase(self, msg: str, confirm: bool) -> Optional[str]: + raise NotImplementedError() + + def is_any_tx_output_on_change_branch(tx: PartialTransaction) -> bool: return any([txout.is_change for txout in tx.outputs()]) diff --git a/electrum/plugins/hw_wallet/qt.py b/electrum/plugins/hw_wallet/qt.py index 4dbd2ac15..309e05783 100644 --- a/electrum/plugins/hw_wallet/qt.py +++ b/electrum/plugins/hw_wallet/qt.py @@ -35,13 +35,14 @@ from electrum.gui.qt.util import (read_QIcon, WWLabel, OkButton, WindowModalDialog, Buttons, CancelButton, TaskThread, char_width_in_lineedit) from electrum.gui.qt.main_window import StatusBarButton, ElectrumWindow +from electrum.gui.qt.installwizard import InstallWizard from electrum.i18n import _ from electrum.logging import Logger from electrum.util import parse_URI, InvalidBitcoinURI, UserCancelled from electrum.plugin import hook, DeviceUnpairableError -from .plugin import OutdatedHwFirmwareException, HW_PluginBase +from .plugin import OutdatedHwFirmwareException, HW_PluginBase, HardwareHandlerBase if TYPE_CHECKING: from electrum.wallet import Abstract_Wallet @@ -50,7 +51,7 @@ # The trickiest thing about this handler was getting windows properly # parented on macOS. -class QtHandlerBase(QObject, Logger): +class QtHandlerBase(HardwareHandlerBase, QObject, Logger): '''An interface between the GUI (here, QT) and the device handling logic for handling I/O.''' @@ -63,7 +64,7 @@ class QtHandlerBase(QObject, Logger): yes_no_signal = pyqtSignal(object) status_signal = pyqtSignal(object) - def __init__(self, win, device): + def __init__(self, win: Union[ElectrumWindow, InstallWizard], device: str): QObject.__init__(self) Logger.__init__(self) self.clear_signal.connect(self.clear_dialog) @@ -267,5 +268,5 @@ def show_address(): dev_name = f"{plugin.device} ({keystore.label})" receive_address_e.addButton("eye1.png", show_address, _("Show on {}").format(dev_name)) - def create_handler(self, window: ElectrumWindow) -> 'QtHandlerBase': + def create_handler(self, window: Union[ElectrumWindow, InstallWizard]) -> 'QtHandlerBase': raise NotImplementedError() diff --git a/electrum/plugins/keepkey/keepkey.py b/electrum/plugins/keepkey/keepkey.py index 3a1c31c4b..e035307f9 100644 --- a/electrum/plugins/keepkey/keepkey.py +++ b/electrum/plugins/keepkey/keepkey.py @@ -282,7 +282,6 @@ def setup_device(self, device_info, wizard, purpose): if client is None: raise UserFacingException(_('Failed to create a client for this device.') + '\n' + _('Make sure it is in the correct state.')) - # fixme: we should use: client.handler = wizard client.handler = self.create_handler(wizard) if not device_info.initialized: self.initialize_device(device_id, wizard, client.handler) @@ -294,7 +293,7 @@ def get_xpub(self, device_id, derivation, xtype, wizard): raise ScriptTypeNotSupported(_('This type of script is not supported with {}.').format(self.device)) devmgr = self.device_manager() client = devmgr.client_by_id(device_id) - client.handler = wizard + client.handler = self.create_handler(wizard) xpub = client.get_xpub(derivation, xtype) client.used() return xpub diff --git a/electrum/plugins/keepkey/qt.py b/electrum/plugins/keepkey/qt.py index eb6f0cb63..fce633df9 100644 --- a/electrum/plugins/keepkey/qt.py +++ b/electrum/plugins/keepkey/qt.py @@ -195,9 +195,6 @@ class QtPlugin(QtPluginBase): # icon_file # pin_matrix_widget_class - def create_handler(self, window): - return QtHandler(window, self.pin_matrix_widget_class(), self.device) - @only_hook_if_libraries_available @hook def receive_menu(self, menu, addrs, wallet): @@ -302,6 +299,9 @@ class Plugin(KeepKeyPlugin, QtPlugin): icon_paired = "keepkey.png" icon_unpaired = "keepkey_unpaired.png" + def create_handler(self, window): + return QtHandler(window, self.pin_matrix_widget_class(), self.device) + @classmethod def pin_matrix_widget_class(self): from keepkeylib.qt.pinmatrix import PinMatrixWidget diff --git a/electrum/plugins/safe_t/qt.py b/electrum/plugins/safe_t/qt.py index 224eec802..aa44495bb 100644 --- a/electrum/plugins/safe_t/qt.py +++ b/electrum/plugins/safe_t/qt.py @@ -71,9 +71,6 @@ class QtPlugin(QtPluginBase): # icon_file # pin_matrix_widget_class - def create_handler(self, window): - return QtHandler(window, self.pin_matrix_widget_class(), self.device) - @only_hook_if_libraries_available @hook def receive_menu(self, menu, addrs, wallet): @@ -176,6 +173,9 @@ class Plugin(SafeTPlugin, QtPlugin): icon_unpaired = "safe-t_unpaired.png" icon_paired = "safe-t.png" + def create_handler(self, window): + return QtHandler(window, self.pin_matrix_widget_class(), self.device) + @classmethod def pin_matrix_widget_class(self): from safetlib.qt.pinmatrix import PinMatrixWidget diff --git a/electrum/plugins/safe_t/safe_t.py b/electrum/plugins/safe_t/safe_t.py index 8c10bf293..22064094c 100644 --- a/electrum/plugins/safe_t/safe_t.py +++ b/electrum/plugins/safe_t/safe_t.py @@ -256,7 +256,6 @@ def setup_device(self, device_info, wizard, purpose): if client is None: raise UserFacingException(_('Failed to create a client for this device.') + '\n' + _('Make sure it is in the correct state.')) - # fixme: we should use: client.handler = wizard client.handler = self.create_handler(wizard) if not device_info.initialized: self.initialize_device(device_id, wizard, client.handler) @@ -268,7 +267,7 @@ def get_xpub(self, device_id, derivation, xtype, wizard): raise ScriptTypeNotSupported(_('This type of script is not supported with {}.').format(self.device)) devmgr = self.device_manager() client = devmgr.client_by_id(device_id) - client.handler = wizard + client.handler = self.create_handler(wizard) xpub = client.get_xpub(derivation, xtype) client.used() return xpub diff --git a/electrum/plugins/trezor/qt.py b/electrum/plugins/trezor/qt.py index 3a8b2cba3..37a45ce6b 100644 --- a/electrum/plugins/trezor/qt.py +++ b/electrum/plugins/trezor/qt.py @@ -169,9 +169,6 @@ class QtPlugin(QtPluginBase): # icon_file # pin_matrix_widget_class - def create_handler(self, window): - return QtHandler(window, self.pin_matrix_widget_class(), self.device) - @only_hook_if_libraries_available @hook def receive_menu(self, menu, addrs, wallet): @@ -377,6 +374,9 @@ class Plugin(TrezorPlugin, QtPlugin): icon_unpaired = "trezor_unpaired.png" icon_paired = "trezor.png" + def create_handler(self, window): + return QtHandler(window, self.pin_matrix_widget_class(), self.device) + @classmethod def pin_matrix_widget_class(self): from trezorlib.qt.pinmatrix import PinMatrixWidget diff --git a/electrum/plugins/trezor/trezor.py b/electrum/plugins/trezor/trezor.py index 71341d91c..d1d94b010 100644 --- a/electrum/plugins/trezor/trezor.py +++ b/electrum/plugins/trezor/trezor.py @@ -282,7 +282,6 @@ def setup_device(self, device_info, wizard, purpose): .format(self.device, client.label(), self.firmware_URL)) raise OutdatedHwFirmwareException(msg) - # fixme: we should use: client.handler = wizard client.handler = self.create_handler(wizard) if not device_info.initialized: self.initialize_device(device_id, wizard, client.handler) @@ -295,7 +294,7 @@ def get_xpub(self, device_id, derivation, xtype, wizard): raise ScriptTypeNotSupported(_('This type of script is not supported with {}.').format(self.device)) devmgr = self.device_manager() client = devmgr.client_by_id(device_id) - client.handler = wizard + client.handler = self.create_handler(wizard) xpub = client.get_xpub(derivation, xtype) client.used() return xpub From 7297e949708e24a1cfbf33959ca336477ad5181a Mon Sep 17 00:00:00 2001 From: SomberNight Date: Tue, 31 Mar 2020 15:11:10 +0200 Subject: [PATCH 28/69] hw wallets: handle cancellation for "query_choice" in wizard E | gui.qt.exception_window.Exception_Hook | exception caught by crash reporter Traceback (most recent call last): File "...\electrum\electrum\plugins\hw_wallet\qt.py", line 193, in win_query_choice self.choice = self.win.query_choice(msg, labels) File "...\electrum\electrum\gui\qt\installwizard.py", line 545, in query_choice self.exec_layout(vbox, '') File "...\electrum\electrum\gui\qt\installwizard.py", line 392, in exec_layout raise UserCancelled electrum.util.UserCancelled --- electrum/plugins/hw_wallet/qt.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/electrum/plugins/hw_wallet/qt.py b/electrum/plugins/hw_wallet/qt.py index 309e05783..3291d3bd9 100644 --- a/electrum/plugins/hw_wallet/qt.py +++ b/electrum/plugins/hw_wallet/qt.py @@ -190,7 +190,10 @@ def clear_dialog(self): self.dialog = None def win_query_choice(self, msg, labels): - self.choice = self.win.query_choice(msg, labels) + try: + self.choice = self.win.query_choice(msg, labels) + except UserCancelled: + self.choice = None self.done.set() def win_yes_no_question(self, msg): From 3ea2872b31a742f7f5a9bae585ee365435566bc1 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Tue, 31 Mar 2020 15:18:24 +0200 Subject: [PATCH 29/69] hw wallets: show e.g. "An unnamed trezor" if no label in select_device related: #6063 --- electrum/plugin.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/electrum/plugin.py b/electrum/plugin.py index c84cd9aea..11d0ca8fe 100644 --- a/electrum/plugin.py +++ b/electrum/plugin.py @@ -305,6 +305,7 @@ class DeviceInfo(NamedTuple): label: Optional[str] = None initialized: Optional[bool] = None exception: Optional[Exception] = None + plugin_name: Optional[str] = None # manufacturer, e.g. "trezor" class HardwarePluginToScan(NamedTuple): @@ -532,13 +533,14 @@ def unpaired_device_infos(self, handler: Optional['HardwareHandlerBase'], plugin except Exception as e: self.logger.error(f'failed to create client for {plugin.name} at {device.path}: {repr(e)}') if include_failing_clients: - infos.append(DeviceInfo(device=device, exception=e)) + infos.append(DeviceInfo(device=device, exception=e, plugin_name=plugin.name)) continue if not client: continue infos.append(DeviceInfo(device=device, label=client.label(), - initialized=client.is_initialized())) + initialized=client.is_initialized(), + plugin_name=plugin.name)) return infos @@ -574,7 +576,7 @@ def select_device(self, plugin: 'HW_PluginBase', handler: 'HardwareHandlerBase', # ask user to select device msg = _("Please select which {} device to use:").format(plugin.device) descriptions = ["{label} ({init}, {transport})" - .format(label=info.label, + .format(label=info.label or _("An unnamed {}").format(info.plugin_name), init=(_("initialized") if info.initialized else _("wiped")), transport=info.device.transport_ui_string) for info in infos] From 570f7b7790e6716cff49ba75992728cab3375e21 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Tue, 31 Mar 2020 15:28:57 +0200 Subject: [PATCH 30/69] qt wizard decrypt wallet with hww: just pass through cancellation E | gui.qt.installwizard.InstallWizard | Traceback (most recent call last): File "...\electrum\electrum\base_wizard.py", line 541, in create_wallet password = k.get_password_for_storage_encryption() File "...\electrum\electrum\keystore.py", line 768, in get_password_for_storage_encryption client = self.plugin.get_client(self) File "...\electrum\electrum\plugins\trezor\trezor.py", line 180, in get_client client = devmgr.client_for_keystore(self, handler, keystore, force_pair) File "...\electrum\electrum\plugin.py", line 465, in client_for_keystore info = self.select_device(plugin, handler, keystore, devices) File "...\electrum\electrum\plugin.py", line 585, in select_device raise UserCancelled() electrum.util.UserCancelled During handling of the above exception, another exception occurred: Traceback (most recent call last): File "...\electrum\electrum\gui\qt\installwizard.py", line 300, in select_storage self.run('choose_hw_device', HWD_SETUP_DECRYPT_WALLET, storage=temp_storage) File "...\electrum\electrum\base_wizard.py", line 109, in run f(*args, **kwargs) File "...\electrum\electrum\base_wizard.py", line 332, in choose_hw_device self.choice_dialog(title=title, message=msg, choices=choices, File "...\electrum\electrum\gui\qt\installwizard.py", line 99, in func_wrapper out = func(*args, **kwargs) File "...\electrum\electrum\gui\qt\installwizard.py", line 536, in choice_dialog self.exec_layout(vbox, title) File "...\electrum\electrum\gui\qt\installwizard.py", line 392, in exec_layout raise UserCancelled electrum.util.UserCancelled --- electrum/gui/qt/installwizard.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/electrum/gui/qt/installwizard.py b/electrum/gui/qt/installwizard.py index 88056d8ae..99ad9cf01 100644 --- a/electrum/gui/qt/installwizard.py +++ b/electrum/gui/qt/installwizard.py @@ -304,6 +304,8 @@ def on_filename(filename): _('If you use a passphrase, make sure it is correct.')) self.reset_stack() return self.select_storage(path, get_wallet_from_daemon) + except UserCancelled: + raise except BaseException as e: self.logger.exception('') self.show_message(title=_('Error'), msg=repr(e)) From 18d245ad5c55b480aae24c5f134083b012beb9dd Mon Sep 17 00:00:00 2001 From: SomberNight Date: Tue, 31 Mar 2020 15:56:54 +0200 Subject: [PATCH 31/69] hw wallets: during wallet creation, make sure to save correct label When initialising a Trezor as part of the wallet creation, device_info.label is still the old (None) label in on_hw_derivation. This is because device_info was created during the initial scan. related: #6063 --- electrum/base_wizard.py | 7 ++++--- electrum/plugins/hw_wallet/plugin.py | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/electrum/base_wizard.py b/electrum/base_wizard.py index f1ff514e5..0157f6f98 100644 --- a/electrum/base_wizard.py +++ b/electrum/base_wizard.py @@ -332,7 +332,7 @@ def failed_getting_device_infos(name, e): self.choice_dialog(title=title, message=msg, choices=choices, run_next=lambda *args: self.on_device(*args, purpose=purpose, storage=storage)) - def on_device(self, name, device_info, *, purpose, storage=None): + def on_device(self, name, device_info: 'DeviceInfo', *, purpose, storage=None): self.plugin = self.plugins.get_plugin(name) assert isinstance(self.plugin, HW_PluginBase) devmgr = self.plugins.device_manager @@ -414,7 +414,7 @@ def derivation_and_script_type_dialog(self, f): self.show_error(e) # let the user choose again - def on_hw_derivation(self, name, device_info, derivation, xtype): + def on_hw_derivation(self, name, device_info: 'DeviceInfo', derivation, xtype): from .keystore import hardware_keystore devmgr = self.plugins.device_manager try: @@ -422,6 +422,7 @@ def on_hw_derivation(self, name, device_info, derivation, xtype): client = devmgr.client_by_id(device_info.device.id_) if not client: raise Exception("failed to find client for device id") root_fingerprint = client.request_root_fingerprint_from_device() + label = client.label() # use this as device_info.label might be outdated! except ScriptTypeNotSupported: raise # this is handled in derivation_dialog except BaseException as e: @@ -434,7 +435,7 @@ def on_hw_derivation(self, name, device_info, derivation, xtype): 'derivation': derivation, 'root_fingerprint': root_fingerprint, 'xpub': xpub, - 'label': device_info.label, + 'label': label, } k = hardware_keystore(d) self.on_keystore(k) diff --git a/electrum/plugins/hw_wallet/plugin.py b/electrum/plugins/hw_wallet/plugin.py index 61129601d..19e06a077 100644 --- a/electrum/plugins/hw_wallet/plugin.py +++ b/electrum/plugins/hw_wallet/plugin.py @@ -26,7 +26,7 @@ from typing import TYPE_CHECKING, Dict, List, Union, Tuple, Sequence, Optional, Type -from electrum.plugin import BasePlugin, hook, Device, DeviceMgr +from electrum.plugin import BasePlugin, hook, Device, DeviceMgr, DeviceInfo from electrum.i18n import _ from electrum.bitcoin import is_address, opcodes from electrum.util import bfh, versiontuple, UserFacingException @@ -64,7 +64,7 @@ def close_wallet(self, wallet: 'Abstract_Wallet'): if isinstance(keystore, self.keystore_class): self.device_manager().unpair_xpub(keystore.xpub) - def setup_device(self, device_info, wizard: 'BaseWizard', purpose): + def setup_device(self, device_info: DeviceInfo, wizard: 'BaseWizard', purpose): """Called when creating a new wallet or when using the device to decrypt an existing wallet. Select the device to use. If the device is uninitialized, go through the initialization process. From e53ce5dee0694cbde0a5b28fca7a59e9ea19d83a Mon Sep 17 00:00:00 2001 From: SomberNight Date: Tue, 31 Mar 2020 18:57:03 +0200 Subject: [PATCH 32/69] (trivial) follow-up 570f7b7790e6716cff49ba75992728cab3375e21 --- electrum/gui/qt/installwizard.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/electrum/gui/qt/installwizard.py b/electrum/gui/qt/installwizard.py index 99ad9cf01..45af0a21a 100644 --- a/electrum/gui/qt/installwizard.py +++ b/electrum/gui/qt/installwizard.py @@ -304,7 +304,7 @@ def on_filename(filename): _('If you use a passphrase, make sure it is correct.')) self.reset_stack() return self.select_storage(path, get_wallet_from_daemon) - except UserCancelled: + except (UserCancelled, GoBack): raise except BaseException as e: self.logger.exception('') From 2d3c2eeea9c225af2ec2d1654363c4b75c5b1158 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Wed, 1 Apr 2020 13:31:49 +0200 Subject: [PATCH 33/69] keystore: add workaround for StoredDict issue #6066 note: not a proper fix... but works for now --- electrum/keystore.py | 4 +++- electrum/plugins/ledger/ledger.py | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/electrum/keystore.py b/electrum/keystore.py index c0ec81c44..d82d3f260 100644 --- a/electrum/keystore.py +++ b/electrum/keystore.py @@ -49,6 +49,7 @@ if TYPE_CHECKING: from .gui.qt.util import TaskThread from .plugins.hw_wallet import HW_PluginBase, HardwareClientBase, HardwareHandlerBase + from .wallet_db import WalletDB class KeyStore(Logger, ABC): @@ -886,8 +887,9 @@ def hardware_keystore(d) -> Hardware_KeyStore: raise WalletFileException(f'unknown hardware type: {hw_type}. ' f'hw_keystores: {list(hw_keystores)}') -def load_keystore(db, name) -> KeyStore: +def load_keystore(db: 'WalletDB', name: str) -> KeyStore: d = db.get(name, {}) + d = dict(d) # convert to dict from StoredDict (see #6066) t = d.get('type') if not t: raise WalletFileException( diff --git a/electrum/plugins/ledger/ledger.py b/electrum/plugins/ledger/ledger.py index da0e29640..d086b9968 100644 --- a/electrum/plugins/ledger/ledger.py +++ b/electrum/plugins/ledger/ledger.py @@ -229,6 +229,7 @@ def __init__(self, d): self.force_watching_only = False self.signing = False self.cfg = d.get('cfg', {'mode': 0}) + self.cfg = dict(self.cfg) # convert to dict from StoredDict (see #6066) def dump(self): obj = Hardware_KeyStore.dump(self) From e68b6447cc3457b5151c148873fbd82b8d0a1f49 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Wed, 1 Apr 2020 14:24:29 +0200 Subject: [PATCH 34/69] hww: catch exceptions when user clicks on hww qt status bar icon E | gui.qt.exception_window.Exception_Hook | exception caught by crash reporter Traceback (most recent call last): File "...\electrum\electrum\plugins\ledger\ledger.py", line 167, in perform_hw1_preflight firmwareInfo = self.dongleObject.getFirmwareVersion() File "...\Python38\site-packages\btchip\btchip.py", line 561, in getFirmwareVersion response = self.dongle.exchange(bytearray(apdu)) File "...\Python38\site-packages\btchip\btchipComm.py", line 127, in exchange raise BTChipException("Invalid status %04x" % sw, sw) btchip.btchipException.BTChipException: Exception : Invalid status 6faa During handling of the above exception, another exception occurred: Traceback (most recent call last): File "...\electrum\electrum\gui\qt\main_window.py", line 120, in onPress self.func() File "...\electrum\electrum\plugins\hw_wallet\qt.py", line 260, in show_settings_dialog device_id = self.choose_device(window, keystore) File "...\electrum\electrum\plugins\hw_wallet\qt.py", line 253, in choose_device info = self.device_manager().select_device(self, keystore.handler, keystore) File "...\electrum\electrum\plugin.py", line 554, in select_device infos = self.unpaired_device_infos(handler, plugin, devices) File "...\electrum\electrum\plugin.py", line 545, in unpaired_device_infos soft_device_id=client.get_soft_device_id())) File "...\electrum\electrum\plugins\ledger\ledger.py", line 88, in get_soft_device_id self._soft_device_id = self.request_root_fingerprint_from_device() File "...\electrum\electrum\plugins\hw_wallet\plugin.py", line 197, in request_root_fingerprint_from_device child_of_root_xpub = self.get_xpub("m/0'", xtype='standard') File "...\electrum\electrum\plugins\ledger\ledger.py", line 55, in catch_exception return func(self, *args, **kwargs) File "...\electrum\electrum\plugins\ledger\ledger.py", line 103, in get_xpub self.checkDevice() File "...\electrum\electrum\plugins\ledger\ledger.py", line 210, in checkDevice self.perform_hw1_preflight() File "...\electrum\electrum\plugins\ledger\ledger.py", line 198, in perform_hw1_preflight raise UserFacingException("Dongle is temporarily locked - please unplug it and replug it again") electrum.util.UserFacingException: Dongle is temporarily locked - please unplug it and replug it again --- electrum/gui/qt/main_window.py | 3 +++ electrum/plugins/hw_wallet/qt.py | 11 +++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/electrum/gui/qt/main_window.py b/electrum/gui/qt/main_window.py index 1a43a7df3..ba5b00871 100644 --- a/electrum/gui/qt/main_window.py +++ b/electrum/gui/qt/main_window.py @@ -379,6 +379,9 @@ def on_error(self, exc_info): elif isinstance(e, UserFacingException): self.show_error(str(e)) else: + # TODO would be nice if we just sent these to the crash reporter... + # anything we don't want to send there, we should explicitly catch + # send_exception_to_crash_reporter(e) try: self.logger.error("on_error", exc_info=exc_info) except OSError: diff --git a/electrum/plugins/hw_wallet/qt.py b/electrum/plugins/hw_wallet/qt.py index 3291d3bd9..8df94ad7c 100644 --- a/electrum/plugins/hw_wallet/qt.py +++ b/electrum/plugins/hw_wallet/qt.py @@ -39,7 +39,7 @@ from electrum.i18n import _ from electrum.logging import Logger -from electrum.util import parse_URI, InvalidBitcoinURI, UserCancelled +from electrum.util import parse_URI, InvalidBitcoinURI, UserCancelled, UserFacingException from electrum.plugin import hook, DeviceUnpairableError from .plugin import OutdatedHwFirmwareException, HW_PluginBase, HardwareHandlerBase @@ -213,7 +213,7 @@ def load_wallet(self: Union['QtPluginBase', HW_PluginBase], wallet: 'Abstract_Wa window.show_error(message) return tooltip = self.device + '\n' + (keystore.label or 'unnamed') - cb = partial(self.show_settings_dialog, window, keystore) + cb = partial(self._on_status_bar_button_click, window=window, keystore=keystore) button = StatusBarButton(read_QIcon(self.icon_unpaired), tooltip, cb) button.icon_paired = self.icon_paired button.icon_unpaired = self.icon_unpaired @@ -226,6 +226,13 @@ def load_wallet(self: Union['QtPluginBase', HW_PluginBase], wallet: 'Abstract_Wa # Trigger a pairing keystore.thread.add(partial(self.get_client, keystore)) + def _on_status_bar_button_click(self, *, window: ElectrumWindow, keystore: 'Hardware_KeyStore'): + try: + self.show_settings_dialog(window=window, keystore=keystore) + except (UserFacingException, UserCancelled) as e: + exc_info = (type(e), e, e.__traceback__) + self.on_task_thread_error(window=window, keystore=keystore, exc_info=exc_info) + def on_task_thread_error(self: Union['QtPluginBase', HW_PluginBase], window: ElectrumWindow, keystore: 'Hardware_KeyStore', exc_info): e = exc_info[1] From c0b170acb7e32add65284223329137ed8bc7245d Mon Sep 17 00:00:00 2001 From: SomberNight Date: Wed, 1 Apr 2020 16:36:41 +0200 Subject: [PATCH 35/69] hww wizard: better handle UserFacingException in one case E | gui.qt.installwizard.InstallWizard | Traceback (most recent call last): File "...\electrum\electrum\base_wizard.py", line 340, in on_device self.plugin.setup_device(device_info, self, purpose) File "...\electrum\electrum\plugins\digitalbitbox\digitalbitbox.py", line 719, in setup_device client.get_xpub("m/44'/0'", 'standard') File "...\electrum\electrum\plugins\digitalbitbox\digitalbitbox.py", line 120, in get_xpub reply = self._get_xpub(bip32_path) File "...\electrum\electrum\plugins\digitalbitbox\digitalbitbox.py", line 114, in _get_xpub if self.check_device_dialog(): File "...\electrum\electrum\plugins\digitalbitbox\digitalbitbox.py", line 223, in check_device_dialog self.recover_or_erase_dialog() # Already seeded File "...\electrum\electrum\plugins\digitalbitbox\digitalbitbox.py", line 244, in recover_or_erase_dialog if not self.dbb_load_backup(): File "...\electrum\electrum\plugins\digitalbitbox\digitalbitbox.py", line 340, in dbb_load_backup raise UserFacingException(backups['error']['message']) electrum.util.UserFacingException: Please insert SD card. --- electrum/base_wizard.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/electrum/base_wizard.py b/electrum/base_wizard.py index 0157f6f98..80df02316 100644 --- a/electrum/base_wizard.py +++ b/electrum/base_wizard.py @@ -40,7 +40,7 @@ from .storage import WalletStorage, StorageEncryptionVersion from .wallet_db import WalletDB from .i18n import _ -from .util import UserCancelled, InvalidPassword, WalletFileException +from .util import UserCancelled, InvalidPassword, WalletFileException, UserFacingException from .simple_config import SimpleConfig from .plugin import Plugins, HardwarePluginLibraryUnavailable from .logging import Logger @@ -356,6 +356,10 @@ def on_device(self, name, device_info: 'DeviceInfo', *, purpose, storage=None): except (UserCancelled, GoBack): self.choose_hw_device(purpose, storage=storage) return + except UserFacingException as e: + self.show_error(str(e)) + self.choose_hw_device(purpose, storage=storage) + return except BaseException as e: self.logger.exception('') self.show_error(str(e)) From 7f1c7955dc401c31d1c519bfcb3e92b714afc334 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Wed, 1 Apr 2020 18:31:08 +0200 Subject: [PATCH 36/69] DeviceMgr: clean-up locks a bit --- electrum/plugin.py | 23 +++++++++++++------ electrum/plugins/coldcard/coldcard.py | 3 +-- .../plugins/digitalbitbox/digitalbitbox.py | 3 +-- electrum/plugins/keepkey/keepkey.py | 3 +-- electrum/plugins/ledger/ledger.py | 3 +-- electrum/plugins/safe_t/safe_t.py | 3 +-- electrum/plugins/trezor/trezor.py | 3 +-- 7 files changed, 22 insertions(+), 19 deletions(-) diff --git a/electrum/plugin.py b/electrum/plugin.py index 11d0ca8fe..d873a64e5 100644 --- a/electrum/plugin.py +++ b/electrum/plugin.py @@ -349,24 +349,31 @@ class DeviceMgr(ThreadJob): This plugin is thread-safe. Currently only devices supported by hidapi are implemented.''' - def __init__(self, config): + def __init__(self, config: SimpleConfig): ThreadJob.__init__(self) # Keyed by xpub. The value is the device id - # has been paired, and None otherwise. + # has been paired, and None otherwise. Needs self.lock. self.xpub_ids = {} # type: Dict[str, str] # A list of clients. The key is the client, the value is - # a (path, id_) pair. + # a (path, id_) pair. Needs self.lock. self.clients = {} # type: Dict[HardwareClientBase, Tuple[Union[str, bytes], str]] # What we recognise. Each entry is a (vendor_id, product_id) # pair. self.recognised_hardware = set() # Custom enumerate functions for devices we don't know about. self.enumerate_func = set() - # For synchronization + # locks: if you need to take multiple ones, acquire them in the order they are defined here! + self._scan_lock = threading.RLock() self.lock = threading.RLock() - self.hid_lock = threading.RLock() + self.config = config + def with_scan_lock(func): + def func_wrapper(self: 'DeviceMgr', *args, **kwargs): + with self._scan_lock: + return func(self, *args, **kwargs) + return func_wrapper + def thread_jobs(self): # Thread job to handle device timeouts return [self] @@ -449,6 +456,7 @@ def client_by_id(self, id_) -> Optional['HardwareClientBase']: self.scan_devices() return self.client_lookup(id_) + @with_scan_lock def client_for_keystore(self, plugin: 'HW_PluginBase', handler: Optional['HardwareHandlerBase'], keystore: 'Hardware_KeyStore', force_pair: bool) -> Optional['HardwareClientBase']: @@ -591,14 +599,14 @@ def select_device(self, plugin: 'HW_PluginBase', handler: 'HardwareHandlerBase', wallet.save_keystore() return info + @with_scan_lock def _scan_devices_with_hid(self) -> List['Device']: try: import hid except ImportError: return [] - with self.hid_lock: - hid_list = hid.enumerate(0, 0) + hid_list = hid.enumerate(0, 0) devices = [] for d in hid_list: @@ -619,6 +627,7 @@ def _scan_devices_with_hid(self) -> List['Device']: transport_ui_string='hid')) return devices + @with_scan_lock def scan_devices(self) -> List['Device']: self.logger.info("scanning devices...") diff --git a/electrum/plugins/coldcard/coldcard.py b/electrum/plugins/coldcard/coldcard.py index 9a9a2deae..3a391993c 100644 --- a/electrum/plugins/coldcard/coldcard.py +++ b/electrum/plugins/coldcard/coldcard.py @@ -547,8 +547,7 @@ def get_client(self, keystore, force_pair=True) -> 'CKCCClient': # Acquire a connection to the hardware device (via USB) devmgr = self.device_manager() handler = keystore.handler - with devmgr.hid_lock: - client = devmgr.client_for_keystore(self, handler, keystore, force_pair) + client = devmgr.client_for_keystore(self, handler, keystore, force_pair) if client is not None: client.ping_check() diff --git a/electrum/plugins/digitalbitbox/digitalbitbox.py b/electrum/plugins/digitalbitbox/digitalbitbox.py index 614b13729..162175dcb 100644 --- a/electrum/plugins/digitalbitbox/digitalbitbox.py +++ b/electrum/plugins/digitalbitbox/digitalbitbox.py @@ -754,8 +754,7 @@ def get_xpub(self, device_id, derivation, xtype, wizard): def get_client(self, keystore, force_pair=True): devmgr = self.device_manager() handler = keystore.handler - with devmgr.hid_lock: - client = devmgr.client_for_keystore(self, handler, keystore, force_pair) + client = devmgr.client_for_keystore(self, handler, keystore, force_pair) if client is not None: client.check_device_dialog() return client diff --git a/electrum/plugins/keepkey/keepkey.py b/electrum/plugins/keepkey/keepkey.py index e035307f9..ac87bd00a 100644 --- a/electrum/plugins/keepkey/keepkey.py +++ b/electrum/plugins/keepkey/keepkey.py @@ -182,8 +182,7 @@ def create_client(self, device, handler): def get_client(self, keystore, force_pair=True) -> Optional['KeepKeyClient']: devmgr = self.device_manager() handler = keystore.handler - with devmgr.hid_lock: - client = devmgr.client_for_keystore(self, handler, keystore, force_pair) + client = devmgr.client_for_keystore(self, handler, keystore, force_pair) # returns the client for a given keystore. can use xpub if client: client.used() diff --git a/electrum/plugins/ledger/ledger.py b/electrum/plugins/ledger/ledger.py index d086b9968..1c2e429f9 100644 --- a/electrum/plugins/ledger/ledger.py +++ b/electrum/plugins/ledger/ledger.py @@ -612,8 +612,7 @@ def get_client(self, keystore, force_pair=True): # All client interaction should not be in the main GUI thread devmgr = self.device_manager() handler = keystore.handler - with devmgr.hid_lock: - client = devmgr.client_for_keystore(self, handler, keystore, force_pair) + client = devmgr.client_for_keystore(self, handler, keystore, force_pair) # returns the client for a given keystore. can use xpub #if client: # client.used() diff --git a/electrum/plugins/safe_t/safe_t.py b/electrum/plugins/safe_t/safe_t.py index 22064094c..1deb7345b 100644 --- a/electrum/plugins/safe_t/safe_t.py +++ b/electrum/plugins/safe_t/safe_t.py @@ -144,8 +144,7 @@ def create_client(self, device, handler): def get_client(self, keystore, force_pair=True) -> Optional['SafeTClient']: devmgr = self.device_manager() handler = keystore.handler - with devmgr.hid_lock: - client = devmgr.client_for_keystore(self, handler, keystore, force_pair) + client = devmgr.client_for_keystore(self, handler, keystore, force_pair) # returns the client for a given keystore. can use xpub if client: client.used() diff --git a/electrum/plugins/trezor/trezor.py b/electrum/plugins/trezor/trezor.py index d1d94b010..b2a638cd5 100644 --- a/electrum/plugins/trezor/trezor.py +++ b/electrum/plugins/trezor/trezor.py @@ -176,8 +176,7 @@ def create_client(self, device, handler): def get_client(self, keystore, force_pair=True) -> Optional['TrezorClientBase']: devmgr = self.device_manager() handler = keystore.handler - with devmgr.hid_lock: - client = devmgr.client_for_keystore(self, handler, keystore, force_pair) + client = devmgr.client_for_keystore(self, handler, keystore, force_pair) # returns the client for a given keystore. can use xpub if client: client.used() From 276631fab7920d3781614d3fabce2f463bac672d Mon Sep 17 00:00:00 2001 From: SomberNight Date: Wed, 1 Apr 2020 18:35:41 +0200 Subject: [PATCH 37/69] digitalbitbox: (trivial) user handler instead of handler.win --- .../plugins/digitalbitbox/digitalbitbox.py | 28 ++++++++----------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/electrum/plugins/digitalbitbox/digitalbitbox.py b/electrum/plugins/digitalbitbox/digitalbitbox.py index 162175dcb..80f9a87c1 100644 --- a/electrum/plugins/digitalbitbox/digitalbitbox.py +++ b/electrum/plugins/digitalbitbox/digitalbitbox.py @@ -234,10 +234,9 @@ def recover_or_erase_dialog(self): (_("Load a wallet from the micro SD card (the current seed is overwritten)")), (_("Erase the Digital Bitbox")) ] - try: - reply = self.handler.win.query_choice(msg, choices) - except Exception: - return # Back button pushed + reply = self.handler.query_choice(msg, choices) + if reply is None: + return # user cancelled if reply == 2: self.dbb_erase() elif reply == 1: @@ -256,10 +255,9 @@ def seed_device_dialog(self): (_("Generate a new random wallet")), (_("Load a wallet from the micro SD card")) ] - try: - reply = self.handler.win.query_choice(msg, choices) - except Exception: - return # Back button pushed + reply = self.handler.query_choice(msg, choices) + if reply is None: + return # user cancelled if reply == 0: self.dbb_generate_wallet() else: @@ -297,10 +295,9 @@ def mobile_pairing_dialog(self): _('Do not pair'), _('Import pairing from the Digital Bitbox desktop app'), ] - try: - reply = self.handler.win.query_choice(_('Mobile pairing options'), choices) - except Exception: - return # Back button pushed + reply = self.handler.query_choice(_('Mobile pairing options'), choices) + if reply is None: + return # user cancelled if reply == 0: if self.plugin.is_mobile_paired(): @@ -338,10 +335,9 @@ def dbb_load_backup(self, show_msg=True): backups = self.hid_send_encrypt(b'{"backup":"list"}') if 'error' in backups: raise UserFacingException(backups['error']['message']) - try: - f = self.handler.win.query_choice(_("Choose a backup file:"), backups['backup']) - except Exception: - return False # Back button pushed + f = self.handler.query_choice(_("Choose a backup file:"), backups['backup']) + if f is None: + return False # user cancelled key = self.backup_password_dialog() if key is None: raise Exception('Canceled by user') From e6d43b60fa275a2b7cc0392442e5b9fddfa86663 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Wed, 1 Apr 2020 18:42:06 +0200 Subject: [PATCH 38/69] qt hww show_settings_dialog: don't scan devices in GUI thread Just makes sense in general. Also, previously, the GUI would freeze if right after startup the user clicked the hww status bar icon (especially with multiple hww connected). --- electrum/gui/qt/__init__.py | 4 ++-- electrum/gui/qt/installwizard.py | 4 +++- electrum/plugin.py | 2 ++ electrum/plugins/hw_wallet/plugin.py | 6 ++++++ electrum/plugins/hw_wallet/qt.py | 7 ++++++- electrum/plugins/keepkey/qt.py | 10 +++++++--- electrum/plugins/safe_t/qt.py | 10 +++++++--- electrum/plugins/trezor/qt.py | 10 +++++++--- 8 files changed, 40 insertions(+), 13 deletions(-) diff --git a/electrum/gui/qt/__init__.py b/electrum/gui/qt/__init__.py index e5fda06e6..1b2a6055f 100644 --- a/electrum/gui/qt/__init__.py +++ b/electrum/gui/qt/__init__.py @@ -303,7 +303,7 @@ def start_new_window(self, path, uri, *, app_is_starting=False): return window def _start_wizard_to_select_or_create_wallet(self, path) -> Optional[Abstract_Wallet]: - wizard = InstallWizard(self.config, self.app, self.plugins) + wizard = InstallWizard(self.config, self.app, self.plugins, gui_object=self) try: path, storage = wizard.select_storage(path, self.daemon.get_wallet) # storage is None if file does not exist @@ -342,7 +342,7 @@ def init_network(self): # Show network dialog if config does not exist if self.daemon.network: if self.config.get('auto_connect') is None: - wizard = InstallWizard(self.config, self.app, self.plugins) + wizard = InstallWizard(self.config, self.app, self.plugins, gui_object=self) wizard.init_network(self.daemon.network) wizard.terminate() diff --git a/electrum/gui/qt/installwizard.py b/electrum/gui/qt/installwizard.py index 45af0a21a..961f88916 100644 --- a/electrum/gui/qt/installwizard.py +++ b/electrum/gui/qt/installwizard.py @@ -31,6 +31,7 @@ if TYPE_CHECKING: from electrum.simple_config import SimpleConfig + from . import ElectrumGui MSG_ENTER_PASSWORD = _("Choose a password to encrypt your wallet keys.") + '\n'\ @@ -121,12 +122,13 @@ class InstallWizard(QDialog, MessageBoxMixin, BaseWizard): accept_signal = pyqtSignal() - def __init__(self, config: 'SimpleConfig', app: QApplication, plugins: 'Plugins'): + def __init__(self, config: 'SimpleConfig', app: QApplication, plugins: 'Plugins', *, gui_object: 'ElectrumGui'): QDialog.__init__(self, None) BaseWizard.__init__(self, config, plugins) self.setWindowTitle('Electrum - ' + _('Install Wizard')) self.app = app self.config = config + self.gui_thread = gui_object.gui_thread self.setMinimumSize(600, 400) self.accept_signal.connect(self.accept) self.title = QLabel() diff --git a/electrum/plugin.py b/electrum/plugin.py index d873a64e5..dbebf3a9f 100644 --- a/electrum/plugin.py +++ b/electrum/plugin.py @@ -556,6 +556,8 @@ def select_device(self, plugin: 'HW_PluginBase', handler: 'HardwareHandlerBase', keystore: 'Hardware_KeyStore', devices: List['Device'] = None) -> 'DeviceInfo': '''Ask the user to select a device to use if there is more than one, and return the DeviceInfo for the device.''' + # ideally this should not be called from the GUI thread... + # assert handler.get_gui_thread() != threading.current_thread(), 'must not be called from GUI thread' while True: infos = self.unpaired_device_infos(handler, plugin, devices) if infos: diff --git a/electrum/plugins/hw_wallet/plugin.py b/electrum/plugins/hw_wallet/plugin.py index 19e06a077..3080b9f0c 100644 --- a/electrum/plugins/hw_wallet/plugin.py +++ b/electrum/plugins/hw_wallet/plugin.py @@ -36,6 +36,7 @@ from electrum.keystore import Xpub, Hardware_KeyStore if TYPE_CHECKING: + import threading from electrum.wallet import Abstract_Wallet from electrum.base_wizard import BaseWizard @@ -210,6 +211,11 @@ def get_wallet(self) -> Optional['Abstract_Wallet']: if hasattr(self.win, 'wallet'): return self.win.wallet + def get_gui_thread(self) -> Optional['threading.Thread']: + if self.win is not None: + if hasattr(self.win, 'gui_thread'): + return self.win.gui_thread + def update_status(self, paired: bool) -> None: pass diff --git a/electrum/plugins/hw_wallet/qt.py b/electrum/plugins/hw_wallet/qt.py index 8df94ad7c..c8432f29e 100644 --- a/electrum/plugins/hw_wallet/qt.py +++ b/electrum/plugins/hw_wallet/qt.py @@ -67,6 +67,7 @@ class QtHandlerBase(HardwareHandlerBase, QObject, Logger): def __init__(self, win: Union[ElectrumWindow, InstallWizard], device: str): QObject.__init__(self) Logger.__init__(self) + assert win.gui_thread == threading.current_thread(), 'must be called from GUI thread' self.clear_signal.connect(self.clear_dialog) self.error_signal.connect(self.error_dialog) self.message_signal.connect(self.message_dialog) @@ -254,6 +255,7 @@ def choose_device(self: Union['QtPluginBase', HW_PluginBase], window: ElectrumWi keystore: 'Hardware_KeyStore') -> Optional[str]: '''This dialog box should be usable even if the user has forgotten their PIN or it is in bootloader mode.''' + assert window.gui_thread != threading.current_thread(), 'must not be called from GUI thread' device_id = self.device_manager().xpub_id(keystore.xpub) if not device_id: try: @@ -264,7 +266,10 @@ def choose_device(self: Union['QtPluginBase', HW_PluginBase], window: ElectrumWi return device_id def show_settings_dialog(self, window: ElectrumWindow, keystore: 'Hardware_KeyStore') -> None: - device_id = self.choose_device(window, keystore) + # default implementation (if no dialog): just try to connect to device + def connect(): + device_id = self.choose_device(window, keystore) + keystore.thread.add(connect) def add_show_address_on_hw_device_button_for_receive_addr(self, wallet: 'Abstract_Wallet', keystore: 'Hardware_KeyStore', diff --git a/electrum/plugins/keepkey/qt.py b/electrum/plugins/keepkey/qt.py index fce633df9..72508ee73 100644 --- a/electrum/plugins/keepkey/qt.py +++ b/electrum/plugins/keepkey/qt.py @@ -208,9 +208,13 @@ def show_address(): menu.addAction(_("Show on {}").format(device_name), show_address) def show_settings_dialog(self, window, keystore): - device_id = self.choose_device(window, keystore) - if device_id: - SettingsDialog(window, self, keystore, device_id).exec_() + def connect(): + device_id = self.choose_device(window, keystore) + return device_id + def show_dialog(device_id): + if device_id: + SettingsDialog(window, self, keystore, device_id).exec_() + keystore.thread.add(connect, on_success=show_dialog) def request_trezor_init_settings(self, wizard, method, device): vbox = QVBoxLayout() diff --git a/electrum/plugins/safe_t/qt.py b/electrum/plugins/safe_t/qt.py index aa44495bb..d83663d53 100644 --- a/electrum/plugins/safe_t/qt.py +++ b/electrum/plugins/safe_t/qt.py @@ -84,9 +84,13 @@ def show_address(keystore=keystore): menu.addAction(_("Show on {}").format(device_name), show_address) def show_settings_dialog(self, window, keystore): - device_id = self.choose_device(window, keystore) - if device_id: - SettingsDialog(window, self, keystore, device_id).exec_() + def connect(): + device_id = self.choose_device(window, keystore) + return device_id + def show_dialog(device_id): + if device_id: + SettingsDialog(window, self, keystore, device_id).exec_() + keystore.thread.add(connect, on_success=show_dialog) def request_safe_t_init_settings(self, wizard, method, device): vbox = QVBoxLayout() diff --git a/electrum/plugins/trezor/qt.py b/electrum/plugins/trezor/qt.py index 37a45ce6b..1bae9039e 100644 --- a/electrum/plugins/trezor/qt.py +++ b/electrum/plugins/trezor/qt.py @@ -182,9 +182,13 @@ def show_address(keystore=keystore): menu.addAction(_("Show on {}").format(device_name), show_address) def show_settings_dialog(self, window, keystore): - device_id = self.choose_device(window, keystore) - if device_id: - SettingsDialog(window, self, keystore, device_id).exec_() + def connect(): + device_id = self.choose_device(window, keystore) + return device_id + def show_dialog(device_id): + if device_id: + SettingsDialog(window, self, keystore, device_id).exec_() + keystore.thread.add(connect, on_success=show_dialog) def request_trezor_init_settings(self, wizard, method, device_id): vbox = QVBoxLayout() From 18c98483acc6adb13b22dfd6d356dac5128cd8e5 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Wed, 1 Apr 2020 18:49:45 +0200 Subject: [PATCH 39/69] wizard: (trivial) add some type hints --- electrum/base_wizard.py | 2 +- electrum/gui/qt/installwizard.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/electrum/base_wizard.py b/electrum/base_wizard.py index 80df02316..a80638483 100644 --- a/electrum/base_wizard.py +++ b/electrum/base_wizard.py @@ -144,7 +144,7 @@ def new(self): self.choice_dialog(title=title, message=message, choices=choices, run_next=self.on_wallet_type) def upgrade_db(self, storage, db): - exc = None + exc = None # type: Optional[Exception] def on_finished(): if exc is None: self.terminate(storage=storage, db=db) diff --git a/electrum/gui/qt/installwizard.py b/electrum/gui/qt/installwizard.py index 961f88916..3d139a0cc 100644 --- a/electrum/gui/qt/installwizard.py +++ b/electrum/gui/qt/installwizard.py @@ -31,6 +31,7 @@ if TYPE_CHECKING: from electrum.simple_config import SimpleConfig + from electrum.wallet_db import WalletDB from . import ElectrumGui @@ -321,7 +322,7 @@ def on_filename(filename): return temp_storage.path, (temp_storage if temp_storage.file_exists() else None) - def run_upgrades(self, storage, db): + def run_upgrades(self, storage: WalletStorage, db: 'WalletDB') -> None: path = storage.path if db.requires_split(): self.hide() @@ -360,8 +361,6 @@ def run_upgrades(self, storage, db): if db.requires_upgrade(): self.upgrade_db(storage, db) - return db - def on_error(self, exc_info): if not isinstance(exc_info[1], UserCancelled): self.logger.error("on_error", exc_info=exc_info) From 81fc3fcce2413bae55fc3e80a43744f7743745aa Mon Sep 17 00:00:00 2001 From: SomberNight Date: Wed, 1 Apr 2020 20:22:39 +0200 Subject: [PATCH 40/69] hww: rm some code duplication: add "scan_and_create_client_for_device" --- electrum/plugins/coldcard/coldcard.py | 11 ++--------- electrum/plugins/digitalbitbox/digitalbitbox.py | 11 ++--------- electrum/plugins/hw_wallet/plugin.py | 9 +++++++++ electrum/plugins/keepkey/keepkey.py | 11 ++--------- electrum/plugins/ledger/ledger.py | 11 ++--------- electrum/plugins/safe_t/safe_t.py | 11 ++--------- electrum/plugins/trezor/trezor.py | 11 ++--------- 7 files changed, 21 insertions(+), 54 deletions(-) diff --git a/electrum/plugins/coldcard/coldcard.py b/electrum/plugins/coldcard/coldcard.py index 3a391993c..72bf07bf6 100644 --- a/electrum/plugins/coldcard/coldcard.py +++ b/electrum/plugins/coldcard/coldcard.py @@ -522,22 +522,15 @@ def create_client(self, device, handler): return None def setup_device(self, device_info, wizard, purpose): - devmgr = self.device_manager() device_id = device_info.device.id_ - client = devmgr.client_by_id(device_id) - if client is None: - raise UserFacingException(_('Failed to create a client for this device.') + '\n' + - _('Make sure it is in the correct state.')) - client.handler = self.create_handler(wizard) + client = self.scan_and_create_client_for_device(device_id=device_id, wizard=wizard) def get_xpub(self, device_id, derivation, xtype, wizard): # this seems to be part of the pairing process only, not during normal ops? # base_wizard:on_hw_derivation if xtype not in self.SUPPORTED_XTYPES: raise ScriptTypeNotSupported(_('This type of script is not supported with {}.').format(self.device)) - devmgr = self.device_manager() - client = devmgr.client_by_id(device_id) - client.handler = self.create_handler(wizard) + client = self.scan_and_create_client_for_device(device_id=device_id, wizard=wizard) client.ping_check() xpub = client.get_xpub(derivation, xtype) diff --git a/electrum/plugins/digitalbitbox/digitalbitbox.py b/electrum/plugins/digitalbitbox/digitalbitbox.py index 80f9a87c1..c801460fa 100644 --- a/electrum/plugins/digitalbitbox/digitalbitbox.py +++ b/electrum/plugins/digitalbitbox/digitalbitbox.py @@ -703,13 +703,8 @@ def create_client(self, device, handler): def setup_device(self, device_info, wizard, purpose): - devmgr = self.device_manager() device_id = device_info.device.id_ - client = devmgr.client_by_id(device_id) - if client is None: - raise Exception(_('Failed to create a client for this device.') + '\n' + - _('Make sure it is in the correct state.')) - client.handler = self.create_handler(wizard) + client = self.scan_and_create_client_for_device(device_id=device_id, wizard=wizard) if purpose == HWD_SETUP_NEW_WALLET: client.setupRunning = True client.get_xpub("m/44'/0'", 'standard') @@ -739,9 +734,7 @@ def get_xpub(self, device_id, derivation, xtype, wizard): raise ScriptTypeNotSupported(_('This type of script is not supported with {}.').format(self.device)) if is_all_public_derivation(derivation): raise Exception(f"The {self.device} does not reveal xpubs corresponding to non-hardened paths. (path: {derivation})") - devmgr = self.device_manager() - client = devmgr.client_by_id(device_id) - client.handler = self.create_handler(wizard) + client = self.scan_and_create_client_for_device(device_id=device_id, wizard=wizard) client.check_device_dialog() xpub = client.get_xpub(derivation, xtype) return xpub diff --git a/electrum/plugins/hw_wallet/plugin.py b/electrum/plugins/hw_wallet/plugin.py index 3080b9f0c..14a2a846a 100644 --- a/electrum/plugins/hw_wallet/plugin.py +++ b/electrum/plugins/hw_wallet/plugin.py @@ -65,6 +65,15 @@ def close_wallet(self, wallet: 'Abstract_Wallet'): if isinstance(keystore, self.keystore_class): self.device_manager().unpair_xpub(keystore.xpub) + def scan_and_create_client_for_device(self, *, device_id: str, wizard: 'BaseWizard') -> 'HardwareClientBase': + devmgr = self.device_manager() + client = devmgr.client_by_id(device_id) + if client is None: + raise UserFacingException(_('Failed to create a client for this device.') + '\n' + + _('Make sure it is in the correct state.')) + client.handler = self.create_handler(wizard) + return client + def setup_device(self, device_info: DeviceInfo, wizard: 'BaseWizard', purpose): """Called when creating a new wallet or when using the device to decrypt an existing wallet. Select the device to use. If the device is diff --git a/electrum/plugins/keepkey/keepkey.py b/electrum/plugins/keepkey/keepkey.py index ac87bd00a..07ff0889d 100644 --- a/electrum/plugins/keepkey/keepkey.py +++ b/electrum/plugins/keepkey/keepkey.py @@ -275,13 +275,8 @@ def _make_node_path(self, xpub, address_n): return self.types.HDNodePathType(node=node, address_n=address_n) def setup_device(self, device_info, wizard, purpose): - devmgr = self.device_manager() device_id = device_info.device.id_ - client = devmgr.client_by_id(device_id) - if client is None: - raise UserFacingException(_('Failed to create a client for this device.') + '\n' + - _('Make sure it is in the correct state.')) - client.handler = self.create_handler(wizard) + client = self.scan_and_create_client_for_device(device_id=device_id, wizard=wizard) if not device_info.initialized: self.initialize_device(device_id, wizard, client.handler) client.get_xpub('m', 'standard') @@ -290,9 +285,7 @@ def setup_device(self, device_info, wizard, purpose): def get_xpub(self, device_id, derivation, xtype, wizard): if xtype not in self.SUPPORTED_XTYPES: raise ScriptTypeNotSupported(_('This type of script is not supported with {}.').format(self.device)) - devmgr = self.device_manager() - client = devmgr.client_by_id(device_id) - client.handler = self.create_handler(wizard) + client = self.scan_and_create_client_for_device(device_id=device_id, wizard=wizard) xpub = client.get_xpub(derivation, xtype) client.used() return xpub diff --git a/electrum/plugins/ledger/ledger.py b/electrum/plugins/ledger/ledger.py index 1c2e429f9..79d8638f3 100644 --- a/electrum/plugins/ledger/ledger.py +++ b/electrum/plugins/ledger/ledger.py @@ -589,21 +589,14 @@ def create_client(self, device, handler): return client def setup_device(self, device_info, wizard, purpose): - devmgr = self.device_manager() device_id = device_info.device.id_ - client = devmgr.client_by_id(device_id) - if client is None: - raise UserFacingException(_('Failed to create a client for this device.') + '\n' + - _('Make sure it is in the correct state.')) - client.handler = self.create_handler(wizard) + client = self.scan_and_create_client_for_device(device_id=device_id, wizard=wizard) client.get_xpub("m/44'/0'", 'standard') # TODO replace by direct derivation once Nano S > 1.1 def get_xpub(self, device_id, derivation, xtype, wizard): if xtype not in self.SUPPORTED_XTYPES: raise ScriptTypeNotSupported(_('This type of script is not supported with {}.').format(self.device)) - devmgr = self.device_manager() - client = devmgr.client_by_id(device_id) - client.handler = self.create_handler(wizard) + client = self.scan_and_create_client_for_device(device_id=device_id, wizard=wizard) client.checkDevice() xpub = client.get_xpub(derivation, xtype) return xpub diff --git a/electrum/plugins/safe_t/safe_t.py b/electrum/plugins/safe_t/safe_t.py index 1deb7345b..6d4711f05 100644 --- a/electrum/plugins/safe_t/safe_t.py +++ b/electrum/plugins/safe_t/safe_t.py @@ -249,13 +249,8 @@ def _make_node_path(self, xpub, address_n): return self.types.HDNodePathType(node=node, address_n=address_n) def setup_device(self, device_info, wizard, purpose): - devmgr = self.device_manager() device_id = device_info.device.id_ - client = devmgr.client_by_id(device_id) - if client is None: - raise UserFacingException(_('Failed to create a client for this device.') + '\n' + - _('Make sure it is in the correct state.')) - client.handler = self.create_handler(wizard) + client = self.scan_and_create_client_for_device(device_id=device_id, wizard=wizard) if not device_info.initialized: self.initialize_device(device_id, wizard, client.handler) client.get_xpub('m', 'standard') @@ -264,9 +259,7 @@ def setup_device(self, device_info, wizard, purpose): def get_xpub(self, device_id, derivation, xtype, wizard): if xtype not in self.SUPPORTED_XTYPES: raise ScriptTypeNotSupported(_('This type of script is not supported with {}.').format(self.device)) - devmgr = self.device_manager() - client = devmgr.client_by_id(device_id) - client.handler = self.create_handler(wizard) + client = self.scan_and_create_client_for_device(device_id=device_id, wizard=wizard) xpub = client.get_xpub(derivation, xtype) client.used() return xpub diff --git a/electrum/plugins/trezor/trezor.py b/electrum/plugins/trezor/trezor.py index b2a638cd5..2c49e621c 100644 --- a/electrum/plugins/trezor/trezor.py +++ b/electrum/plugins/trezor/trezor.py @@ -268,12 +268,8 @@ def _make_node_path(self, xpub, address_n): return HDNodePathType(node=node, address_n=address_n) def setup_device(self, device_info, wizard, purpose): - devmgr = self.device_manager() device_id = device_info.device.id_ - client = devmgr.client_by_id(device_id) - if client is None: - raise UserFacingException(_('Failed to create a client for this device.') + '\n' + - _('Make sure it is in the correct state.')) + client = self.scan_and_create_client_for_device(device_id=device_id, wizard=wizard) if not client.is_uptodate(): msg = (_('Outdated {} firmware for device labelled {}. Please ' @@ -281,7 +277,6 @@ def setup_device(self, device_info, wizard, purpose): .format(self.device, client.label(), self.firmware_URL)) raise OutdatedHwFirmwareException(msg) - client.handler = self.create_handler(wizard) if not device_info.initialized: self.initialize_device(device_id, wizard, client.handler) is_creating_wallet = purpose == HWD_SETUP_NEW_WALLET @@ -291,9 +286,7 @@ def setup_device(self, device_info, wizard, purpose): def get_xpub(self, device_id, derivation, xtype, wizard): if xtype not in self.SUPPORTED_XTYPES: raise ScriptTypeNotSupported(_('This type of script is not supported with {}.').format(self.device)) - devmgr = self.device_manager() - client = devmgr.client_by_id(device_id) - client.handler = self.create_handler(wizard) + client = self.scan_and_create_client_for_device(device_id=device_id, wizard=wizard) xpub = client.get_xpub(derivation, xtype) client.used() return xpub From 371f55a0f922cf57196ae3d8d7c2fabc7a9f0f08 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Wed, 1 Apr 2020 21:05:19 +0200 Subject: [PATCH 41/69] hww: fix some threading issues in wizard fixes #3377 related: #6064 (passphrase dialog not rendered correctly) --- electrum/base_wizard.py | 8 ++++++++ electrum/gui/qt/installwizard.py | 20 +++++++++++++++++++ .../plugins/digitalbitbox/digitalbitbox.py | 3 ++- electrum/plugins/hw_wallet/plugin.py | 2 ++ electrum/plugins/keepkey/keepkey.py | 3 ++- electrum/plugins/ledger/ledger.py | 3 ++- electrum/plugins/safe_t/safe_t.py | 3 ++- electrum/plugins/trezor/trezor.py | 3 ++- 8 files changed, 40 insertions(+), 5 deletions(-) diff --git a/electrum/base_wizard.py b/electrum/base_wizard.py index a80638483..a6f898ec4 100644 --- a/electrum/base_wizard.py +++ b/electrum/base_wizard.py @@ -158,6 +158,13 @@ def do_upgrade(): exc = e self.waiting_dialog(do_upgrade, _('Upgrading wallet format...'), on_finished=on_finished) + def run_task_without_blocking_gui(self, task, *, msg: str = None) -> Any: + """Perform a task in a thread without blocking the GUI. + Returns the result of 'task', or raises the same exception. + This method blocks until 'task' is finished. + """ + raise NotImplementedError() + def load_2fa(self): self.data['wallet_type'] = '2fa' self.data['use_trustedcoin'] = True @@ -421,6 +428,7 @@ def derivation_and_script_type_dialog(self, f): def on_hw_derivation(self, name, device_info: 'DeviceInfo', derivation, xtype): from .keystore import hardware_keystore devmgr = self.plugins.device_manager + assert isinstance(self.plugin, HW_PluginBase) try: xpub = self.plugin.get_xpub(device_info.device.id_, derivation, xtype, self) client = devmgr.client_by_id(device_info.device.id_) diff --git a/electrum/gui/qt/installwizard.py b/electrum/gui/qt/installwizard.py index 3d139a0cc..e791ac296 100644 --- a/electrum/gui/qt/installwizard.py +++ b/electrum/gui/qt/installwizard.py @@ -529,6 +529,26 @@ def waiting_dialog(self, task, msg, on_finished=None): if on_finished: on_finished() + def run_task_without_blocking_gui(self, task, *, msg=None): + assert self.gui_thread == threading.current_thread(), 'must be called from GUI thread' + if msg is None: + msg = _("Please wait...") + + exc = None # type: Optional[Exception] + res = None + def task_wrapper(): + nonlocal exc + nonlocal res + try: + task() + except Exception as e: + exc = e + self.waiting_dialog(task_wrapper, msg=msg) + if exc is None: + return res + else: + raise exc + @wizard_dialog def choice_dialog(self, title, message, choices, run_next): c_values = [x[0] for x in choices] diff --git a/electrum/plugins/digitalbitbox/digitalbitbox.py b/electrum/plugins/digitalbitbox/digitalbitbox.py index c801460fa..bf8294254 100644 --- a/electrum/plugins/digitalbitbox/digitalbitbox.py +++ b/electrum/plugins/digitalbitbox/digitalbitbox.py @@ -707,7 +707,8 @@ def setup_device(self, device_info, wizard, purpose): client = self.scan_and_create_client_for_device(device_id=device_id, wizard=wizard) if purpose == HWD_SETUP_NEW_WALLET: client.setupRunning = True - client.get_xpub("m/44'/0'", 'standard') + wizard.run_task_without_blocking_gui( + task=lambda: client.get_xpub("m/44'/0'", 'standard')) def is_mobile_paired(self): diff --git a/electrum/plugins/hw_wallet/plugin.py b/electrum/plugins/hw_wallet/plugin.py index 14a2a846a..f1e28c6eb 100644 --- a/electrum/plugins/hw_wallet/plugin.py +++ b/electrum/plugins/hw_wallet/plugin.py @@ -78,6 +78,8 @@ def setup_device(self, device_info: DeviceInfo, wizard: 'BaseWizard', purpose): """Called when creating a new wallet or when using the device to decrypt an existing wallet. Select the device to use. If the device is uninitialized, go through the initialization process. + + Runs in GUI thread. """ raise NotImplementedError() diff --git a/electrum/plugins/keepkey/keepkey.py b/electrum/plugins/keepkey/keepkey.py index 07ff0889d..3c76ef09a 100644 --- a/electrum/plugins/keepkey/keepkey.py +++ b/electrum/plugins/keepkey/keepkey.py @@ -279,7 +279,8 @@ def setup_device(self, device_info, wizard, purpose): client = self.scan_and_create_client_for_device(device_id=device_id, wizard=wizard) if not device_info.initialized: self.initialize_device(device_id, wizard, client.handler) - client.get_xpub('m', 'standard') + wizard.run_task_without_blocking_gui( + task=lambda: client.get_xpub("m", 'standard')) client.used() def get_xpub(self, device_id, derivation, xtype, wizard): diff --git a/electrum/plugins/ledger/ledger.py b/electrum/plugins/ledger/ledger.py index 79d8638f3..8e0040f5a 100644 --- a/electrum/plugins/ledger/ledger.py +++ b/electrum/plugins/ledger/ledger.py @@ -591,7 +591,8 @@ def create_client(self, device, handler): def setup_device(self, device_info, wizard, purpose): device_id = device_info.device.id_ client = self.scan_and_create_client_for_device(device_id=device_id, wizard=wizard) - client.get_xpub("m/44'/0'", 'standard') # TODO replace by direct derivation once Nano S > 1.1 + wizard.run_task_without_blocking_gui( + task=lambda: client.get_xpub("m/44'/0'", 'standard')) # TODO replace by direct derivation once Nano S > 1.1 def get_xpub(self, device_id, derivation, xtype, wizard): if xtype not in self.SUPPORTED_XTYPES: diff --git a/electrum/plugins/safe_t/safe_t.py b/electrum/plugins/safe_t/safe_t.py index 6d4711f05..4bf6381e2 100644 --- a/electrum/plugins/safe_t/safe_t.py +++ b/electrum/plugins/safe_t/safe_t.py @@ -253,7 +253,8 @@ def setup_device(self, device_info, wizard, purpose): client = self.scan_and_create_client_for_device(device_id=device_id, wizard=wizard) if not device_info.initialized: self.initialize_device(device_id, wizard, client.handler) - client.get_xpub('m', 'standard') + wizard.run_task_without_blocking_gui( + task=lambda: client.get_xpub("m", 'standard')) client.used() def get_xpub(self, device_id, derivation, xtype, wizard): diff --git a/electrum/plugins/trezor/trezor.py b/electrum/plugins/trezor/trezor.py index 2c49e621c..f32cc5227 100644 --- a/electrum/plugins/trezor/trezor.py +++ b/electrum/plugins/trezor/trezor.py @@ -280,7 +280,8 @@ def setup_device(self, device_info, wizard, purpose): if not device_info.initialized: self.initialize_device(device_id, wizard, client.handler) is_creating_wallet = purpose == HWD_SETUP_NEW_WALLET - client.get_xpub('m', 'standard', creating=is_creating_wallet) + wizard.run_task_without_blocking_gui( + task=lambda: client.get_xpub('m', 'standard', creating=is_creating_wallet)) client.used() def get_xpub(self, device_id, derivation, xtype, wizard): From 4c10a830f3243d154b440f0250ec7399dd4cb733 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Thu, 12 Mar 2020 01:44:42 +0100 Subject: [PATCH 42/69] lnmsg: rewrite LN msg encoding/decoding --- electrum/channel_db.py | 4 +- electrum/lightning.json | 903 --------------------------------- electrum/lnchannel.py | 5 +- electrum/lnmsg.py | 343 ++++++++----- electrum/lnpeer.py | 14 +- electrum/lnwire/README.md | 5 + electrum/lnwire/onion_wire.csv | 53 ++ electrum/lnwire/peer_wire.csv | 210 ++++++++ 8 files changed, 489 insertions(+), 1048 deletions(-) delete mode 100644 electrum/lightning.json create mode 100644 electrum/lnwire/README.md create mode 100644 electrum/lnwire/onion_wire.csv create mode 100644 electrum/lnwire/peer_wire.csv diff --git a/electrum/channel_db.py b/electrum/channel_db.py index 66dec951e..54fc45460 100644 --- a/electrum/channel_db.py +++ b/electrum/channel_db.py @@ -321,11 +321,12 @@ def get_recent_peers(self): return ret # note: currently channel announcements are trusted by default (trusted=True); - # they are not verified. Verifying them would make the gossip sync + # they are not SPV-verified. Verifying them would make the gossip sync # even slower; especially as servers will start throttling us. # It would probably put significant strain on servers if all clients # verified the complete gossip. def add_channel_announcement(self, msg_payloads, *, trusted=True): + # note: signatures have already been verified. if type(msg_payloads) is dict: msg_payloads = [msg_payloads] added = 0 @@ -499,6 +500,7 @@ def verify_channel_update(self, payload): raise Exception(f'failed verifying channel update for {short_channel_id}') def add_node_announcement(self, msg_payloads): + # note: signatures have already been verified. if type(msg_payloads) is dict: msg_payloads = [msg_payloads] new_nodes = {} diff --git a/electrum/lightning.json b/electrum/lightning.json deleted file mode 100644 index d6794e67c..000000000 --- a/electrum/lightning.json +++ /dev/null @@ -1,903 +0,0 @@ -{ - "init": { - "type": "16", - "payload": { - "gflen": { - "position": "0", - "length": "2" - }, - "globalfeatures": { - "position": "2", - "length": "gflen" - }, - "lflen": { - "position": "2+gflen", - "length": "2" - }, - "localfeatures": { - "position": "4+gflen", - "length": "lflen" - } - } - }, - "error": { - "type": "17", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "len": { - "position": "32", - "length": "2" - }, - "data": { - "position": "34", - "length": "len" - } - } - }, - "ping": { - "type": "18", - "payload": { - "num_pong_bytes": { - "position": "0", - "length": "2" - }, - "byteslen": { - "position": "2", - "length": "2" - }, - "ignored": { - "position": "4", - "length": "byteslen" - } - } - }, - "pong": { - "type": "19", - "payload": { - "byteslen": { - "position": "0", - "length": "2" - }, - "ignored": { - "position": "2", - "length": "byteslen" - } - } - }, - "open_channel": { - "type": "32", - "payload": { - "chain_hash": { - "position": "0", - "length": "32" - }, - "temporary_channel_id": { - "position": "32", - "length": "32" - }, - "funding_satoshis": { - "position": "64", - "length": "8" - }, - "push_msat": { - "position": "72", - "length": "8" - }, - "dust_limit_satoshis": { - "position": "80", - "length": "8" - }, - "max_htlc_value_in_flight_msat": { - "position": "88", - "length": "8" - }, - "channel_reserve_satoshis": { - "position": "96", - "length": "8" - }, - "htlc_minimum_msat": { - "position": "104", - "length": "8" - }, - "feerate_per_kw": { - "position": "112", - "length": "4" - }, - "to_self_delay": { - "position": "116", - "length": "2" - }, - "max_accepted_htlcs": { - "position": "118", - "length": "2" - }, - "funding_pubkey": { - "position": "120", - "length": "33" - }, - "revocation_basepoint": { - "position": "153", - "length": "33" - }, - "payment_basepoint": { - "position": "186", - "length": "33" - }, - "delayed_payment_basepoint": { - "position": "219", - "length": "33" - }, - "htlc_basepoint": { - "position": "252", - "length": "33" - }, - "first_per_commitment_point": { - "position": "285", - "length": "33" - }, - "channel_flags": { - "position": "318", - "length": "1" - }, - "shutdown_len": { - "position": "319", - "length": "2", - "feature": "option_upfront_shutdown_script" - }, - "shutdown_scriptpubkey": { - "position": "321", - "length": "shutdown_len", - "feature": "option_upfront_shutdown_script" - } - } - }, - "accept_channel": { - "type": "33", - "payload": { - "temporary_channel_id": { - "position": "0", - "length": "32" - }, - "dust_limit_satoshis": { - "position": "32", - "length": "8" - }, - "max_htlc_value_in_flight_msat": { - "position": "40", - "length": "8" - }, - "channel_reserve_satoshis": { - "position": "48", - "length": "8" - }, - "htlc_minimum_msat": { - "position": "56", - "length": "8" - }, - "minimum_depth": { - "position": "64", - "length": "4" - }, - "to_self_delay": { - "position": "68", - "length": "2" - }, - "max_accepted_htlcs": { - "position": "70", - "length": "2" - }, - "funding_pubkey": { - "position": "72", - "length": "33" - }, - "revocation_basepoint": { - "position": "105", - "length": "33" - }, - "payment_basepoint": { - "position": "138", - "length": "33" - }, - "delayed_payment_basepoint": { - "position": "171", - "length": "33" - }, - "htlc_basepoint": { - "position": "204", - "length": "33" - }, - "first_per_commitment_point": { - "position": "237", - "length": "33" - }, - "shutdown_len": { - "position": "270", - "length": "2", - "feature": "option_upfront_shutdown_script" - }, - "shutdown_scriptpubkey": { - "position": "272", - "length": "shutdown_len", - "feature": "option_upfront_shutdown_script" - } - } - }, - "funding_created": { - "type": "34", - "payload": { - "temporary_channel_id": { - "position": "0", - "length": "32" - }, - "funding_txid": { - "position": "32", - "length": "32" - }, - "funding_output_index": { - "position": "64", - "length": "2" - }, - "signature": { - "position": "66", - "length": "64" - } - } - }, - "funding_signed": { - "type": "35", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "signature": { - "position": "32", - "length": "64" - } - } - }, - "funding_locked": { - "type": "36", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "next_per_commitment_point": { - "position": "32", - "length": "33" - } - } - }, - "shutdown": { - "type": "38", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "len": { - "position": "32", - "length": "2" - }, - "scriptpubkey": { - "position": "34", - "length": "len" - } - } - }, - "closing_signed": { - "type": "39", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "fee_satoshis": { - "position": "32", - "length": "8" - }, - "signature": { - "position": "40", - "length": "64" - } - } - }, - "update_add_htlc": { - "type": "128", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "id": { - "position": "32", - "length": "8" - }, - "amount_msat": { - "position": "40", - "length": "8" - }, - "payment_hash": { - "position": "48", - "length": "32" - }, - "cltv_expiry": { - "position": "80", - "length": "4" - }, - "onion_routing_packet": { - "position": "84", - "length": "1366" - } - } - }, - "update_fulfill_htlc": { - "type": "130", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "id": { - "position": "32", - "length": "8" - }, - "payment_preimage": { - "position": "40", - "length": "32" - } - } - }, - "update_fail_htlc": { - "type": "131", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "id": { - "position": "32", - "length": "8" - }, - "len": { - "position": "40", - "length": "2" - }, - "reason": { - "position": "42", - "length": "len" - } - } - }, - "update_fail_malformed_htlc": { - "type": "135", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "id": { - "position": "32", - "length": "8" - }, - "sha256_of_onion": { - "position": "40", - "length": "32" - }, - "failure_code": { - "position": "72", - "length": "2" - } - } - }, - "commitment_signed": { - "type": "132", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "signature": { - "position": "32", - "length": "64" - }, - "num_htlcs": { - "position": "96", - "length": "2" - }, - "htlc_signature": { - "position": "98", - "length": "num_htlcs*64" - } - } - }, - "revoke_and_ack": { - "type": "133", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "per_commitment_secret": { - "position": "32", - "length": "32" - }, - "next_per_commitment_point": { - "position": "64", - "length": "33" - } - } - }, - "update_fee": { - "type": "134", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "feerate_per_kw": { - "position": "32", - "length": "4" - } - } - }, - "channel_reestablish": { - "type": "136", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "next_local_commitment_number": { - "position": "32", - "length": "8" - }, - "next_remote_revocation_number": { - "position": "40", - "length": "8" - }, - "your_last_per_commitment_secret": { - "position": "48", - "length": "32", - "feature": "option_data_loss_protect" - }, - "my_current_per_commitment_point": { - "position": "80", - "length": "33", - "feature": "option_data_loss_protect" - } - } - }, - "invalid_realm": { - "type": "PERM|1", - "payload": {} - }, - "temporary_node_failure": { - "type": "NODE|2", - "payload": {} - }, - "permanent_node_failure": { - "type": "PERM|NODE|2", - "payload": {} - }, - "required_node_feature_missing": { - "type": "PERM|NODE|3", - "payload": {} - }, - "invalid_onion_version": { - "type": "BADONION|PERM|4", - "payload": { - "sha256_of_onion": { - "position": "0", - "length": "32" - } - } - }, - "invalid_onion_hmac": { - "type": "BADONION|PERM|5", - "payload": { - "sha256_of_onion": { - "position": "0", - "length": "32" - } - } - }, - "invalid_onion_key": { - "type": "BADONION|PERM|6", - "payload": { - "sha256_of_onion": { - "position": "0", - "length": "32" - } - } - }, - "temporary_channel_failure": { - "type": "UPDATE|7", - "payload": { - "len": { - "position": "0", - "length": "2" - }, - "channel_update": { - "position": "2", - "length": "len" - } - } - }, - "permanent_channel_failure": { - "type": "PERM|8", - "payload": {} - }, - "required_channel_feature_missing": { - "type": "PERM|9", - "payload": {} - }, - "unknown_next_peer": { - "type": "PERM|10", - "payload": {} - }, - "amount_below_minimum": { - "type": "UPDATE|11", - "payload": { - "htlc_msat": { - "position": "0", - "length": "8" - }, - "len": { - "position": "8", - "length": "2" - }, - "channel_update": { - "position": "10", - "length": "len" - } - } - }, - "fee_insufficient": { - "type": "UPDATE|12", - "payload": { - "htlc_msat": { - "position": "0", - "length": "8" - }, - "len": { - "position": "8", - "length": "2" - }, - "channel_update": { - "position": "10", - "length": "len" - } - } - }, - "incorrect_cltv_expiry": { - "type": "UPDATE|13", - "payload": { - "cltv_expiry": { - "position": "0", - "length": "4" - }, - "len": { - "position": "4", - "length": "2" - }, - "channel_update": { - "position": "6", - "length": "len" - } - } - }, - "expiry_too_soon": { - "type": "UPDATE|14", - "payload": { - "len": { - "position": "0", - "length": "2" - }, - "channel_update": { - "position": "2", - "length": "len" - } - } - }, - "unknown_payment_hash": { - "type": "PERM|15", - "payload": {} - }, - "incorrect_payment_amount": { - "type": "PERM|16", - "payload": {} - }, - "final_expiry_too_soon": { - "type": "17", - "payload": {} - }, - "final_incorrect_cltv_expiry": { - "type": "18", - "payload": { - "cltv_expiry": { - "position": "0", - "length": "4" - } - } - }, - "final_incorrect_htlc_amount": { - "type": "19", - "payload": { - "incoming_htlc_amt": { - "position": "0", - "length": "8" - } - } - }, - "channel_disabled": { - "type": "UPDATE|20", - "payload": {} - }, - "expiry_too_far": { - "type": "21", - "payload": {} - }, - "announcement_signatures": { - "type": "259", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "short_channel_id": { - "position": "32", - "length": "8" - }, - "node_signature": { - "position": "40", - "length": "64" - }, - "bitcoin_signature": { - "position": "104", - "length": "64" - } - } - }, - "channel_announcement": { - "type": "256", - "payload": { - "node_signature_1": { - "position": "0", - "length": "64" - }, - "node_signature_2": { - "position": "64", - "length": "64" - }, - "bitcoin_signature_1": { - "position": "128", - "length": "64" - }, - "bitcoin_signature_2": { - "position": "192", - "length": "64" - }, - "len": { - "position": "256", - "length": "2" - }, - "features": { - "position": "258", - "length": "len" - }, - "chain_hash": { - "position": "258+len", - "length": "32" - }, - "short_channel_id": { - "position": "290+len", - "length": "8" - }, - "node_id_1": { - "position": "298+len", - "length": "33" - }, - "node_id_2": { - "position": "331+len", - "length": "33" - }, - "bitcoin_key_1": { - "position": "364+len", - "length": "33" - }, - "bitcoin_key_2": { - "position": "397+len", - "length": "33" - } - } - }, - "node_announcement": { - "type": "257", - "payload": { - "signature": { - "position": "0", - "length": "64" - }, - "flen": { - "position": "64", - "length": "2" - }, - "features": { - "position": "66", - "length": "flen" - }, - "timestamp": { - "position": "66+flen", - "length": "4" - }, - "node_id": { - "position": "70+flen", - "length": "33" - }, - "rgb_color": { - "position": "103+flen", - "length": "3" - }, - "alias": { - "position": "106+flen", - "length": "32" - }, - "addrlen": { - "position": "138+flen", - "length": "2" - }, - "addresses": { - "position": "140+flen", - "length": "addrlen" - } - } - }, - "channel_update": { - "type": "258", - "payload": { - "signature": { - "position": "0", - "length": "64" - }, - "chain_hash": { - "position": "64", - "length": "32" - }, - "short_channel_id": { - "position": "96", - "length": "8" - }, - "timestamp": { - "position": "104", - "length": "4" - }, - "message_flags": { - "position": "108", - "length": "1" - }, - "channel_flags": { - "position": "109", - "length": "1" - }, - "cltv_expiry_delta": { - "position": "110", - "length": "2" - }, - "htlc_minimum_msat": { - "position": "112", - "length": "8" - }, - "fee_base_msat": { - "position": "120", - "length": "4" - }, - "fee_proportional_millionths": { - "position": "124", - "length": "4" - }, - "htlc_maximum_msat": { - "position": "128", - "length": "8", - "feature": "option_channel_htlc_max" - } - } - }, - "query_short_channel_ids": { - "type": "261", - "payload": { - "chain_hash": { - "position": "0", - "length": "32" - }, - "len": { - "position": "32", - "length": "2" - }, - "encoded_short_ids": { - "position": "34", - "length": "len" - } - } - }, - "reply_short_channel_ids_end": { - "type": "262", - "payload": { - "chain_hash": { - "position": "0", - "length": "32" - }, - "complete": { - "position": "32", - "length": "1" - } - } - }, - "query_channel_range": { - "type": "263", - "payload": { - "chain_hash": { - "position": "0", - "length": "32" - }, - "first_blocknum": { - "position": "32", - "length": "4" - }, - "number_of_blocks": { - "position": "36", - "length": "4" - } - } - }, - "reply_channel_range": { - "type": "264", - "payload": { - "chain_hash": { - "position": "0", - "length": "32" - }, - "first_blocknum": { - "position": "32", - "length": "4" - }, - "number_of_blocks": { - "position": "36", - "length": "4" - }, - "complete": { - "position": "40", - "length": "1" - }, - "len": { - "position": "41", - "length": "2" - }, - "encoded_short_ids": { - "position": "43", - "length": "len" - } - } - }, - "gossip_timestamp_filter": { - "type": "265", - "payload": { - "chain_hash": { - "position": "0", - "length": "32" - }, - "first_timestamp": { - "position": "32", - "length": "4" - }, - "timestamp_range": { - "position": "36", - "length": "4" - } - } - } -} diff --git a/electrum/lnchannel.py b/electrum/lnchannel.py index d38bd4824..bd52175b7 100644 --- a/electrum/lnchannel.py +++ b/electrum/lnchannel.py @@ -249,7 +249,8 @@ def construct_channel_announcement_without_sigs(self) -> bytes: node_ids = sorted_node_ids bitcoin_keys.reverse() - chan_ann = encode_msg("channel_announcement", + chan_ann = encode_msg( + "channel_announcement", len=0, features=b'', chain_hash=constants.net.rev_genesis_bytes(), @@ -257,7 +258,7 @@ def construct_channel_announcement_without_sigs(self) -> bytes: node_id_1=node_ids[0], node_id_2=node_ids[1], bitcoin_key_1=bitcoin_keys[0], - bitcoin_key_2=bitcoin_keys[1] + bitcoin_key_2=bitcoin_keys[1], ) self._chan_ann_without_sigs = chan_ann diff --git a/electrum/lnmsg.py b/electrum/lnmsg.py index a755756b4..2ef48ae97 100644 --- a/electrum/lnmsg.py +++ b/electrum/lnmsg.py @@ -1,152 +1,225 @@ -import json import os -from typing import Callable, Tuple -from collections import OrderedDict - -def _eval_length_term(x, ma: dict) -> int: - """ - Evaluate a term of the simple language used - to specify lightning message field lengths. - - If `x` is an integer, it is returned as is, - otherwise it is treated as a variable and - looked up in `ma`. - - If the value in `ma` was no integer, it is - assumed big-endian bytes and decoded. - - Returns evaluated result as int - """ - try: - x = int(x) - except ValueError: - x = ma[x] - try: - x = int(x) - except ValueError: - x = int.from_bytes(x, byteorder='big') - return x - -def _eval_exp_with_ctx(exp, ctx: dict) -> int: - """ - Evaluate simple mathematical expression given - in `exp` with context (variables assigned) - from the dict `ctx`. - - Returns evaluated result as int - """ - exp = str(exp) - if "*" in exp: - assert "+" not in exp - result = 1 - for term in exp.split("*"): - result *= _eval_length_term(term, ctx) - return result - return sum(_eval_length_term(x, ctx) for x in exp.split("+")) - -def _make_handler(msg_name: str, v: dict) -> Callable[[bytes], Tuple[str, dict]]: - """ - Generate a message handler function (taking bytes) - for message type `msg_name` with specification `v` - - Check lib/lightning.json, `msg_name` could be 'init', - and `v` could be - - { type: 16, payload: { 'gflen': ..., ... }, ... } - - Returns function taking bytes - """ - def handler(data: bytes) -> Tuple[str, dict]: - ma = {} # map of field name -> field data; after parsing msg - pos = 0 - for fieldname in v["payload"]: - poslenMap = v["payload"][fieldname] - if "feature" in poslenMap and pos == len(data): - continue - #assert pos == _eval_exp_with_ctx(poslenMap["position"], ma) # this assert is expensive... - length = poslenMap["length"] - length = _eval_exp_with_ctx(length, ma) - ma[fieldname] = data[pos:pos+length] - pos += length - # BOLT-01: "MUST ignore any additional data within a message beyond the length that it expects for that type." - assert pos <= len(data), (msg_name, pos, len(data)) - return msg_name, ma - return handler +import csv +import io +from typing import Callable, Tuple, Any, Dict, List, Sequence, Union + + +class MalformedMsg(Exception): + pass + + +class UnknownMsgFieldType(MalformedMsg): + pass + + +class UnexpectedEndOfStream(MalformedMsg): + pass + + +def _assert_can_read_at_least_n_bytes(fd: io.BytesIO, n: int) -> None: + cur_pos = fd.tell() + end_pos = fd.seek(0, io.SEEK_END) + fd.seek(cur_pos) + if end_pos - cur_pos < n: + raise UnexpectedEndOfStream(f"cur_pos={cur_pos}. end_pos={end_pos}. wants to read: {n}") + + +# TODO return int when it makes sense +def _read_field(*, fd: io.BytesIO, field_type: str, count: int) -> bytes: + if not fd: raise Exception() + assert isinstance(count, int) and count >= 0, f"{count!r} must be non-neg int" + if count == 0: + return b"" + type_len = None + if field_type == 'byte': + type_len = 1 + elif field_type == 'u16': + type_len = 2 + elif field_type == 'u32': + type_len = 4 + elif field_type == 'u64': + type_len = 8 + # TODO tu16/tu32/tu64 + elif field_type == 'chain_hash': + type_len = 32 + elif field_type == 'channel_id': + type_len = 32 + elif field_type == 'sha256': + type_len = 32 + elif field_type == 'signature': + type_len = 64 + elif field_type == 'point': + type_len = 33 + elif field_type == 'short_channel_id': + type_len = 8 + if type_len is None: + raise UnknownMsgFieldType(f"unexpected field type: {field_type!r}") + total_len = count * type_len + _assert_can_read_at_least_n_bytes(fd, total_len) + return fd.read(total_len) + + +def _write_field(*, fd: io.BytesIO, field_type: str, count: int, + value: Union[bytes, int]) -> None: + if not fd: raise Exception() + assert isinstance(count, int) and count >= 0, f"{count!r} must be non-neg int" + if count == 0: + return + type_len = None + if field_type == 'byte': + type_len = 1 + elif field_type == 'u16': + type_len = 2 + elif field_type == 'u32': + type_len = 4 + elif field_type == 'u64': + type_len = 8 + # TODO tu16/tu32/tu64 + elif field_type == 'chain_hash': + type_len = 32 + elif field_type == 'channel_id': + type_len = 32 + elif field_type == 'sha256': + type_len = 32 + elif field_type == 'signature': + type_len = 64 + elif field_type == 'point': + type_len = 33 + elif field_type == 'short_channel_id': + type_len = 8 + if type_len is None: + raise UnknownMsgFieldType(f"unexpected fundamental type: {field_type!r}") + total_len = count * type_len + if isinstance(value, int) and (count == 1 or field_type == 'byte'): + value = int.to_bytes(value, length=total_len, byteorder="big", signed=False) + if not isinstance(value, (bytes, bytearray)): + raise Exception(f"can only write bytes into fd. got: {value!r}") + if total_len != len(value): + raise Exception(f"unexpected field size. expected: {total_len}, got {len(value)}") + nbytes_written = fd.write(value) + if nbytes_written != len(value): + raise Exception(f"tried to write {len(value)} bytes, but only wrote {nbytes_written}!?") + class LNSerializer: def __init__(self): - message_types = {} - path = os.path.join(os.path.dirname(__file__), 'lightning.json') - with open(path) as f: - structured = json.loads(f.read(), object_pairs_hook=OrderedDict) - - for msg_name in structured: - v = structured[msg_name] - # these message types are skipped since their types collide - # (for example with pong, which also uses type=19) - # we don't need them yet - if msg_name in ["final_incorrect_cltv_expiry", "final_incorrect_htlc_amount"]: - continue - if len(v["payload"]) == 0: - continue - try: - num = int(v["type"]) - except ValueError: - #print("skipping", k) - continue - byts = num.to_bytes(2, 'big') - assert byts not in message_types, (byts, message_types[byts].__name__, msg_name) - names = [x.__name__ for x in message_types.values()] - assert msg_name + "_handler" not in names, (msg_name, names) - message_types[byts] = _make_handler(msg_name, v) - message_types[byts].__name__ = msg_name + "_handler" - - assert message_types[b"\x00\x10"].__name__ == "init_handler" - self.structured = structured - self.message_types = message_types - - def encode_msg(self, msg_type : str, **kwargs) -> bytes: + self.msg_scheme_from_type = {} # type: Dict[bytes, List[Sequence[str]]] + self.msg_type_from_name = {} # type: Dict[str, bytes] + path = os.path.join(os.path.dirname(__file__), "lnwire", "peer_wire.csv") + with open(path, newline='') as f: + csvreader = csv.reader(f) + for row in csvreader: + #print(f">>> {row!r}") + if row[0] == "msgtype": + msg_type_name = row[1] + msg_type_int = int(row[2]) + msg_type_bytes = msg_type_int.to_bytes(2, 'big') + assert msg_type_bytes not in self.msg_scheme_from_type, f"type collision? for {msg_type_name}" + assert msg_type_name not in self.msg_type_from_name, f"type collision? for {msg_type_name}" + row[2] = msg_type_int + self.msg_scheme_from_type[msg_type_bytes] = [tuple(row)] + self.msg_type_from_name[msg_type_name] = msg_type_bytes + elif row[0] == "msgdata": + assert msg_type_name == row[1] + self.msg_scheme_from_type[msg_type_bytes].append(tuple(row)) + else: + pass # TODO + + def encode_msg(self, msg_type: str, **kwargs) -> bytes: """ Encode kwargs into a Lightning message (bytes) of the type given in the msg_type string """ - typ = self.structured[msg_type] - data = int(typ["type"]).to_bytes(2, 'big') - lengths = {} - for k in typ["payload"]: - poslenMap = typ["payload"][k] - if k not in kwargs and "feature" in poslenMap: - continue - param = kwargs.get(k, 0) - leng = _eval_exp_with_ctx(poslenMap["length"], lengths) - try: - clone = dict(lengths) - clone.update(kwargs) - leng = _eval_exp_with_ctx(poslenMap["length"], clone) - except KeyError: - pass - try: - if not isinstance(param, bytes): - assert isinstance(param, int), "field {} is neither bytes or int".format(k) - param = param.to_bytes(leng, 'big') - except ValueError: - raise Exception("{} does not fit in {} bytes".format(k, leng)) - lengths[k] = len(param) - if lengths[k] != leng: - raise Exception("field {} is {} bytes long, should be {} bytes long".format(k, lengths[k], leng)) - data += param - return data - - def decode_msg(self, data : bytes) -> Tuple[str, dict]: + #print(f">>> encode_msg. msg_type={msg_type}, payload={kwargs!r}") + msg_type_bytes = self.msg_type_from_name[msg_type] + scheme = self.msg_scheme_from_type[msg_type_bytes] + with io.BytesIO() as fd: + fd.write(msg_type_bytes) + for row in scheme: + if row[0] == "msgtype": + pass + elif row[0] == "msgdata": + field_name = row[2] + field_type = row[3] + field_count_str = row[4] + #print(f">>> encode_msg. msgdata. field_name={field_name!r}. field_type={field_type!r}. field_count_str={field_count_str!r}") + if field_count_str == "": + field_count = 1 + else: + try: + field_count = int(field_count_str) + except ValueError: + field_count = kwargs[field_count_str] + if isinstance(field_count, (bytes, bytearray)): + field_count = int.from_bytes(field_count, byteorder="big") + assert isinstance(field_count, int) + try: + field_value = kwargs[field_name] + except KeyError: + if len(row) > 5: + break # optional feature field not present + else: + field_value = 0 # default mandatory fields to zero + #print(f">>> encode_msg. writing field: {field_name}. value={field_value!r}. field_type={field_type!r}. count={field_count!r}") + try: + _write_field(fd=fd, + field_type=field_type, + count=field_count, + value=field_value) + #print(f">>> encode_msg. so far: {fd.getvalue().hex()}") + except UnknownMsgFieldType as e: + pass # TODO + else: + pass # TODO + return fd.getvalue() + + def decode_msg(self, data: bytes) -> Tuple[str, dict]: """ Decode Lightning message by reading the first two bytes to determine message type. Returns message type string and parsed message contents dict """ - typ = data[:2] - k, parsed = self.message_types[typ](data[2:]) - return k, parsed + #print(f"decode_msg >>> {data.hex()}") + assert len(data) >= 2 + msg_type_bytes = data[:2] + msg_type_int = int.from_bytes(msg_type_bytes, byteorder="big", signed=False) + scheme = self.msg_scheme_from_type[msg_type_bytes] + assert scheme[0][2] == msg_type_int + msg_type_name = scheme[0][1] + parsed = {} + with io.BytesIO(data[2:]) as fd: + for row in scheme: + #print(f"row: {row!r}") + if row[0] == "msgtype": + pass + elif row[0] == "msgdata": + field_name = row[2] + field_type = row[3] + field_count_str = row[4] + if field_count_str == "": + field_count = 1 + else: + try: + field_count = int(field_count_str) + except ValueError: + field_count = int.from_bytes(parsed[field_count_str], byteorder="big") + #print(f">> count={field_count}. parsed={parsed}") + try: + parsed[field_name] = _read_field(fd=fd, + field_type=field_type, + count=field_count) + except UnknownMsgFieldType as e: + pass # TODO + except UnexpectedEndOfStream as e: + if len(row) > 5: + break # optional feature field not present + else: + raise + else: + pass # TODO + return msg_type_name, parsed + _inst = LNSerializer() encode_msg = _inst.encode_msg diff --git a/electrum/lnpeer.py b/electrum/lnpeer.py index 91853884d..2b0aa87c1 100644 --- a/electrum/lnpeer.py +++ b/electrum/lnpeer.py @@ -131,7 +131,7 @@ def is_initialized(self) -> bool: async def initialize(self): if isinstance(self.transport, LNTransport): await self.transport.handshake() - self.send_message("init", gflen=0, lflen=2, localfeatures=self.localfeatures) + self.send_message("init", gflen=0, flen=2, features=self.localfeatures) self._sent_init = True self.maybe_set_initialized() @@ -201,7 +201,7 @@ def on_init(self, payload): return # if they required some even flag we don't have, they will close themselves # but if we require an even flag they don't have, we close - their_localfeatures = int.from_bytes(payload['localfeatures'], byteorder="big") + their_localfeatures = int.from_bytes(payload['features'], byteorder="big") # TODO feature bit unification try: self.localfeatures = ln_compare_features(self.localfeatures, their_localfeatures) except IncompatibleLightningFeatures as e: @@ -760,16 +760,16 @@ async def reestablish_channel(self, chan: Channel): self.send_message( "channel_reestablish", channel_id=chan_id, - next_local_commitment_number=next_local_ctn, - next_remote_revocation_number=oldest_unrevoked_remote_ctn, + next_commitment_number=next_local_ctn, + next_revocation_number=oldest_unrevoked_remote_ctn, your_last_per_commitment_secret=last_rev_secret, my_current_per_commitment_point=latest_point) self.logger.info(f'channel_reestablish ({chan.get_id_for_log()}): sent channel_reestablish with ' f'(next_local_ctn={next_local_ctn}, ' f'oldest_unrevoked_remote_ctn={oldest_unrevoked_remote_ctn})') msg = await self.wait_for_message('channel_reestablish', chan_id) - their_next_local_ctn = int.from_bytes(msg["next_local_commitment_number"], 'big') - their_oldest_unrevoked_remote_ctn = int.from_bytes(msg["next_remote_revocation_number"], 'big') + their_next_local_ctn = int.from_bytes(msg["next_commitment_number"], 'big') + their_oldest_unrevoked_remote_ctn = int.from_bytes(msg["next_revocation_number"], 'big') their_local_pcp = msg.get("my_current_per_commitment_point") their_claim_of_our_last_per_commitment_secret = msg.get("your_last_per_commitment_secret") self.logger.info(f'channel_reestablish ({chan.get_id_for_log()}): received channel_reestablish with ' @@ -818,7 +818,7 @@ async def reestablish_channel(self, chan: Channel): if oldest_unrevoked_local_ctn != their_oldest_unrevoked_remote_ctn: if oldest_unrevoked_local_ctn - 1 == their_oldest_unrevoked_remote_ctn: # A node: - # if next_remote_revocation_number is equal to the commitment number of the last revoke_and_ack + # if next_revocation_number is equal to the commitment number of the last revoke_and_ack # the receiving node sent, AND the receiving node hasn't already received a closing_signed: # MUST re-send the revoke_and_ack. last_secret, last_point = chan.get_secret_and_point(LOCAL, oldest_unrevoked_local_ctn - 1) diff --git a/electrum/lnwire/README.md b/electrum/lnwire/README.md new file mode 100644 index 000000000..72fd48f37 --- /dev/null +++ b/electrum/lnwire/README.md @@ -0,0 +1,5 @@ +These files are generated from the BOLT repository: +``` +$ python3 tools/extract-formats.py 01-*.md 02-*.md 07-*.md > peer_wire.csv +$ python3 tools/extract-formats.py 04-*.md > onion_wire.csv +``` diff --git a/electrum/lnwire/onion_wire.csv b/electrum/lnwire/onion_wire.csv new file mode 100644 index 000000000..7d0258239 --- /dev/null +++ b/electrum/lnwire/onion_wire.csv @@ -0,0 +1,53 @@ +tlvtype,tlv_payload,amt_to_forward,2 +tlvdata,tlv_payload,amt_to_forward,amt_to_forward,tu64, +tlvtype,tlv_payload,outgoing_cltv_value,4 +tlvdata,tlv_payload,outgoing_cltv_value,outgoing_cltv_value,tu32, +tlvtype,tlv_payload,short_channel_id,6 +tlvdata,tlv_payload,short_channel_id,short_channel_id,short_channel_id, +tlvtype,tlv_payload,payment_data,8 +tlvdata,tlv_payload,payment_data,payment_secret,byte,32 +tlvdata,tlv_payload,payment_data,total_msat,tu64, +msgtype,invalid_realm,PERM|1 +msgtype,temporary_node_failure,NODE|2 +msgtype,permanent_node_failure,PERM|NODE|2 +msgtype,required_node_feature_missing,PERM|NODE|3 +msgtype,invalid_onion_version,BADONION|PERM|4 +msgdata,invalid_onion_version,sha256_of_onion,sha256, +msgtype,invalid_onion_hmac,BADONION|PERM|5 +msgdata,invalid_onion_hmac,sha256_of_onion,sha256, +msgtype,invalid_onion_key,BADONION|PERM|6 +msgdata,invalid_onion_key,sha256_of_onion,sha256, +msgtype,temporary_channel_failure,UPDATE|7 +msgdata,temporary_channel_failure,len,u16, +msgdata,temporary_channel_failure,channel_update,byte,len +msgtype,permanent_channel_failure,PERM|8 +msgtype,required_channel_feature_missing,PERM|9 +msgtype,unknown_next_peer,PERM|10 +msgtype,amount_below_minimum,UPDATE|11 +msgdata,amount_below_minimum,htlc_msat,u64, +msgdata,amount_below_minimum,len,u16, +msgdata,amount_below_minimum,channel_update,byte,len +msgtype,fee_insufficient,UPDATE|12 +msgdata,fee_insufficient,htlc_msat,u64, +msgdata,fee_insufficient,len,u16, +msgdata,fee_insufficient,channel_update,byte,len +msgtype,incorrect_cltv_expiry,UPDATE|13 +msgdata,incorrect_cltv_expiry,cltv_expiry,u32, +msgdata,incorrect_cltv_expiry,len,u16, +msgdata,incorrect_cltv_expiry,channel_update,byte,len +msgtype,expiry_too_soon,UPDATE|14 +msgdata,expiry_too_soon,len,u16, +msgdata,expiry_too_soon,channel_update,byte,len +msgtype,incorrect_or_unknown_payment_details,PERM|15 +msgdata,incorrect_or_unknown_payment_details,htlc_msat,u64, +msgdata,incorrect_or_unknown_payment_details,height,u32, +msgtype,final_incorrect_cltv_expiry,18 +msgdata,final_incorrect_cltv_expiry,cltv_expiry,u32, +msgtype,final_incorrect_htlc_amount,19 +msgdata,final_incorrect_htlc_amount,incoming_htlc_amt,u64, +msgtype,channel_disabled,UPDATE|20 +msgtype,expiry_too_far,21 +msgtype,invalid_onion_payload,PERM|22 +msgdata,invalid_onion_payload,type,varint, +msgdata,invalid_onion_payload,offset,u16, +msgtype,mpp_timeout,23 diff --git a/electrum/lnwire/peer_wire.csv b/electrum/lnwire/peer_wire.csv new file mode 100644 index 000000000..a128e9c71 --- /dev/null +++ b/electrum/lnwire/peer_wire.csv @@ -0,0 +1,210 @@ +msgtype,init,16 +msgdata,init,gflen,u16, +msgdata,init,globalfeatures,byte,gflen +msgdata,init,flen,u16, +msgdata,init,features,byte,flen +msgdata,init,tlvs,init_tlvs, +tlvtype,init_tlvs,networks,1 +tlvdata,init_tlvs,networks,chains,chain_hash,... +msgtype,error,17 +msgdata,error,channel_id,channel_id, +msgdata,error,len,u16, +msgdata,error,data,byte,len +msgtype,ping,18 +msgdata,ping,num_pong_bytes,u16, +msgdata,ping,byteslen,u16, +msgdata,ping,ignored,byte,byteslen +msgtype,pong,19 +msgdata,pong,byteslen,u16, +msgdata,pong,ignored,byte,byteslen +tlvtype,n1,tlv1,1 +tlvdata,n1,tlv1,amount_msat,tu64, +tlvtype,n1,tlv2,2 +tlvdata,n1,tlv2,scid,short_channel_id, +tlvtype,n1,tlv3,3 +tlvdata,n1,tlv3,node_id,point, +tlvdata,n1,tlv3,amount_msat_1,u64, +tlvdata,n1,tlv3,amount_msat_2,u64, +tlvtype,n1,tlv4,254 +tlvdata,n1,tlv4,cltv_delta,u16, +tlvtype,n2,tlv1,0 +tlvdata,n2,tlv1,amount_msat,tu64, +tlvtype,n2,tlv2,11 +tlvdata,n2,tlv2,cltv_expiry,tu32, +msgtype,open_channel,32 +msgdata,open_channel,chain_hash,chain_hash, +msgdata,open_channel,temporary_channel_id,byte,32 +msgdata,open_channel,funding_satoshis,u64, +msgdata,open_channel,push_msat,u64, +msgdata,open_channel,dust_limit_satoshis,u64, +msgdata,open_channel,max_htlc_value_in_flight_msat,u64, +msgdata,open_channel,channel_reserve_satoshis,u64, +msgdata,open_channel,htlc_minimum_msat,u64, +msgdata,open_channel,feerate_per_kw,u32, +msgdata,open_channel,to_self_delay,u16, +msgdata,open_channel,max_accepted_htlcs,u16, +msgdata,open_channel,funding_pubkey,point, +msgdata,open_channel,revocation_basepoint,point, +msgdata,open_channel,payment_basepoint,point, +msgdata,open_channel,delayed_payment_basepoint,point, +msgdata,open_channel,htlc_basepoint,point, +msgdata,open_channel,first_per_commitment_point,point, +msgdata,open_channel,channel_flags,byte, +msgdata,open_channel,shutdown_len,u16,,option_upfront_shutdown_script +msgdata,open_channel,shutdown_scriptpubkey,byte,shutdown_len,option_upfront_shutdown_script +msgtype,accept_channel,33 +msgdata,accept_channel,temporary_channel_id,byte,32 +msgdata,accept_channel,dust_limit_satoshis,u64, +msgdata,accept_channel,max_htlc_value_in_flight_msat,u64, +msgdata,accept_channel,channel_reserve_satoshis,u64, +msgdata,accept_channel,htlc_minimum_msat,u64, +msgdata,accept_channel,minimum_depth,u32, +msgdata,accept_channel,to_self_delay,u16, +msgdata,accept_channel,max_accepted_htlcs,u16, +msgdata,accept_channel,funding_pubkey,point, +msgdata,accept_channel,revocation_basepoint,point, +msgdata,accept_channel,payment_basepoint,point, +msgdata,accept_channel,delayed_payment_basepoint,point, +msgdata,accept_channel,htlc_basepoint,point, +msgdata,accept_channel,first_per_commitment_point,point, +msgdata,accept_channel,shutdown_len,u16,,option_upfront_shutdown_script +msgdata,accept_channel,shutdown_scriptpubkey,byte,shutdown_len,option_upfront_shutdown_script +msgtype,funding_created,34 +msgdata,funding_created,temporary_channel_id,byte,32 +msgdata,funding_created,funding_txid,sha256, +msgdata,funding_created,funding_output_index,u16, +msgdata,funding_created,signature,signature, +msgtype,funding_signed,35 +msgdata,funding_signed,channel_id,channel_id, +msgdata,funding_signed,signature,signature, +msgtype,funding_locked,36 +msgdata,funding_locked,channel_id,channel_id, +msgdata,funding_locked,next_per_commitment_point,point, +msgtype,shutdown,38 +msgdata,shutdown,channel_id,channel_id, +msgdata,shutdown,len,u16, +msgdata,shutdown,scriptpubkey,byte,len +msgtype,closing_signed,39 +msgdata,closing_signed,channel_id,channel_id, +msgdata,closing_signed,fee_satoshis,u64, +msgdata,closing_signed,signature,signature, +msgtype,update_add_htlc,128 +msgdata,update_add_htlc,channel_id,channel_id, +msgdata,update_add_htlc,id,u64, +msgdata,update_add_htlc,amount_msat,u64, +msgdata,update_add_htlc,payment_hash,sha256, +msgdata,update_add_htlc,cltv_expiry,u32, +msgdata,update_add_htlc,onion_routing_packet,byte,1366 +msgtype,update_fulfill_htlc,130 +msgdata,update_fulfill_htlc,channel_id,channel_id, +msgdata,update_fulfill_htlc,id,u64, +msgdata,update_fulfill_htlc,payment_preimage,byte,32 +msgtype,update_fail_htlc,131 +msgdata,update_fail_htlc,channel_id,channel_id, +msgdata,update_fail_htlc,id,u64, +msgdata,update_fail_htlc,len,u16, +msgdata,update_fail_htlc,reason,byte,len +msgtype,update_fail_malformed_htlc,135 +msgdata,update_fail_malformed_htlc,channel_id,channel_id, +msgdata,update_fail_malformed_htlc,id,u64, +msgdata,update_fail_malformed_htlc,sha256_of_onion,sha256, +msgdata,update_fail_malformed_htlc,failure_code,u16, +msgtype,commitment_signed,132 +msgdata,commitment_signed,channel_id,channel_id, +msgdata,commitment_signed,signature,signature, +msgdata,commitment_signed,num_htlcs,u16, +msgdata,commitment_signed,htlc_signature,signature,num_htlcs +msgtype,revoke_and_ack,133 +msgdata,revoke_and_ack,channel_id,channel_id, +msgdata,revoke_and_ack,per_commitment_secret,byte,32 +msgdata,revoke_and_ack,next_per_commitment_point,point, +msgtype,update_fee,134 +msgdata,update_fee,channel_id,channel_id, +msgdata,update_fee,feerate_per_kw,u32, +msgtype,channel_reestablish,136 +msgdata,channel_reestablish,channel_id,channel_id, +msgdata,channel_reestablish,next_commitment_number,u64, +msgdata,channel_reestablish,next_revocation_number,u64, +msgdata,channel_reestablish,your_last_per_commitment_secret,byte,32,option_data_loss_protect,option_static_remotekey +msgdata,channel_reestablish,my_current_per_commitment_point,point,,option_data_loss_protect,option_static_remotekey +msgtype,announcement_signatures,259 +msgdata,announcement_signatures,channel_id,channel_id, +msgdata,announcement_signatures,short_channel_id,short_channel_id, +msgdata,announcement_signatures,node_signature,signature, +msgdata,announcement_signatures,bitcoin_signature,signature, +msgtype,channel_announcement,256 +msgdata,channel_announcement,node_signature_1,signature, +msgdata,channel_announcement,node_signature_2,signature, +msgdata,channel_announcement,bitcoin_signature_1,signature, +msgdata,channel_announcement,bitcoin_signature_2,signature, +msgdata,channel_announcement,len,u16, +msgdata,channel_announcement,features,byte,len +msgdata,channel_announcement,chain_hash,chain_hash, +msgdata,channel_announcement,short_channel_id,short_channel_id, +msgdata,channel_announcement,node_id_1,point, +msgdata,channel_announcement,node_id_2,point, +msgdata,channel_announcement,bitcoin_key_1,point, +msgdata,channel_announcement,bitcoin_key_2,point, +msgtype,node_announcement,257 +msgdata,node_announcement,signature,signature, +msgdata,node_announcement,flen,u16, +msgdata,node_announcement,features,byte,flen +msgdata,node_announcement,timestamp,u32, +msgdata,node_announcement,node_id,point, +msgdata,node_announcement,rgb_color,byte,3 +msgdata,node_announcement,alias,byte,32 +msgdata,node_announcement,addrlen,u16, +msgdata,node_announcement,addresses,byte,addrlen +msgtype,channel_update,258 +msgdata,channel_update,signature,signature, +msgdata,channel_update,chain_hash,chain_hash, +msgdata,channel_update,short_channel_id,short_channel_id, +msgdata,channel_update,timestamp,u32, +msgdata,channel_update,message_flags,byte, +msgdata,channel_update,channel_flags,byte, +msgdata,channel_update,cltv_expiry_delta,u16, +msgdata,channel_update,htlc_minimum_msat,u64, +msgdata,channel_update,fee_base_msat,u32, +msgdata,channel_update,fee_proportional_millionths,u32, +msgdata,channel_update,htlc_maximum_msat,u64,,option_channel_htlc_max +msgtype,query_short_channel_ids,261,gossip_queries +msgdata,query_short_channel_ids,chain_hash,chain_hash, +msgdata,query_short_channel_ids,len,u16, +msgdata,query_short_channel_ids,encoded_short_ids,byte,len +msgdata,query_short_channel_ids,tlvs,query_short_channel_ids_tlvs, +tlvtype,query_short_channel_ids_tlvs,query_flags,1 +tlvdata,query_short_channel_ids_tlvs,query_flags,encoding_type,u8, +tlvdata,query_short_channel_ids_tlvs,query_flags,encoded_query_flags,byte,... +msgtype,reply_short_channel_ids_end,262,gossip_queries +msgdata,reply_short_channel_ids_end,chain_hash,chain_hash, +msgdata,reply_short_channel_ids_end,complete,byte, +msgtype,query_channel_range,263,gossip_queries +msgdata,query_channel_range,chain_hash,chain_hash, +msgdata,query_channel_range,first_blocknum,u32, +msgdata,query_channel_range,number_of_blocks,u32, +msgdata,query_channel_range,tlvs,query_channel_range_tlvs, +tlvtype,query_channel_range_tlvs,query_option,1 +tlvdata,query_channel_range_tlvs,query_option,query_option_flags,varint, +msgtype,reply_channel_range,264,gossip_queries +msgdata,reply_channel_range,chain_hash,chain_hash, +msgdata,reply_channel_range,first_blocknum,u32, +msgdata,reply_channel_range,number_of_blocks,u32, +msgdata,reply_channel_range,complete,byte, +msgdata,reply_channel_range,len,u16, +msgdata,reply_channel_range,encoded_short_ids,byte,len +msgdata,reply_channel_range,tlvs,reply_channel_range_tlvs, +tlvtype,reply_channel_range_tlvs,timestamps_tlv,1 +tlvdata,reply_channel_range_tlvs,timestamps_tlv,encoding_type,u8, +tlvdata,reply_channel_range_tlvs,timestamps_tlv,encoded_timestamps,byte,... +tlvtype,reply_channel_range_tlvs,checksums_tlv,3 +tlvdata,reply_channel_range_tlvs,checksums_tlv,checksums,channel_update_checksums,... +subtype,channel_update_timestamps +subtypedata,channel_update_timestamps,timestamp_node_id_1,u32, +subtypedata,channel_update_timestamps,timestamp_node_id_2,u32, +subtype,channel_update_checksums +subtypedata,channel_update_checksums,checksum_node_id_1,u32, +subtypedata,channel_update_checksums,checksum_node_id_2,u32, +msgtype,gossip_timestamp_filter,265,gossip_queries +msgdata,gossip_timestamp_filter,chain_hash,chain_hash, +msgdata,gossip_timestamp_filter,first_timestamp,u32, +msgdata,gossip_timestamp_filter,timestamp_range,u32, From 3a73f6ee5cc3cca995ab1164c8083843a0bf0046 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Thu, 12 Mar 2020 04:08:13 +0100 Subject: [PATCH 43/69] lnmsg.decode_msg: dict values for numbers are int, instead of BE bytes Will be useful for TLVs where it makes sense to do the conversion in lnmsg, as it might be more complicated than just int.from_bytes(). --- electrum/channel_db.py | 20 ++++++------ electrum/lnchannel.py | 12 +++---- electrum/lnmsg.py | 17 +++++++--- electrum/lnpeer.py | 57 ++++++++++++++++----------------- electrum/tests/test_lnrouter.py | 37 +++++++++++---------- 5 files changed, 75 insertions(+), 68 deletions(-) diff --git a/electrum/channel_db.py b/electrum/channel_db.py index 54fc45460..aef25effe 100644 --- a/electrum/channel_db.py +++ b/electrum/channel_db.py @@ -102,14 +102,14 @@ class Policy(NamedTuple): def from_msg(payload: dict) -> 'Policy': return Policy( key = payload['short_channel_id'] + payload['start_node'], - cltv_expiry_delta = int.from_bytes(payload['cltv_expiry_delta'], "big"), - htlc_minimum_msat = int.from_bytes(payload['htlc_minimum_msat'], "big"), - htlc_maximum_msat = int.from_bytes(payload['htlc_maximum_msat'], "big") if 'htlc_maximum_msat' in payload else None, - fee_base_msat = int.from_bytes(payload['fee_base_msat'], "big"), - fee_proportional_millionths = int.from_bytes(payload['fee_proportional_millionths'], "big"), + cltv_expiry_delta = payload['cltv_expiry_delta'], + htlc_minimum_msat = payload['htlc_minimum_msat'], + htlc_maximum_msat = payload.get('htlc_maximum_msat', None), + fee_base_msat = payload['fee_base_msat'], + fee_proportional_millionths = payload['fee_proportional_millionths'], message_flags = int.from_bytes(payload['message_flags'], "big"), channel_flags = int.from_bytes(payload['channel_flags'], "big"), - timestamp = int.from_bytes(payload['timestamp'], "big") + timestamp = payload['timestamp'], ) @staticmethod @@ -154,7 +154,7 @@ def from_msg(payload) -> Tuple['NodeInfo', Sequence['LNPeerAddr']]: alias = alias.decode('utf8') except: alias = '' - timestamp = int.from_bytes(payload['timestamp'], "big") + timestamp = payload['timestamp'] node_info = NodeInfo(node_id=node_id, features=features, timestamp=timestamp, alias=alias) return node_info, peer_addrs @@ -393,7 +393,7 @@ def add_channel_updates(self, payloads, max_age=None, verify=True) -> Categorize now = int(time.time()) for payload in payloads: short_channel_id = ShortChannelID(payload['short_channel_id']) - timestamp = int.from_bytes(payload['timestamp'], "big") + timestamp = payload['timestamp'] if max_age and now - timestamp > max_age: expired.append(payload) continue @@ -408,7 +408,7 @@ def add_channel_updates(self, payloads, max_age=None, verify=True) -> Categorize known.append(payload) # compare updates to existing database entries for payload in known: - timestamp = int.from_bytes(payload['timestamp'], "big") + timestamp = payload['timestamp'] start_node = payload['start_node'] short_channel_id = ShortChannelID(payload['short_channel_id']) key = (start_node, short_channel_id) @@ -673,7 +673,7 @@ def get_policy_for_node(self, short_channel_id: bytes, node_id: bytes, *, return now = int(time.time()) remote_update_decoded = decode_msg(remote_update_raw)[1] - remote_update_decoded['timestamp'] = now.to_bytes(4, byteorder="big") + remote_update_decoded['timestamp'] = now remote_update_decoded['start_node'] = node_id return Policy.from_msg(remote_update_decoded) elif node_id == chan.get_local_pubkey(): # outgoing direction (from us) diff --git a/electrum/lnchannel.py b/electrum/lnchannel.py index bd52175b7..4fecf6a03 100644 --- a/electrum/lnchannel.py +++ b/electrum/lnchannel.py @@ -218,13 +218,13 @@ def get_outgoing_gossip_channel_update(self) -> bytes: short_channel_id=self.short_channel_id, channel_flags=channel_flags, message_flags=b'\x01', - cltv_expiry_delta=lnutil.NBLOCK_OUR_CLTV_EXPIRY_DELTA.to_bytes(2, byteorder="big"), - htlc_minimum_msat=self.config[REMOTE].htlc_minimum_msat.to_bytes(8, byteorder="big"), - htlc_maximum_msat=htlc_maximum_msat.to_bytes(8, byteorder="big"), - fee_base_msat=lnutil.OUR_FEE_BASE_MSAT.to_bytes(4, byteorder="big"), - fee_proportional_millionths=lnutil.OUR_FEE_PROPORTIONAL_MILLIONTHS.to_bytes(4, byteorder="big"), + cltv_expiry_delta=lnutil.NBLOCK_OUR_CLTV_EXPIRY_DELTA, + htlc_minimum_msat=self.config[REMOTE].htlc_minimum_msat, + htlc_maximum_msat=htlc_maximum_msat, + fee_base_msat=lnutil.OUR_FEE_BASE_MSAT, + fee_proportional_millionths=lnutil.OUR_FEE_PROPORTIONAL_MILLIONTHS, chain_hash=constants.net.rev_genesis_bytes(), - timestamp=now.to_bytes(4, byteorder="big"), + timestamp=now, ) sighash = sha256d(chan_upd[2 + 64:]) sig = ecc.ECPrivkey(self.lnworker.node_keypair.privkey).sign(sighash, ecc.sig_string_from_r_and_s) diff --git a/electrum/lnmsg.py b/electrum/lnmsg.py index 2ef48ae97..4e362efae 100644 --- a/electrum/lnmsg.py +++ b/electrum/lnmsg.py @@ -1,7 +1,7 @@ import os import csv import io -from typing import Callable, Tuple, Any, Dict, List, Sequence, Union +from typing import Callable, Tuple, Any, Dict, List, Sequence, Union, Optional class MalformedMsg(Exception): @@ -24,8 +24,7 @@ def _assert_can_read_at_least_n_bytes(fd: io.BytesIO, n: int) -> None: raise UnexpectedEndOfStream(f"cur_pos={cur_pos}. end_pos={end_pos}. wants to read: {n}") -# TODO return int when it makes sense -def _read_field(*, fd: io.BytesIO, field_type: str, count: int) -> bytes: +def _read_field(*, fd: io.BytesIO, field_type: str, count: int) -> Union[bytes, int]: if not fd: raise Exception() assert isinstance(count, int) and count >= 0, f"{count!r} must be non-neg int" if count == 0: @@ -35,10 +34,19 @@ def _read_field(*, fd: io.BytesIO, field_type: str, count: int) -> bytes: type_len = 1 elif field_type == 'u16': type_len = 2 + assert count == 1, count + _assert_can_read_at_least_n_bytes(fd, type_len) + return int.from_bytes(fd.read(type_len), byteorder="big", signed=False) elif field_type == 'u32': type_len = 4 + assert count == 1, count + _assert_can_read_at_least_n_bytes(fd, type_len) + return int.from_bytes(fd.read(type_len), byteorder="big", signed=False) elif field_type == 'u64': type_len = 8 + assert count == 1, count + _assert_can_read_at_least_n_bytes(fd, type_len) + return int.from_bytes(fd.read(type_len), byteorder="big", signed=False) # TODO tu16/tu32/tu64 elif field_type == 'chain_hash': type_len = 32 @@ -203,7 +211,8 @@ def decode_msg(self, data: bytes) -> Tuple[str, dict]: try: field_count = int(field_count_str) except ValueError: - field_count = int.from_bytes(parsed[field_count_str], byteorder="big") + field_count = parsed[field_count_str] + assert isinstance(field_count, int) #print(f">> count={field_count}. parsed={parsed}") try: parsed[field_name] = _read_field(fd=fd, diff --git a/electrum/lnpeer.py b/electrum/lnpeer.py index 2b0aa87c1..684cbe859 100644 --- a/electrum/lnpeer.py +++ b/electrum/lnpeer.py @@ -180,7 +180,7 @@ def on_error(self, payload): self.ordered_message_queues[chan_id].put_nowait((None, {'error':payload['data']})) def on_ping(self, payload): - l = int.from_bytes(payload['num_pong_bytes'], 'big') + l = payload['num_pong_bytes'] self.send_message('pong', byteslen=l) def on_pong(self, payload): @@ -417,8 +417,8 @@ def decode_short_ids(self, encoded): return ids def on_reply_channel_range(self, payload): - first = int.from_bytes(payload['first_blocknum'], 'big') - num = int.from_bytes(payload['number_of_blocks'], 'big') + first = payload['first_blocknum'] + num = payload['number_of_blocks'] complete = bool(int.from_bytes(payload['complete'], 'big')) encoded = payload['encoded_short_ids'] ids = self.decode_short_ids(encoded) @@ -541,27 +541,27 @@ async def channel_establishment_flow(self, password: Optional[str], funding_tx: ) payload = await self.wait_for_message('accept_channel', temp_channel_id) remote_per_commitment_point = payload['first_per_commitment_point'] - funding_txn_minimum_depth = int.from_bytes(payload['minimum_depth'], 'big') + funding_txn_minimum_depth = payload['minimum_depth'] if funding_txn_minimum_depth <= 0: raise Exception(f"minimum depth too low, {funding_txn_minimum_depth}") if funding_txn_minimum_depth > 30: raise Exception(f"minimum depth too high, {funding_txn_minimum_depth}") - remote_dust_limit_sat = int.from_bytes(payload['dust_limit_satoshis'], byteorder='big') + remote_dust_limit_sat = payload['dust_limit_satoshis'] remote_reserve_sat = self.validate_remote_reserve(payload["channel_reserve_satoshis"], remote_dust_limit_sat, funding_sat) if remote_dust_limit_sat > remote_reserve_sat: raise Exception(f"Remote Lightning peer reports dust_limit_sat > reserve_sat which is a BOLT-02 protocol violation.") - htlc_min = int.from_bytes(payload['htlc_minimum_msat'], 'big') + htlc_min = payload['htlc_minimum_msat'] if htlc_min > MAXIMUM_HTLC_MINIMUM_MSAT_ACCEPTED: raise Exception(f"Remote Lightning peer reports htlc_minimum_msat={htlc_min} mSAT," + f" which is above Electrums required maximum limit of that parameter ({MAXIMUM_HTLC_MINIMUM_MSAT_ACCEPTED} mSAT).") - remote_max = int.from_bytes(payload['max_htlc_value_in_flight_msat'], 'big') + remote_max = payload['max_htlc_value_in_flight_msat'] if remote_max < MINIMUM_MAX_HTLC_VALUE_IN_FLIGHT_ACCEPTED: raise Exception(f"Remote Lightning peer reports max_htlc_value_in_flight_msat at only {remote_max} mSAT" + f" which is below Electrums required minimum ({MINIMUM_MAX_HTLC_VALUE_IN_FLIGHT_ACCEPTED} mSAT).") - max_accepted_htlcs = int.from_bytes(payload["max_accepted_htlcs"], 'big') + max_accepted_htlcs = payload["max_accepted_htlcs"] if max_accepted_htlcs > 483: raise Exception("Remote Lightning peer reports max_accepted_htlcs > 483, which is a BOLT-02 protocol violation.") - remote_to_self_delay = int.from_bytes(payload['to_self_delay'], byteorder='big') + remote_to_self_delay = payload['to_self_delay'] if remote_to_self_delay > MAXIMUM_REMOTE_TO_SELF_DELAY_ACCEPTED: raise Exception(f"Remote Lightning peer reports to_self_delay={remote_to_self_delay}," + f" which is above Electrums required maximum ({MAXIMUM_REMOTE_TO_SELF_DELAY_ACCEPTED})") @@ -647,9 +647,9 @@ async def on_open_channel(self, payload): # payload['channel_flags'] if payload['chain_hash'] != constants.net.rev_genesis_bytes(): raise Exception('wrong chain_hash') - funding_sat = int.from_bytes(payload['funding_satoshis'], 'big') - push_msat = int.from_bytes(payload['push_msat'], 'big') - feerate = int.from_bytes(payload['feerate_per_kw'], 'big') + funding_sat = payload['funding_satoshis'] + push_msat = payload['push_msat'] + feerate = payload['feerate_per_kw'] temp_chan_id = payload['temporary_channel_id'] local_config = self.make_local_config(funding_sat, push_msat, REMOTE) # for the first commitment transaction @@ -674,11 +674,11 @@ async def on_open_channel(self, payload): first_per_commitment_point=per_commitment_point_first, ) funding_created = await self.wait_for_message('funding_created', temp_chan_id) - funding_idx = int.from_bytes(funding_created['funding_output_index'], 'big') + funding_idx = funding_created['funding_output_index'] funding_txid = bh2u(funding_created['funding_txid'][::-1]) channel_id, funding_txid_bytes = channel_id_from_funding_tx(funding_txid, funding_idx) remote_balance_sat = funding_sat * 1000 - push_msat - remote_dust_limit_sat = int.from_bytes(payload['dust_limit_satoshis'], byteorder='big') # TODO validate + remote_dust_limit_sat = payload['dust_limit_satoshis'] # TODO validate remote_reserve_sat = self.validate_remote_reserve(payload['channel_reserve_satoshis'], remote_dust_limit_sat, funding_sat) remote_config = RemoteConfig( payment_basepoint=OnlyPubkeyKeypair(payload['payment_basepoint']), @@ -686,13 +686,13 @@ async def on_open_channel(self, payload): htlc_basepoint=OnlyPubkeyKeypair(payload['htlc_basepoint']), delayed_basepoint=OnlyPubkeyKeypair(payload['delayed_payment_basepoint']), revocation_basepoint=OnlyPubkeyKeypair(payload['revocation_basepoint']), - to_self_delay=int.from_bytes(payload['to_self_delay'], 'big'), + to_self_delay=payload['to_self_delay'], dust_limit_sat=remote_dust_limit_sat, - max_htlc_value_in_flight_msat=int.from_bytes(payload['max_htlc_value_in_flight_msat'], 'big'), # TODO validate - max_accepted_htlcs=int.from_bytes(payload['max_accepted_htlcs'], 'big'), # TODO validate + max_htlc_value_in_flight_msat=payload['max_htlc_value_in_flight_msat'], # TODO validate + max_accepted_htlcs=payload['max_accepted_htlcs'], # TODO validate initial_msat=remote_balance_sat, reserve_sat = remote_reserve_sat, - htlc_minimum_msat=int.from_bytes(payload['htlc_minimum_msat'], 'big'), # TODO validate + htlc_minimum_msat=payload['htlc_minimum_msat'], # TODO validate next_per_commitment_point=payload['first_per_commitment_point'], current_per_commitment_point=None, ) @@ -718,8 +718,7 @@ async def on_open_channel(self, payload): chan.set_state(channel_states.OPENING) self.lnworker.add_new_channel(chan) - def validate_remote_reserve(self, payload_field: bytes, dust_limit: int, funding_sat: int) -> int: - remote_reserve_sat = int.from_bytes(payload_field, 'big') + def validate_remote_reserve(self, remote_reserve_sat: int, dust_limit: int, funding_sat: int) -> int: if remote_reserve_sat < dust_limit: raise Exception('protocol violation: reserve < dust_limit') if remote_reserve_sat > funding_sat/100: @@ -768,8 +767,8 @@ async def reestablish_channel(self, chan: Channel): f'(next_local_ctn={next_local_ctn}, ' f'oldest_unrevoked_remote_ctn={oldest_unrevoked_remote_ctn})') msg = await self.wait_for_message('channel_reestablish', chan_id) - their_next_local_ctn = int.from_bytes(msg["next_commitment_number"], 'big') - their_oldest_unrevoked_remote_ctn = int.from_bytes(msg["next_revocation_number"], 'big') + their_next_local_ctn = msg["next_commitment_number"] + their_oldest_unrevoked_remote_ctn = msg["next_revocation_number"] their_local_pcp = msg.get("my_current_per_commitment_point") their_claim_of_our_last_per_commitment_secret = msg.get("your_last_per_commitment_secret") self.logger.info(f'channel_reestablish ({chan.get_id_for_log()}): received channel_reestablish with ' @@ -1005,7 +1004,7 @@ def send_announcement_signatures(self, chan: Channel): return msg_hash, node_signature, bitcoin_signature def on_update_fail_htlc(self, chan: Channel, payload): - htlc_id = int.from_bytes(payload["id"], "big") + htlc_id = payload["id"] reason = payload["reason"] self.logger.info(f"on_update_fail_htlc. chan {chan.short_channel_id}. htlc_id {htlc_id}") chan.receive_fail_htlc(htlc_id, error_bytes=reason) # TODO handle exc and maybe fail channel (e.g. bad htlc_id) @@ -1083,7 +1082,7 @@ def on_commitment_signed(self, chan: Channel, payload): def on_update_fulfill_htlc(self, chan: Channel, payload): preimage = payload["payment_preimage"] payment_hash = sha256(preimage) - htlc_id = int.from_bytes(payload["id"], "big") + htlc_id = payload["id"] self.logger.info(f"on_update_fulfill_htlc. chan {chan.short_channel_id}. htlc_id {htlc_id}") chan.receive_htlc_settle(preimage, htlc_id) # TODO handle exc and maybe fail channel (e.g. bad htlc_id) self.lnworker.save_preimage(payment_hash, preimage) @@ -1103,10 +1102,10 @@ def on_update_fail_malformed_htlc(self, chan: Channel, payload): def on_update_add_htlc(self, chan: Channel, payload): payment_hash = payload["payment_hash"] - htlc_id = int.from_bytes(payload["id"], 'big') + htlc_id = payload["id"] self.logger.info(f"on_update_add_htlc. chan {chan.short_channel_id}. htlc_id {htlc_id}") - cltv_expiry = int.from_bytes(payload["cltv_expiry"], 'big') - amount_msat_htlc = int.from_bytes(payload["amount_msat"], 'big') + cltv_expiry = payload["cltv_expiry"] + amount_msat_htlc = payload["amount_msat"] onion_packet = payload["onion_routing_packet"] if chan.get_state() != channel_states.OPEN: raise RemoteMisbehaving(f"received update_add_htlc while chan.get_state() != OPEN. state was {chan.get_state()}") @@ -1258,7 +1257,7 @@ def on_revoke_and_ack(self, chan: Channel, payload): self.maybe_send_commitment(chan) def on_update_fee(self, chan: Channel, payload): - feerate = int.from_bytes(payload["feerate_per_kw"], "big") + feerate = payload["feerate_per_kw"] chan.update_fee(feerate, False) async def maybe_update_fee(self, chan: Channel): @@ -1378,7 +1377,7 @@ def verify_signature(tx, sig): while True: # FIXME: the remote SHOULD send closing_signed, but some don't. cs_payload = await self.wait_for_message('closing_signed', chan.channel_id) - their_fee = int.from_bytes(cs_payload['fee_satoshis'], 'big') + their_fee = cs_payload['fee_satoshis'] if their_fee > max_fee: raise Exception(f'the proposed fee exceeds the base fee of the latest commitment transaction {is_local, their_fee, max_fee}') their_sig = cs_payload['signature'] diff --git a/electrum/tests/test_lnrouter.py b/electrum/tests/test_lnrouter.py index db93b925a..59b6996fc 100644 --- a/electrum/tests/test_lnrouter.py +++ b/electrum/tests/test_lnrouter.py @@ -57,46 +57,45 @@ class fake_network: 'bitcoin_key_1': b'\x02bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', 'bitcoin_key_2': b'\x02cccccccccccccccccccccccccccccccc', 'short_channel_id': bfh('0000000000000001'), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), - 'len': b'\x00\x00', 'features': b''}, trusted=True) + 'len': 0, 'features': b''}, trusted=True) self.assertEqual(cdb.num_channels, 1) cdb.add_channel_announcement({'node_id_1': b'\x02bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', 'node_id_2': b'\x02eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee', 'bitcoin_key_1': b'\x02bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', 'bitcoin_key_2': b'\x02eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee', 'short_channel_id': bfh('0000000000000002'), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), - 'len': b'\x00\x00', 'features': b''}, trusted=True) + 'len': 0, 'features': b''}, trusted=True) cdb.add_channel_announcement({'node_id_1': b'\x02aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 'node_id_2': b'\x02bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', 'bitcoin_key_1': b'\x02aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 'bitcoin_key_2': b'\x02bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', 'short_channel_id': bfh('0000000000000003'), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), - 'len': b'\x00\x00', 'features': b''}, trusted=True) + 'len': 0, 'features': b''}, trusted=True) cdb.add_channel_announcement({'node_id_1': b'\x02cccccccccccccccccccccccccccccccc', 'node_id_2': b'\x02dddddddddddddddddddddddddddddddd', 'bitcoin_key_1': b'\x02cccccccccccccccccccccccccccccccc', 'bitcoin_key_2': b'\x02dddddddddddddddddddddddddddddddd', 'short_channel_id': bfh('0000000000000004'), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), - 'len': b'\x00\x00', 'features': b''}, trusted=True) + 'len': 0, 'features': b''}, trusted=True) cdb.add_channel_announcement({'node_id_1': b'\x02dddddddddddddddddddddddddddddddd', 'node_id_2': b'\x02eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee', 'bitcoin_key_1': b'\x02dddddddddddddddddddddddddddddddd', 'bitcoin_key_2': b'\x02eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee', 'short_channel_id': bfh('0000000000000005'), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), - 'len': b'\x00\x00', 'features': b''}, trusted=True) + 'len': 0, 'features': b''}, trusted=True) cdb.add_channel_announcement({'node_id_1': b'\x02aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 'node_id_2': b'\x02dddddddddddddddddddddddddddddddd', 'bitcoin_key_1': b'\x02aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 'bitcoin_key_2': b'\x02dddddddddddddddddddddddddddddddd', 'short_channel_id': bfh('0000000000000006'), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), - 'len': b'\x00\x00', 'features': b''}, trusted=True) - o = lambda i: i.to_bytes(8, "big") - cdb.add_channel_update({'short_channel_id': bfh('0000000000000001'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': b'\x00\x00\x00\x00'}) - cdb.add_channel_update({'short_channel_id': bfh('0000000000000001'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': b'\x00\x00\x00\x00'}) - cdb.add_channel_update({'short_channel_id': bfh('0000000000000002'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': o(99), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': b'\x00\x00\x00\x00'}) - cdb.add_channel_update({'short_channel_id': bfh('0000000000000002'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': b'\x00\x00\x00\x00'}) - cdb.add_channel_update({'short_channel_id': bfh('0000000000000003'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': b'\x00\x00\x00\x00'}) - cdb.add_channel_update({'short_channel_id': bfh('0000000000000003'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': b'\x00\x00\x00\x00'}) - cdb.add_channel_update({'short_channel_id': bfh('0000000000000004'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': b'\x00\x00\x00\x00'}) - cdb.add_channel_update({'short_channel_id': bfh('0000000000000004'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': b'\x00\x00\x00\x00'}) - cdb.add_channel_update({'short_channel_id': bfh('0000000000000005'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': b'\x00\x00\x00\x00'}) - cdb.add_channel_update({'short_channel_id': bfh('0000000000000005'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(999), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': b'\x00\x00\x00\x00'}) - cdb.add_channel_update({'short_channel_id': bfh('0000000000000006'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(99999999), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': b'\x00\x00\x00\x00'}) - cdb.add_channel_update({'short_channel_id': bfh('0000000000000006'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': b'\x00\x00\x00\x00'}) + 'len': 0, 'features': b''}, trusted=True) + cdb.add_channel_update({'short_channel_id': bfh('0000000000000001'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0}) + cdb.add_channel_update({'short_channel_id': bfh('0000000000000001'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0}) + cdb.add_channel_update({'short_channel_id': bfh('0000000000000002'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': 99, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0}) + cdb.add_channel_update({'short_channel_id': bfh('0000000000000002'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0}) + cdb.add_channel_update({'short_channel_id': bfh('0000000000000003'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0}) + cdb.add_channel_update({'short_channel_id': bfh('0000000000000003'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0}) + cdb.add_channel_update({'short_channel_id': bfh('0000000000000004'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0}) + cdb.add_channel_update({'short_channel_id': bfh('0000000000000004'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0}) + cdb.add_channel_update({'short_channel_id': bfh('0000000000000005'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0}) + cdb.add_channel_update({'short_channel_id': bfh('0000000000000005'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 999, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0}) + cdb.add_channel_update({'short_channel_id': bfh('0000000000000006'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 99999999, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0}) + cdb.add_channel_update({'short_channel_id': bfh('0000000000000006'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0}) path = path_finder.find_path_for_payment(b'\x02aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', b'\x02eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee', 100000) self.assertEqual([(b'\x02bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', b'\x00\x00\x00\x00\x00\x00\x00\x03'), (b'\x02eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee', b'\x00\x00\x00\x00\x00\x00\x00\x02'), From 69497522630b95a632875cf87bbeec7d1d7b3242 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Fri, 13 Mar 2020 21:20:31 +0100 Subject: [PATCH 44/69] lnmsg: initial TLV implementation --- electrum/lnmsg.py | 272 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 239 insertions(+), 33 deletions(-) diff --git a/electrum/lnmsg.py b/electrum/lnmsg.py index 4e362efae..49afb7319 100644 --- a/electrum/lnmsg.py +++ b/electrum/lnmsg.py @@ -2,6 +2,7 @@ import csv import io from typing import Callable, Tuple, Any, Dict, List, Sequence, Union, Optional +from collections import OrderedDict class MalformedMsg(Exception): @@ -16,12 +17,56 @@ class UnexpectedEndOfStream(MalformedMsg): pass -def _assert_can_read_at_least_n_bytes(fd: io.BytesIO, n: int) -> None: +class FieldEncodingNotMinimal(MalformedMsg): + pass + + +class UnknownMandatoryTLVRecordType(MalformedMsg): + pass + + +def _num_remaining_bytes_to_read(fd: io.BytesIO) -> int: cur_pos = fd.tell() end_pos = fd.seek(0, io.SEEK_END) fd.seek(cur_pos) - if end_pos - cur_pos < n: - raise UnexpectedEndOfStream(f"cur_pos={cur_pos}. end_pos={end_pos}. wants to read: {n}") + return end_pos - cur_pos + + +def _assert_can_read_at_least_n_bytes(fd: io.BytesIO, n: int) -> None: + nremaining = _num_remaining_bytes_to_read(fd) + if nremaining < n: + raise UnexpectedEndOfStream(f"wants to read {n} bytes but only {nremaining} bytes left") + + +def bigsize_from_int(i: int) -> bytes: + assert i >= 0, i + if i < 0xfd: + return int.to_bytes(i, length=1, byteorder="big", signed=False) + elif i < 0x1_0000: + return b"\xfd" + int.to_bytes(i, length=2, byteorder="big", signed=False) + elif i < 0x1_0000_0000: + return b"\xfe" + int.to_bytes(i, length=4, byteorder="big", signed=False) + else: + return b"\xff" + int.to_bytes(i, length=8, byteorder="big", signed=False) + + +def read_int_from_bigsize(fd: io.BytesIO) -> Optional[int]: + try: + first = fd.read(1)[0] + except IndexError: + return None # end of file + if first < 0xfd: + return first + elif first == 0xfd: + _assert_can_read_at_least_n_bytes(fd, 2) + return int.from_bytes(fd.read(2), byteorder="big", signed=False) + elif first == 0xfe: + _assert_can_read_at_least_n_bytes(fd, 4) + return int.from_bytes(fd.read(4), byteorder="big", signed=False) + elif first == 0xff: + _assert_can_read_at_least_n_bytes(fd, 8) + return int.from_bytes(fd.read(8), byteorder="big", signed=False) + raise Exception() def _read_field(*, fd: io.BytesIO, field_type: str, count: int) -> Union[bytes, int]: @@ -32,22 +77,36 @@ def _read_field(*, fd: io.BytesIO, field_type: str, count: int) -> Union[bytes, type_len = None if field_type == 'byte': type_len = 1 - elif field_type == 'u16': - type_len = 2 + elif field_type in ('u16', 'u32', 'u64'): + if field_type == 'u16': + type_len = 2 + elif field_type == 'u32': + type_len = 4 + else: + assert field_type == 'u64' + type_len = 8 assert count == 1, count _assert_can_read_at_least_n_bytes(fd, type_len) return int.from_bytes(fd.read(type_len), byteorder="big", signed=False) - elif field_type == 'u32': - type_len = 4 + elif field_type in ('tu16', 'tu32', 'tu64'): + if field_type == 'tu16': + type_len = 2 + elif field_type == 'tu32': + type_len = 4 + else: + assert field_type == 'tu64' + type_len = 8 assert count == 1, count - _assert_can_read_at_least_n_bytes(fd, type_len) - return int.from_bytes(fd.read(type_len), byteorder="big", signed=False) - elif field_type == 'u64': - type_len = 8 + raw = fd.read(type_len) + if len(raw) > 0 and raw[0] == 0x00: + raise FieldEncodingNotMinimal() + return int.from_bytes(raw, byteorder="big", signed=False) + elif field_type == 'varint': assert count == 1, count - _assert_can_read_at_least_n_bytes(fd, type_len) - return int.from_bytes(fd.read(type_len), byteorder="big", signed=False) - # TODO tu16/tu32/tu64 + val = read_int_from_bigsize(fd) + if val is None: + raise UnexpectedEndOfStream() + return val elif field_type == 'chain_hash': type_len = 32 elif field_type == 'channel_id': @@ -82,7 +141,35 @@ def _write_field(*, fd: io.BytesIO, field_type: str, count: int, type_len = 4 elif field_type == 'u64': type_len = 8 - # TODO tu16/tu32/tu64 + elif field_type in ('tu16', 'tu32', 'tu64'): + if field_type == 'tu16': + type_len = 2 + elif field_type == 'tu32': + type_len = 4 + else: + assert field_type == 'tu64' + type_len = 8 + assert count == 1, count + if isinstance(value, int): + value = int.to_bytes(value, length=type_len, byteorder="big", signed=False) + if not isinstance(value, (bytes, bytearray)): + raise Exception(f"can only write bytes into fd. got: {value!r}") + while len(value) > 0 and value[0] == 0x00: + value = value[1:] + nbytes_written = fd.write(value) + if nbytes_written != len(value): + raise Exception(f"tried to write {len(value)} bytes, but only wrote {nbytes_written}!?") + return + elif field_type == 'varint': + assert count == 1, count + if isinstance(value, int): + value = bigsize_from_int(value) + if not isinstance(value, (bytes, bytearray)): + raise Exception(f"can only write bytes into fd. got: {value!r}") + nbytes_written = fd.write(value) + if nbytes_written != len(value): + raise Exception(f"tried to write {len(value)} bytes, but only wrote {nbytes_written}!?") + return elif field_type == 'chain_hash': type_len = 32 elif field_type == 'channel_id': @@ -109,16 +196,55 @@ def _write_field(*, fd: io.BytesIO, field_type: str, count: int, raise Exception(f"tried to write {len(value)} bytes, but only wrote {nbytes_written}!?") +def _read_tlv_record(*, fd: io.BytesIO) -> Tuple[int, bytes]: + if not fd: raise Exception() + tlv_type = _read_field(fd=fd, field_type="varint", count=1) + tlv_len = _read_field(fd=fd, field_type="varint", count=1) + tlv_val = _read_field(fd=fd, field_type="byte", count=tlv_len) + return tlv_type, tlv_val + + +def _write_tlv_record(*, fd: io.BytesIO, tlv_type: int, tlv_val: bytes) -> None: + if not fd: raise Exception() + tlv_len = len(tlv_val) + _write_field(fd=fd, field_type="varint", count=1, value=tlv_type) + _write_field(fd=fd, field_type="varint", count=1, value=tlv_len) + _write_field(fd=fd, field_type="byte", count=tlv_len, value=tlv_val) + + +def _resolve_field_count(field_count_str: str, *, vars_dict: dict) -> int: + if field_count_str == "": + field_count = 1 + elif field_count_str == "...": + raise NotImplementedError() # TODO... + else: + try: + field_count = int(field_count_str) + except ValueError: + field_count = vars_dict[field_count_str] + if isinstance(field_count, (bytes, bytearray)): + field_count = int.from_bytes(field_count, byteorder="big") + assert isinstance(field_count, int) + return field_count + + class LNSerializer: def __init__(self): + # TODO msg_type could be 'int' everywhere... self.msg_scheme_from_type = {} # type: Dict[bytes, List[Sequence[str]]] self.msg_type_from_name = {} # type: Dict[str, bytes] + + self.in_tlv_stream_get_tlv_record_scheme_from_type = {} # type: Dict[str, Dict[int, List[Sequence[str]]]] + self.in_tlv_stream_get_record_type_from_name = {} # type: Dict[str, Dict[str, int]] + self.in_tlv_stream_get_record_name_from_type = {} # type: Dict[str, Dict[int, str]] + path = os.path.join(os.path.dirname(__file__), "lnwire", "peer_wire.csv") with open(path, newline='') as f: csvreader = csv.reader(f) for row in csvreader: #print(f">>> {row!r}") if row[0] == "msgtype": + # msgtype,,[,