ln: do not add_peer in open_channel, but add_peer in gui when opening channel to unknown peer

ln: Merge OpenChannel and HTLCStateMachine

.gitignore

@@ -21,3 +21,7 @@ bin/
 # tox files
 .cache/
 .coverage
+
+# kivy
+gui/kivy/theming/light-0.png
+gui/kivy/theming/light.atlas

contrib/deterministic-build/requirements.txt

@@ -62,3 +62,5 @@ wheel==0.31.1 \
 colorama==0.3.9 \
     --hash=sha256:463f8483208e921368c9f306094eb6f725c6ca42b0f97e313cb5d5512459feda \
     --hash=sha256:48eb22f4f8461b1df5734a074b57042430fb06e1d61bd1e11b078c0fe6d7a1f1
+bitstring==3.1.5
+cryptography

contrib/requirements/requirements.txt

@@ -8,3 +8,5 @@ dnspython
 jsonrpclib-pelix
 PySocks>=1.6.6
 qdarkstyle<3.0
+bitstring
+cryptography

electrum

@@ -403,6 +403,8 @@ if __name__ == '__main__':
         constants.set_regtest()
     elif config.get('simnet'):
         constants.set_simnet()
+    else:
+        raise Exception('lightning branch not available on mainnet')
 
     # run non-RPC commands separately
     if cmdname in ['create', 'restore']:

gui/kivy/main.kv

@@ -451,6 +451,12 @@ BoxLayout:
                 ActionOvrButton:
                     name: 'network'
                     text: _('Network')
+                ActionOvrButton:
+                    name: 'lightning_payer_dialog'
+                    text: _('Pay Lightning Invoice')
+                ActionOvrButton:
+                    name: 'lightning_channels_dialog'
+                    text: _('Lightning Channels')
                 ActionOvrButton:
                     name: 'settings'
                     text: _('Settings')

gui/kivy/main_window.py

@@ -70,6 +70,8 @@ Label.register('Roboto',
 
 from electrum.util import base_units, NoDynamicFeeEstimates
 
+from .uix.dialogs.lightning_payer import LightningPayerDialog
+from .uix.dialogs.lightning_channels import LightningChannelsDialog
 
 class ElectrumWindow(App):
 
@@ -578,6 +580,14 @@ class ElectrumWindow(App):
         self._settings_dialog.update()
         self._settings_dialog.open()
 
+    def lightning_payer_dialog(self):
+        d = LightningPayerDialog(self)
+        d.open()
+
+    def lightning_channels_dialog(self):
+        d = LightningChannelsDialog(self)
+        d.open()
+
     def popup_dialog(self, name):
         if name == 'settings':
             self.settings_dialog()
@@ -595,6 +605,8 @@ class ElectrumWindow(App):
                 ref.data = xpub
                 master_public_keys_layout.add_widget(ref)
             popup.open()
+        elif name.endswith("_dialog"):
+            getattr(self, name)()
         else:
             popup = Builder.load_file('gui/kivy/uix/ui_screens/'+name+'.kv')
             popup.open()

gui/kivy/uix/dialogs/installwizard.py

@@ -762,7 +762,11 @@ class RestoreSeedDialog(WizardDialog):
         from electrum.mnemonic import Mnemonic
         from electrum.old_mnemonic import words as old_wordlist
         self.words = set(Mnemonic('en').wordlist).union(set(old_wordlist))
-        self.ids.text_input_seed.text = test_seed if is_test else ''
+
+        import subprocess
+        proc = subprocess.run(["xclip","-sel","clipboard","-o"], stdout=subprocess.PIPE)
+        self.ids.text_input_seed.text = proc.stdout.decode("ascii") if is_test else ''
+
         self.message = _('Please type your seed phrase using the virtual keyboard.')
         self.title = _('Enter Seed')
         self.ext = False

gui/kivy/uix/dialogs/lightning_channels.py

@@ -0,0 +1,123 @@
+import binascii
+from kivy.lang import Builder
+from kivy.factory import Factory
+from kivy.uix.popup import Popup
+from kivy.clock import Clock
+from electrum_gui.kivy.uix.context_menu import ContextMenu
+
+Builder.load_string('''
+<LightningChannelItem@CardItem>
+    details: {}
+    active: False
+    channelId: '<channelId not set>'
+    Label:
+        text: root.channelId
+
+<LightningChannelsDialog@Popup>:
+    name: 'lightning_channels'
+    BoxLayout:
+        id: box
+        orientation: 'vertical'
+        spacing: '1dp'
+        ScrollView:
+            GridLayout:
+                cols: 1
+                id: lightning_channels_container
+                size_hint: 1, None
+                height: self.minimum_height
+                spacing: '2dp'
+                padding: '12dp'
+
+<ChannelDetailsItem@BoxLayout>:
+    canvas.before:
+        Color:
+            rgba: 0.5, 0.5, 0.5, 1
+        Rectangle:
+            size: self.size
+            pos: self.pos
+    value: ''
+    Label:
+        text: root.value
+        text_size: self.size # this makes the text not overflow, but wrap
+
+<ChannelDetailsRow@BoxLayout>:
+    keyName: ''
+    value: ''
+    ChannelDetailsItem:
+        value: root.keyName
+        size_hint_x: 0.5 # this makes the column narrower
+
+    # see https://blog.kivy.org/2014/07/wrapping-text-in-kivys-label/
+    ScrollView:
+        Label:
+            text: root.value
+            size_hint_y: None
+            text_size: self.width, None
+            height: self.texture_size[1]
+
+<ChannelDetailsList@RecycleView>:
+    scroll_type: ['bars', 'content']
+    scroll_wheel_distance: dp(114)
+    bar_width: dp(10)
+    viewclass: 'ChannelDetailsRow'
+    RecycleBoxLayout:
+        default_size: None, dp(56)
+        default_size_hint: 1, None
+        size_hint_y: None
+        height: self.minimum_height
+        orientation: 'vertical'
+        spacing: dp(2)
+
+<ChannelDetailsPopup@Popup>:
+    id: popuproot
+    data: []
+    ChannelDetailsList:
+        data: popuproot.data
+''')
+
+class ChannelDetailsPopup(Popup):
+    def __init__(self, data, **kwargs):
+        super(ChanenlDetailsPopup,self).__init__(**kwargs)
+        self.data = data
+
+class LightningChannelsDialog(Factory.Popup):
+    def __init__(self, app):
+        super(LightningChannelsDialog, self).__init__()
+        self.clocks = []
+        self.app = app
+        self.context_menu = None
+        self.app.wallet.lnworker.subscribe_channel_list_updates_from_other_thread(self.rpc_result_handler)
+
+    def show_channel_details(self, obj):
+        p = Factory.ChannelDetailsPopup()
+        p.data = [{'keyName': key, 'value': str(obj.details[key])} for key in obj.details.keys()]
+        p.open()
+
+    def close_channel(self, obj):
+        print("UNIMPLEMENTED asked to close channel", obj.channelId) # TODO
+
+    def show_menu(self, obj):
+        self.hide_menu()
+        self.context_menu = ContextMenu(obj, [("Close", self.close_channel),
+            ("Details", self.show_channel_details)])
+        self.ids.box.add_widget(self.context_menu)
+
+    def hide_menu(self):
+        if self.context_menu is not None:
+            self.ids.box.remove_widget(self.context_menu)
+            self.context_menu = None
+
+    def rpc_result_handler(self, res):
+        channel_cards = self.ids.lightning_channels_container
+        channel_cards.clear_widgets()
+        if "channels" in res:
+          for i in res["channels"]:
+            item = Factory.LightningChannelItem()
+            item.screen = self
+            print(i)
+            item.channelId = i["chan_id"]
+            item.active = i["active"]
+            item.details = i
+            channel_cards.add_widget(item)
+        else:
+          self.app.show_info(res)

gui/kivy/uix/dialogs/lightning_payer.py

@@ -0,0 +1,93 @@
+import binascii
+from kivy.lang import Builder
+from kivy.factory import Factory
+from electrum_gui.kivy.i18n import _
+from kivy.clock import mainthread
+from electrum.lnaddr import lndecode
+
+Builder.load_string('''
+<LightningPayerDialog@Popup>
+    id: s
+    name: 'lightning_payer'
+    invoice_data: ''
+    BoxLayout:
+        orientation: "vertical"
+        BlueButton:
+            text: s.invoice_data if s.invoice_data else _('Lightning invoice')
+            shorten: True
+            on_release: Clock.schedule_once(lambda dt: app.show_info(_('Copy and paste the lightning invoice using the Paste button, or use the camera to scan a QR code.')))
+        GridLayout:
+            cols: 4
+            size_hint: 1, None
+            height: '48dp'
+            IconButton:
+                id: qr
+                on_release: Clock.schedule_once(lambda dt: app.scan_qr(on_complete=s.on_lightning_qr))
+                icon: 'atlas://gui/kivy/theming/light/camera'
+            Button:
+                text: _('Paste')
+                on_release: s.do_paste()
+            Button:
+                text: _('Paste using xclip')
+                on_release: s.do_paste_xclip()
+            Button:
+                text: _('Clear')
+                on_release: s.do_clear()
+        Button:
+            size_hint: 1, None
+            height: '48dp'
+            text: _('Open channel to pubkey in invoice')
+            on_release: s.do_open_channel()
+        Button:
+            size_hint: 1, None
+            height: '48dp'
+            text: _('Pay pasted/scanned invoice')
+            on_release: s.do_pay()
+''')
+
+class LightningPayerDialog(Factory.Popup):
+    def __init__(self, app):
+        super(LightningPayerDialog, self).__init__()
+        self.app = app
+
+    #def open(self, *args, **kwargs):
+    #    super(LightningPayerDialog, self).open(*args, **kwargs)
+    #def dismiss(self, *args, **kwargs):
+    #    super(LightningPayerDialog, self).dismiss(*args, **kwargs)
+
+    def do_paste_xclip(self):
+        import subprocess
+        proc = subprocess.run(["xclip","-sel","clipboard","-o"], stdout=subprocess.PIPE)
+        self.invoice_data = proc.stdout.decode("ascii")
+
+    def do_paste(self):
+        contents = self.app._clipboard.paste()
+        if not contents:
+            self.app.show_info(_("Clipboard is empty"))
+            return
+        self.invoice_data = contents
+
+    def do_clear(self):
+        self.invoice_data = ""
+
+    def do_open_channel(self):
+        compressed_pubkey_bytes = lndecode(self.invoice_data).pubkey.serialize()
+        hexpubkey = binascii.hexlify(compressed_pubkey_bytes).decode("ascii")
+        local_amt = 200000
+        push_amt = 100000
+
+        def on_success(pw):
+            # node_id, local_amt, push_amt, emit_function, get_password
+            self.app.wallet.lnworker.open_channel_from_other_thread(hexpubkey, local_amt, push_amt, mainthread(lambda parent: self.app.show_info(_("Channel open, waiting for locking..."))), lambda: pw)
+
+        if self.app.wallet.has_keystore_encryption():
+            # wallet, msg, on_success (Tuple[str, str] -> ()), on_failure (() -> ())
+            self.app.password_dialog(self.app.wallet, _("Password needed for opening channel"), on_success, lambda: self.app.show_error(_("Failed getting password from you")))
+        else:
+            on_success("")
+
+    def do_pay(self):
+        self.app.wallet.lnworker.pay_invoice_from_other_thread(self.invoice_data)
+
+    def on_lightning_qr(self, data):
+        self.invoice_data = str(data)

gui/qt/__init__.py

@@ -51,7 +51,6 @@ from electrum.util import (UserCancelled, print_error,
 
 from .installwizard import InstallWizard
 
-
 try:
     from . import icons_rc
 except Exception as e:

gui/qt/channels_list.py

@@ -0,0 +1,174 @@
+# -*- coding: utf-8 -*-
+from PyQt5 import QtCore, QtWidgets
+from PyQt5.QtWidgets import *
+import concurrent.futures
+
+from electrum.util import inv_dict, bh2u, bfh
+from electrum.i18n import _
+from electrum.lnhtlc import HTLCStateMachine
+from .util import MyTreeWidget, SortableTreeWidgetItem, WindowModalDialog, Buttons, OkButton, CancelButton
+from .amountedit import BTCAmountEdit
+
+class ChannelsList(MyTreeWidget):
+    update_rows = QtCore.pyqtSignal()
+    update_single_row = QtCore.pyqtSignal(HTLCStateMachine)
+
+    def __init__(self, parent):
+        MyTreeWidget.__init__(self, parent, self.create_menu, [_('Node ID'), _('Balance'), _('Remote'), _('Status')], 0)
+        self.main_window = parent
+        self.update_rows.connect(self.do_update_rows)
+        self.update_single_row.connect(self.do_update_single_row)
+        self.status = QLabel('')
+
+    def format_fields(self, chan):
+        status = self.parent.wallet.lnworker.channel_state[chan.channel_id]
+        return [
+            bh2u(chan.node_id),
+            self.parent.format_amount(chan.local_state.amount_msat//1000),
+            self.parent.format_amount(chan.remote_state.amount_msat//1000),
+            status
+        ]
+
+    def create_menu(self, position):
+        menu = QMenu()
+        channel_id = self.currentItem().data(0, QtCore.Qt.UserRole)
+        print('ID', bh2u(channel_id))
+        def close():
+            suc, msg = self.parent.wallet.lnworker.close_channel(channel_id)
+            assert suc # TODO show error message in dialog
+        menu.addAction(_("Close channel"), close)
+        menu.exec_(self.viewport().mapToGlobal(position))
+
+    @QtCore.pyqtSlot(HTLCStateMachine)
+    def do_update_single_row(self, chan):
+        for i in range(self.topLevelItemCount()):
+            item = self.topLevelItem(i)
+            if item.data(0, QtCore.Qt.UserRole) == chan.channel_id:
+                for i, v in enumerate(self.format_fields(chan)):
+                    item.setData(i, QtCore.Qt.DisplayRole, v)
+
+    @QtCore.pyqtSlot()
+    def do_update_rows(self):
+        self.clear()
+        for chan in self.parent.wallet.lnworker.channels.values():
+            item = SortableTreeWidgetItem(self.format_fields(chan))
+            item.setData(0, QtCore.Qt.UserRole, chan.channel_id)
+            self.insertTopLevelItem(0, item)
+
+    def get_toolbar(self):
+        b = QPushButton(_('Open Channel'))
+        b.clicked.connect(self.new_channel_dialog)
+        h = QHBoxLayout()
+        h.addWidget(self.status)
+        h.addStretch()
+        h.addWidget(b)
+        return h
+
+    def on_update(self):
+        n = len(self.parent.network.lightning_nodes)
+        np = len(self.parent.wallet.lnworker.peers)
+        self.status.setText(_('{} peers, {} nodes').format(np, n))
+
+    def new_channel_dialog(self):
+        d = WindowModalDialog(self.parent, _('Open Channel'))
+        d.setFixedWidth(700)
+        vbox = QVBoxLayout(d)
+        h = QGridLayout()
+        local_nodeid = QLineEdit()
+        local_nodeid.setText(bh2u(self.parent.wallet.lnworker.pubkey))
+        local_nodeid.setReadOnly(True)
+        local_nodeid.setCursorPosition(0)
+        remote_nodeid = QLineEdit()
+        local_amt_inp = BTCAmountEdit(self.parent.get_decimal_point)
+        local_amt_inp.setAmount(200000)
+        push_amt_inp = BTCAmountEdit(self.parent.get_decimal_point)
+        push_amt_inp.setAmount(0)
+        h.addWidget(QLabel(_('Your Node ID')), 0, 0)
+        h.addWidget(local_nodeid, 0, 1)
+        h.addWidget(QLabel(_('Remote Node ID or connection string')), 1, 0)
+        h.addWidget(remote_nodeid, 1, 1)
+        h.addWidget(QLabel('Local amount'), 2, 0)
+        h.addWidget(local_amt_inp, 2, 1)
+        h.addWidget(QLabel('Push amount'), 3, 0)
+        h.addWidget(push_amt_inp, 3, 1)
+        vbox.addLayout(h)
+        vbox.addLayout(Buttons(CancelButton(d), OkButton(d)))
+        if not d.exec_():
+            return
+        connect_contents = str(remote_nodeid.text())
+        rest = None
+        try:
+            nodeid_hex, rest = connect_contents.split("@")
+        except ValueError:
+            nodeid_hex = connect_contents
+        try:
+            node_id = bfh(nodeid_hex)
+            assert len(node_id) == 33
+        except:
+            self.parent.show_error(_('Invalid node ID, must be 33 bytes and hexadecimal'))
+            return
+
+        local_amt = local_amt_inp.get_amount()
+        push_amt = push_amt_inp.get_amount()
+
+        if local_amt < 200000:
+            self.parent.show_error(_('You must specify an decent amount (>=2 mBTC) for ' + \
+                                     'the initial local balance of the channel.'))
+            return
+
+        known = node_id in self.parent.network.lightning_nodes
+        connected = node_id in self.parent.wallet.lnworker.peers
+
+        open_channel_info = (node_id, local_amt, push_amt)
+
+        if connected:
+            peer = self.parent.wallet.lnworker.peers[node_id]
+            if not peer.initialized.done():
+                self.parent.show_error(_('Peer not initialized'))
+                return
+            self.gui_open_channel(None, peer, open_channel_info)
+            return
+
+        if rest is not None: # perfer ip/port from input field
+            try:
+                host, port = rest.split(":")
+            except ValueError:
+                self.parent.show_error(_('Connection strings must be in <node_pubkey>@<host>:<port> format'))
+
+        elif known: # then use config file
+            node = self.network.lightning_nodes.get(node_id)
+            host, port = node['addresses'][0]
+        else:
+            self.parent.show_error(_('Unknown node:') + ' ' + nodeid_hex)
+            return
+
+        try:
+            int(port)
+        except:
+            self.parent.show_error(_('Port number must be decimal'))
+            return
+
+        peer, coro = self.parent.wallet.lnworker.add_peer(host, port, node_id, aiosafe=False)
+        q = QtCore.QTimer()
+        q.singleShot(3000, lambda: self.gui_open_channel(coro, peer, open_channel_info))
+
+    def gui_open_channel(self, coro, peer, open_channel_info):
+        if coro is not None:
+            try:
+                raise coro.exception(timeout=0)
+            except concurrent.futures.TimeoutError:
+                pass
+            except concurrent.futures.CancelledError:
+                print("Ignoring CancelledError, probably shutting down since main_loop cancelled...")
+                return
+            except Exception as e:
+                self.parent.show_error(_('LN main loop threw:') + ' ' + str(e))
+                return
+
+        if not peer.initialized.done():
+            self.parent.show_error(_('Peer not initialized within 3 seconds'))
+            return
+        self.main_window.protect(self.open_channel, open_channel_info)
+
+    def open_channel(self, *args, **kwargs):
+        self.parent.wallet.lnworker.open_channel(*args, **kwargs)

gui/qt/main_window.py

@@ -62,6 +62,7 @@ from .transaction_dialog import show_transaction
 from .fee_slider import FeeSlider
 from .util import *
 from .installwizard import WIF_HELP_TEXT
+from .channels_list import ChannelsList
 
 
 class StatusBarButton(QPushButton):
@@ -139,6 +140,7 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError):
         self.utxo_tab = self.create_utxo_tab()
         self.console_tab = self.create_console_tab()
         self.contacts_tab = self.create_contacts_tab()
+        self.channels_tab = self.create_channels_tab(wallet)
         tabs.addTab(self.create_history_tab(), QIcon(":icons/tab_history.png"), _('History'))
         tabs.addTab(self.send_tab, QIcon(":icons/tab_send.png"), _('Send'))
         tabs.addTab(self.receive_tab, QIcon(":icons/tab_receive.png"), _('Receive'))
@@ -152,6 +154,7 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError):
                 tabs.addTab(tab, icon, description.replace("&", ""))
 
         add_optional_tab(tabs, self.addresses_tab, QIcon(":icons/tab_addresses.png"), _("&Addresses"), "addresses")
+        add_optional_tab(tabs, self.channels_tab, QIcon(":icons/lightning.png"), _("Channels"), "channels")
         add_optional_tab(tabs, self.utxo_tab, QIcon(":icons/tab_coins.png"), _("Co&ins"), "utxo")
         add_optional_tab(tabs, self.contacts_tab, QIcon(":icons/tab_contacts.png"), _("Con&tacts"), "contacts")
         add_optional_tab(tabs, self.console_tab, QIcon(":icons/tab_console.png"), _("Con&sole"), "console")
@@ -184,7 +187,8 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError):
         if self.network:
             self.network_signal.connect(self.on_network_qt)
             interests = ['updated', 'new_transaction', 'status',
-                         'banner', 'verified', 'fee']
+                         'banner', 'verified', 'fee', 'on_quotes',
+                         'on_history', 'channel', 'channels']
             # To avoid leaking references to "self" that prevent the
             # window from being GC-ed when closed, callbacks should be
             # methods of this class only, and specifically not be
@@ -192,8 +196,6 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError):
             self.network.register_callback(self.on_network, interests)
             # set initial message
             self.console.showMessage(self.network.banner)
-            self.network.register_callback(self.on_quotes, ['on_quotes'])
-            self.network.register_callback(self.on_history, ['on_history'])
             self.new_fx_quotes_signal.connect(self.on_fx_quotes)
             self.new_fx_history_signal.connect(self.on_fx_history)
 
@@ -203,9 +205,6 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError):
         self.connect_slots(gui_object.timer)
         self.fetch_alias()
 
-    def on_history(self, b):
-        self.new_fx_history_signal.emit()
-
     def setup_exception_hook(self):
         Exception_Hook(self)
 
@@ -214,9 +213,6 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError):
         self.history_list.update()
         self.address_list.update()
 
-    def on_quotes(self, b):
-        self.new_fx_quotes_signal.emit()
-
     def on_fx_quotes(self):
         self.update_status()
         # Refresh edits with the new rate
@@ -297,6 +293,14 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError):
         elif event in ['status', 'banner', 'verified', 'fee']:
             # Handle in GUI thread
             self.network_signal.emit(event, args)
+        elif event == 'on_quotes':
+            self.new_fx_quotes_signal.emit()
+        elif event == 'on_history':
+            self.new_fx_history_signal.emit()
+        elif event == 'channels':
+            self.channels_list.update_rows.emit(*args)
+        elif event == 'channel':
+            self.channels_list.update_single_row.emit(*args)
         else:
             self.print_error("unexpected network message:", event, args)
 
@@ -347,6 +351,7 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError):
         self.history_list.update()
         self.address_list.update()
         self.utxo_list.update()
+        wallet.lnworker.on_channels_updated()
         self.need_update.set()
         # Once GUI has been initialized check if we want to announce something since the callback has been called before the GUI was initialized
         self.notify_transactions()
@@ -357,6 +362,7 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError):
         self.update_console()
         self.clear_receive_tab()
         self.request_list.update()
+        self.channels_list.update()
         self.tabs.show()
         self.init_geometry()
         if self.config.get('hide_gui') and self.gui_object.tray.isVisible():
@@ -517,6 +523,7 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError):
         view_menu = menubar.addMenu(_("&View"))
         add_toggle_action(view_menu, self.addresses_tab)
         add_toggle_action(view_menu, self.utxo_tab)
+        add_toggle_action(view_menu, self.channels_tab)
         add_toggle_action(view_menu, self.contacts_tab)
         add_toggle_action(view_menu, self.console_tab)
 
@@ -759,6 +766,11 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError):
         self.invoice_list.update()
         self.update_completions()
 
+    def create_channels_tab(self, wallet):
+        self.channels_list = ChannelsList(self)
+        t = self.channels_list.get_toolbar()
+        return self.create_list_tab(self.channels_list, t)
+
     def create_history_tab(self):
         from .history_list import HistoryList
         self.history_list = l = HistoryList(self)
@@ -784,16 +796,6 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError):
         grid.setSpacing(8)
         grid.setColumnStretch(3, 1)
 
-        self.receive_address_e = ButtonsLineEdit()
-        self.receive_address_e.addCopyButton(self.app)
-        self.receive_address_e.setReadOnly(True)
-        msg = _('Bitcoin address where the payment should be received. Note that each payment request uses a different Bitcoin address.')
-        self.receive_address_label = HelpLabel(_('Receiving address'), msg)
-        self.receive_address_e.textChanged.connect(self.update_receive_qr)
-        self.receive_address_e.setFocusPolicy(Qt.ClickFocus)
-        grid.addWidget(self.receive_address_label, 0, 0)
-        grid.addWidget(self.receive_address_e, 0, 1, 1, -1)
-
         self.receive_message_e = QLineEdit()
         grid.addWidget(QLabel(_('Description')), 1, 0)
         grid.addWidget(self.receive_message_e, 1, 1, 1, -1)
@@ -828,23 +830,30 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError):
         self.expires_label.hide()
         grid.addWidget(self.expires_label, 3, 1)
 
-        self.save_request_button = QPushButton(_('Save'))
-        self.save_request_button.clicked.connect(self.save_payment_request)
+        self.receive_type = QComboBox()
+        self.receive_type.addItems([_('Bitcoin address'), _('Lightning')])
+        grid.addWidget(QLabel(_('Type')), 4, 0)
+        grid.addWidget(self.receive_type, 4, 1)
 
-        self.new_request_button = QPushButton(_('New'))
-        self.new_request_button.clicked.connect(self.new_payment_request)
+        self.save_request_button = QPushButton(_('Create'))
+        self.save_request_button.clicked.connect(self.create_invoice)
+
+        self.receive_buttons = buttons = QHBoxLayout()
+        buttons.addWidget(self.save_request_button)
+        buttons.addStretch(1)
+        grid.addLayout(buttons, 4, 2, 1, 2)
+
+        self.receive_address_e = ButtonsTextEdit()
+        self.receive_address_e.addCopyButton(self.app)
+        self.receive_address_e.setReadOnly(True)
+        self.receive_address_e.textChanged.connect(self.update_receive_qr)
+        self.receive_address_e.setFocusPolicy(Qt.ClickFocus)
 
         self.receive_qr = QRCodeWidget(fixedSize=200)
         self.receive_qr.mouseReleaseEvent = lambda x: self.toggle_qr_window()
         self.receive_qr.enterEvent = lambda x: self.app.setOverrideCursor(QCursor(Qt.PointingHandCursor))
         self.receive_qr.leaveEvent = lambda x: self.app.setOverrideCursor(QCursor(Qt.ArrowCursor))
 
-        self.receive_buttons = buttons = QHBoxLayout()
-        buttons.addStretch(1)
-        buttons.addWidget(self.save_request_button)
-        buttons.addWidget(self.new_request_button)
-        grid.addLayout(buttons, 4, 1, 1, 2)
-
         self.receive_requests_label = QLabel(_('Requests'))
 
         from .request_list import RequestList
@@ -855,14 +864,19 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError):
         vbox_g.addLayout(grid)
         vbox_g.addStretch()
 
+        hbox_r = QHBoxLayout()
+        hbox_r.addWidget(self.receive_qr)
+        hbox_r.addWidget(self.receive_address_e)
+
         hbox = QHBoxLayout()
         hbox.addLayout(vbox_g)
-        hbox.addWidget(self.receive_qr)
+        hbox.addLayout(hbox_r)
 
         w = QWidget()
         w.searchable_list = self.request_list
         vbox = QVBoxLayout(w)
         vbox.addLayout(hbox)
+
         vbox.addStretch(1)
         vbox.addWidget(self.receive_requests_label)
         vbox.addWidget(self.request_list)
@@ -876,13 +890,18 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError):
         self.request_list.update()
         self.clear_receive_tab()
 
+    def delete_lightning_payreq(self, payreq_key):
+        self.wallet.lnworker.delete_invoice(payreq_key)
+        self.request_list.update()
+        self.clear_receive_tab()
+
     def get_request_URI(self, addr):
         req = self.wallet.receive_requests[addr]
         message = self.wallet.labels.get(addr, '')
         amount = req['amount']
         URI = util.create_URI(addr, amount, message)
-        if req.get('time'):
-            URI += "&time=%d"%req.get('time')
+        #if req.get('time'):
+        #    URI += "&time=%d"%req.get('time')
         if req.get('exp'):
             URI += "&exp=%d"%req.get('exp')
         if req.get('name') and req.get('sig'):
@@ -913,15 +932,34 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError):
                 else:
                     return
 
-    def save_payment_request(self):
-        addr = str(self.receive_address_e.text())
+    def create_invoice(self):
         amount = self.receive_amount_e.get_amount()
         message = self.receive_message_e.text()
-        if not message and not amount:
-            self.show_error(_('No message or amount'))
-            return False
         i = self.expires_combo.currentIndex()
         expiration = list(map(lambda x: x[1], expiration_values))[i]
+        if self.receive_type.currentIndex() == 1:
+            self.create_lightning_request(amount, message, expiration)
+        else:
+            self.create_bitcoin_request(amount, message, expiration)
+        self.request_list.update()
+
+    def create_lightning_request(self, amount, message, expiration):
+        req = self.wallet.lnworker.add_invoice(amount, message)
+
+    def create_bitcoin_request(self, amount, message, expiration):
+        addr = self.wallet.get_unused_address()
+        if addr is None:
+            if not self.wallet.is_deterministic():
+                msg = [
+                    _('No more addresses in your wallet.'),
+                    _('You are using a non-deterministic wallet, which cannot create new addresses.'),
+                    _('If you want to create new addresses, use a deterministic wallet instead.')
+                   ]
+                self.show_message(' '.join(msg))
+                return
+            if not self.question(_("Warning: The next address will not be recovered automatically if you restore your wallet from seed; you may need to add it manually.\n\nThis occurs because you have too many unused addresses in your wallet. To avoid this situation, use the existing addresses first.\n\nCreate anyway?")):
+                return
+            addr = self.wallet.create_new_address(False)
         req = self.wallet.make_payment_request(addr, amount, message, expiration)
         try:
             self.wallet.add_payment_request(req, self.config)
@@ -930,9 +968,8 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError):
             self.show_error(_('Error adding payment request') + ':\n' + str(e))
         else:
             self.sign_payment_request(addr)
-            self.save_request_button.setEnabled(False)
+            #self.save_request_button.setEnabled(False)
         finally:
-            self.request_list.update()
             self.address_list.update()
 
     def view_and_paste(self, title, msg, data):
@@ -958,26 +995,6 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError):
             self.show_message(_("Request saved successfully"))
             self.saved = True
 
-    def new_payment_request(self):
-        addr = self.wallet.get_unused_address()
-        if addr is None:
-            if not self.wallet.is_deterministic():
-                msg = [
-                    _('No more addresses in your wallet.'),
-                    _('You are using a non-deterministic wallet, which cannot create new addresses.'),
-                    _('If you want to create new addresses, use a deterministic wallet instead.')
-                   ]
-                self.show_message(' '.join(msg))
-                return
-            if not self.question(_("Warning: The next address will not be recovered automatically if you restore your wallet from seed; you may need to add it manually.\n\nThis occurs because you have too many unused addresses in your wallet. To avoid this situation, use the existing addresses first.\n\nCreate anyway?")):
-                return
-            addr = self.wallet.create_new_address(False)
-        self.set_receive_address(addr)
-        self.expires_label.hide()
-        self.expires_combo.show()
-        self.new_request_button.setEnabled(False)
-        self.receive_message_e.setFocus(1)
-
     def set_receive_address(self, addr):
         self.receive_address_e.setText(addr)
         self.receive_message_e.setText('')
@@ -1020,14 +1037,10 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError):
         self.new_request_button.setEnabled(True)
 
     def update_receive_qr(self):
-        addr = str(self.receive_address_e.text())
-        amount = self.receive_amount_e.get_amount()
-        message = self.receive_message_e.text()
-        self.save_request_button.setEnabled((amount is not None) or (message != ""))
-        uri = util.create_URI(addr, amount, message)
+        uri = str(self.receive_address_e.text())
         self.receive_qr.setData(uri)
         if self.qr_window and self.qr_window.isVisible():
-            self.qr_window.set_content(addr, amount, message, uri)
+            self.qr_window.set_content(uri, '', '', uri)
 
     def set_feerounding_text(self, num_satoshis_added):
         self.feerounding_text = (_('Additional {} satoshis are going to be added.')
@@ -1431,6 +1444,10 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError):
             return func(self, *args, **kwargs)
         return request_password
 
+    @protected
+    def protect(self, func, args, password):
+        return func(*args, password)
+
     def is_send_fee_frozen(self):
         return self.fee_e.isVisible() and self.fee_e.isModified() \
                and (self.fee_e.text() or self.fee_e.hasFocus())
@@ -1496,7 +1513,15 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError):
     def do_preview(self):
         self.do_send(preview = True)
 
+    def pay_lightning_invoice(self, invoice):
+        f = self.wallet.lnworker.pay(invoice)
+        self.do_clear()
+
     def do_send(self, preview = False):
+        if self.payto_e.is_lightning:
+            self.pay_lightning_invoice(self.payto_e.lightning_invoice)
+            return
+        #
         if run_hook('abort_send', self):
             return
         r = self.read_send_tab()
@@ -1694,6 +1719,23 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError):
         else:
             self.payment_request_error_signal.emit()
 
+    def parse_lightning_invoice(self, invoice):
+        from electrum.lnaddr import lndecode
+        lnaddr = lndecode(invoice, expected_hrp=constants.net.SEGWIT_HRP)
+        pubkey = bh2u(lnaddr.pubkey.serialize())
+        for k,v in lnaddr.tags:
+            if k == 'd':
+                description = v
+                break
+        else:
+             description = ''
+        self.payto_e.setFrozen(True)
+        self.payto_e.setText(pubkey)
+        self.message_e.setText(description)
+        self.amount_e.setAmount(lnaddr.amount * COIN)
+        #self.amount_e.textEdited.emit("")
+        self.payto_e.is_lightning = True
+
     def pay_to_URI(self, URI):
         if not URI:
             return
@@ -1730,6 +1772,7 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError):
         self.not_enough_funds = False
         self.payment_request = None
         self.payto_e.is_pr = False
+        self.payto_e.is_lightning = False
         for e in [self.payto_e, self.message_e, self.amount_e, self.fiat_send_e,
                   self.fee_e, self.feerate_e]:
             e.setText('')
@@ -1754,8 +1797,8 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError):
         w.searchable_list = l
         vbox = QVBoxLayout()
         w.setLayout(vbox)
-        vbox.setContentsMargins(0, 0, 0, 0)
-        vbox.setSpacing(0)
+        #vbox.setContentsMargins(0, 0, 0, 0)
+        #vbox.setSpacing(0)
         if toolbar:
             vbox.addLayout(toolbar)
         vbox.addWidget(l)

gui/qt/paytoedit.py

@@ -56,10 +56,9 @@ class PayToEdit(CompletionTextEdit, ScanQRTextEdit):
         self.errors = []
         self.is_pr = False
         self.is_alias = False
-        self.scan_f = win.pay_to_URI
+        self.is_lightning = False
         self.update_size()
         self.payto_address = None
-
         self.previous_payto = ''
 
     def setFrozen(self, b):
@@ -127,7 +126,11 @@ class PayToEdit(CompletionTextEdit, ScanQRTextEdit):
         if len(lines) == 1:
             data = lines[0]
             if data.startswith("bitcoin:"):
-                self.scan_f(data)
+                self.win.pay_to_URI(data)
+                return
+            if data.startswith("ln"):
+                self.win.parse_lightning_invoice(data)
+                self.lightning_invoice = data
                 return
             try:
                 self.payto_address = self.parse_output(data)
@@ -201,7 +204,7 @@ class PayToEdit(CompletionTextEdit, ScanQRTextEdit):
     def qr_input(self):
         data = super(PayToEdit,self).qr_input()
         if data.startswith("bitcoin:"):
-            self.scan_f(data)
+            self.win.pay_to_URI(data)
             # TODO: update fee
 
     def resolve(self):

gui/qt/request_list.py

@@ -29,21 +29,36 @@ from electrum.plugins import run_hook
 from electrum.paymentrequest import PR_UNKNOWN
 from PyQt5.QtGui import *
 from PyQt5.QtCore import *
-from PyQt5.QtWidgets import QTreeWidgetItem, QMenu
+from PyQt5.QtWidgets import QTreeWidgetItem, QMenu, QHeaderView
+
+from electrum.bitcoin import COIN
+
 from .util import MyTreeWidget, pr_tooltips, pr_icons
 
 
-class RequestList(MyTreeWidget):
-    filter_columns = [0, 1, 2, 3, 4]  # Date, Account, Address, Description, Amount
+REQUEST_TYPE_BITCOIN = 0
+REQUEST_TYPE_LN = 1
 
 
+class RequestList(MyTreeWidget):
+    filter_columns = [0, 1, 2, 3]  # Date, Address, Description, Amount
+
     def __init__(self, parent):
-        MyTreeWidget.__init__(self, parent, self.create_menu, [_('Date'), _('Address'), '', _('Description'), _('Amount'), _('Status')], 3)
+        MyTreeWidget.__init__(self, parent, self.create_menu, [_('Date'), _('Address'), _('Description'), _('Amount'), _('Status')], 2)
         self.currentItemChanged.connect(self.item_changed)
         self.itemClicked.connect(self.item_changed)
         self.setSortingEnabled(True)
         self.setColumnWidth(0, 180)
-        self.hideColumn(1)
+        self.setColumnWidth(1, 250)
+
+    def update_headers(self, headers):
+        self.setColumnCount(len(headers))
+        self.setHeaderLabels(headers)
+        self.header().setStretchLastSection(False)
+        for col in range(len(headers)):
+            if col in [1]: continue
+            sm = QHeaderView.Stretch if col == self.stretch_column else QHeaderView.ResizeToContents
+            self.header().setSectionResizeMode(col, sm)
 
     def item_changed(self, item):
         if item is None:
@@ -51,25 +66,25 @@ class RequestList(MyTreeWidget):
         if not item.isSelected():
             return
         addr = str(item.text(1))
-        req = self.wallet.receive_requests.get(addr)
-        if req is None:
-            self.update()
-            return
-        expires = age(req['time'] + req['exp']) if req.get('exp') else _('Never')
-        amount = req['amount']
-        message = self.wallet.labels.get(addr, '')
         self.parent.receive_address_e.setText(addr)
-        self.parent.receive_message_e.setText(message)
-        self.parent.receive_amount_e.setAmount(amount)
-        self.parent.expires_combo.hide()
-        self.parent.expires_label.show()
-        self.parent.expires_label.setText(expires)
-        self.parent.new_request_button.setEnabled(True)
+        #req = self.wallet.receive_requests.get(addr)
+        #if req is None:
+        #    self.update()
+        #    return
+        #expires = age(req['time'] + req['exp']) if req.get('exp') else _('Never')
+        #amount = req['amount']
+        #message = self.wallet.labels.get(addr, '')
+        #self.parent.receive_message_e.setText(message)
+        #self.parent.receive_amount_e.setAmount(amount)
+        #self.parent.expires_combo.hide()
+        #self.parent.expires_label.show()
+        #self.parent.expires_label.setText(expires)
+        #self.parent.new_request_button.setEnabled(True)
 
     def on_update(self):
         self.wallet = self.parent.wallet
         # hide receive tab if no receive requests available
-        b = len(self.wallet.receive_requests) > 0
+        b = len(self.wallet.receive_requests) > 0 or len(self.wallet.lnworker.invoices) > 0
         self.setVisible(b)
         self.parent.receive_requests_label.setVisible(b)
         if not b:
@@ -77,12 +92,12 @@ class RequestList(MyTreeWidget):
             self.parent.expires_combo.show()
 
         # update the receive address if necessary
-        current_address = self.parent.receive_address_e.text()
+        #current_address = self.parent.receive_address_e.text()
         domain = self.wallet.get_receiving_addresses()
-        addr = self.wallet.get_unused_address()
-        if not current_address in domain and addr:
-            self.parent.set_receive_address(addr)
-        self.parent.new_request_button.setEnabled(addr != current_address)
+        #addr = self.wallet.get_unused_address()
+        #if not current_address in domain and addr:
+        #    self.parent.set_receive_address(addr)
+        #self.parent.new_request_button.setEnabled(addr != current_address)
 
         # clear the list and fill it again
         self.clear()
@@ -99,20 +114,51 @@ class RequestList(MyTreeWidget):
             signature = req.get('sig')
             requestor = req.get('name', '')
             amount_str = self.parent.format_amount(amount) if amount else ""
-            item = QTreeWidgetItem([date, address, '', message, amount_str, pr_tooltips.get(status,'')])
+            URI = self.parent.get_request_URI(address)
+            item = QTreeWidgetItem([date, URI, message, amount_str, pr_tooltips.get(status,'')])
             if signature is not None:
-                item.setIcon(2, self.icon_cache.get(":icons/seal.png"))
-                item.setToolTip(2, 'signed by '+ requestor)
+                item.setIcon(1, self.icon_cache.get(":icons/seal.png"))
+                item.setToolTip(1, 'signed by '+ requestor)
             if status is not PR_UNKNOWN:
                 item.setIcon(6, self.icon_cache.get(pr_icons.get(status)))
+            item.setData(0, Qt.UserRole, REQUEST_TYPE_BITCOIN)
+            item.setData(0, Qt.UserRole+1, address)
+            self.addTopLevelItem(item)
+        # lightning
+        for payreq_key, r in self.wallet.lnworker.invoices.items():
+            from electrum.lnaddr import lndecode
+            import electrum.constants as constants
+            lnaddr = lndecode(r, expected_hrp=constants.net.SEGWIT_HRP)
+            amount_sat = lnaddr.amount*COIN if lnaddr.amount else None
+            amount_str = self.parent.format_amount(amount_sat) if amount_sat else ''
+            for k,v in lnaddr.tags:
+                if k == 'd':
+                    description = v
+                    break
+                else:
+                    description = ''
+            date = format_time(lnaddr.date)
+            item = QTreeWidgetItem([date, r, description, amount_str, ''])
+            item.setIcon(1, self.icon_cache.get(":icons/lightning.png"))
+            item.setData(0, Qt.UserRole, REQUEST_TYPE_LN)
+            item.setData(0, Qt.UserRole+1, payreq_key)
             self.addTopLevelItem(item)
-
 
     def create_menu(self, position):
         item = self.itemAt(position)
         if not item:
             return
-        addr = str(item.text(1))
+        request_type = item.data(0, Qt.UserRole)
+        menu = None
+        if request_type == REQUEST_TYPE_BITCOIN:
+            menu = self.create_menu_bitcoin_payreq(item)
+        elif request_type == REQUEST_TYPE_LN:
+            menu = self.create_menu_ln_payreq(item)
+        if menu:
+            menu.exec_(self.viewport().mapToGlobal(position))
+
+    def create_menu_bitcoin_payreq(self, item):
+        addr = str(item.data(0, Qt.UserRole + 1))
         req = self.wallet.receive_requests.get(addr)
         if req is None:
             self.update()
@@ -126,4 +172,19 @@ class RequestList(MyTreeWidget):
         menu.addAction(_("Save as BIP70 file"), lambda: self.parent.export_payment_request(addr))
         menu.addAction(_("Delete"), lambda: self.parent.delete_payment_request(addr))
         run_hook('receive_list_menu', menu, addr)
-        menu.exec_(self.viewport().mapToGlobal(position))
+        return menu
+
+    def create_menu_ln_payreq(self, item):
+        payreq_key = item.data(0, Qt.UserRole + 1)
+        req = self.wallet.lnworker.invoices.get(payreq_key)
+        if req is None:
+            self.update()
+            return
+        column = self.currentColumn()
+        column_title = self.headerItem().text(column)
+        column_data = item.text(column)
+        menu = QMenu(self)
+        menu.addAction(_("Copy {}").format(column_title), lambda: self.parent.app.clipboard().setText(column_data))
+        menu.addAction(_("Copy URI"), lambda: self.parent.view_and_paste('URI', '', req))
+        menu.addAction(_("Delete"), lambda: self.parent.delete_lightning_payreq(payreq_key))
+        return menu

icons.qrc

@@ -22,6 +22,7 @@
     <file>icons/key.png</file>
     <file>icons/ledger.png</file>
     <file>icons/ledger_unpaired.png</file>
+    <file>icons/lightning.png</file>
     <file>icons/lock.png</file>
     <file>icons/microphone.png</file>
     <file>icons/network.png</file>

lib/commands.py

@@ -23,6 +23,7 @@
 # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
+import queue
 import sys
 import datetime
 import copy
@@ -680,6 +681,31 @@ class Commands:
         # for the python console
         return sorted(known_commands.keys())
 
+    # lightning network commands
+    @command('wpn')
+    def open_channel(self, node_id, amount, channel_push=0, password=None):
+        f = self.wallet.lnworker.open_channel(bytes.fromhex(node_id), satoshis(amount), satoshis(channel_push), password)
+        return f.result()
+
+    @command('wn')
+    def reestablish_channel(self):
+        self.wallet.lnworker.reestablish_channel()
+
+    @command('wn')
+    def lnpay(self, invoice):
+        f = self.wallet.lnworker.pay(invoice)
+        return f.result()
+
+    @command('wn')
+    def addinvoice(self, requested_amount, message):
+        # using requested_amount because it is documented in param_descriptions
+        return self.wallet.lnworker.add_invoice(satoshis(requested_amount), message)
+
+    @command('wn')
+    def listchannels(self):
+        return self.wallet.lnworker.list_channels()
+
+
 param_descriptions = {
     'privkey': 'Private key. Type \'?\' to get a prompt.',
     'destination': 'Bitcoin address, contact or alias',
@@ -727,6 +753,7 @@ command_options = {
     'timeout':     (None, "Timeout in seconds"),
     'force':       (None, "Create new address beyond gap limit, if no more addresses are available."),
     'pending':     (None, "Show only pending requests."),
+    'channel_push':(None, 'Push initial amount (in BTC)'),
     'expired':     (None, "Show only expired requests."),
     'paid':        (None, "Show only paid requests."),
     'show_addresses': (None, "Show input and output addresses"),

lib/constants.py

@@ -95,6 +95,16 @@ class BitcoinTestnet:
     BIP44_COIN_TYPE = 1
 
 
+class BitcoinSimnet(BitcoinTestnet):
+    ADDRTYPE_P2PKH = 0x3f
+    ADDRTYPE_P2SH = 0x7b
+    SEGWIT_HRP = "sb"
+    GENESIS = "683e86bd5c6d110d91b94b97137ba6bfe02dbbdb8e3dff722a669b5d69d77af6"
+    WIF_PREFIX = 0x00
+    DEFAULT_SERVERS = read_json('servers_regtest.json', {})  # Note: regtest!
+    CHECKPOINTS = []
+
+
 class BitcoinRegtest(BitcoinTestnet):
 
     SEGWIT_HRP = "bcrt"
@@ -122,6 +132,9 @@ def set_mainnet():
     global net
     net = BitcoinMainnet
 
+def set_simnet():
+    global net
+    net = BitcoinSimnet
 
 def set_testnet():
     global net

lib/crypto.py

@@ -132,11 +132,15 @@ def Hash(x: bytes) -> bytes:
 
 
 def hash_160(x: bytes) -> bytes:
+    return ripemd(sha256(x))
+
+def ripemd(x):
     try:
         md = hashlib.new('ripemd160')
-        md.update(sha256(x))
+        md.update(x)
         return md.digest()
     except BaseException:
         from . import ripemd
-        md = ripemd.new(sha256(x))
+        md = ripemd.new(x)
         return md.digest()
+

lib/daemon.py

@@ -282,7 +282,11 @@ class Daemon(DaemonThread):
             kwargs[x] = (config_options.get(x) if x in ['password', 'new_password'] else config.get(x))
         cmd_runner = Commands(config, wallet, self.network)
         func = getattr(cmd_runner, cmd.name)
-        result = func(*args, **kwargs)
+        try:
+            result = func(*args, **kwargs)
+        except TypeError as e:
+            # we are catching here because JSON-RPC-Pelix would throw away the trace of e
+            raise Exception("TypeError occured in Electrum") from e
         return result
 
     def run(self):

lib/ecc.py

@@ -252,9 +252,9 @@ class ECPubkey(object):
     def __ne__(self, other):
         return not (self == other)
 
-    def verify_message_for_address(self, sig65: bytes, message: bytes) -> None:
+    def verify_message_for_address(self, sig65: bytes, message: bytes, algo=lambda x: Hash(msg_magic(x))) -> None:
         assert_bytes(message)
-        h = Hash(msg_magic(message))
+        h = algo(message)
         public_key, compressed = self.from_signature65(sig65, h)
         # check public key
         if public_key != self:
@@ -303,6 +303,13 @@ def msg_magic(message: bytes) -> bytes:
     return b"\x18Bitcoin Signed Message:\n" + length + message
 
 
+def verify_signature(pubkey, sig, h):
+    try:
+        ECPubkey(pubkey).verify_message_hash(sig, h)
+    except:
+        return False
+    return True
+
 def verify_message_with_address(address: str, sig65: bytes, message: bytes):
     from .bitcoin import pubkey_to_address
     assert_bytes(sig65, message)
@@ -384,12 +391,12 @@ class ECPrivkey(ECPubkey):
                          sigencode=der_sig_from_r_and_s,
                          sigdecode=get_r_and_s_from_der_sig)
 
-    def sign_message(self, message: bytes, is_compressed: bool) -> bytes:
+    def sign_message(self, message: bytes, is_compressed: bool, algo=lambda x: Hash(msg_magic(x))) -> bytes:
         def bruteforce_recid(sig_string):
             for recid in range(4):
                 sig65 = construct_sig65(sig_string, recid, is_compressed)
                 try:
-                    self.verify_message_for_address(sig65, message)
+                    self.verify_message_for_address(sig65, message, algo)
                     return sig65, recid
                 except Exception as e:
                     continue
@@ -397,7 +404,7 @@ class ECPrivkey(ECPubkey):
                 raise Exception("error: cannot sign message. no recid fits..")
 
         message = to_bytes(message, 'utf8')
-        msg_hash = Hash(msg_magic(message))
+        msg_hash = algo(message)
         sig_string = self.sign(msg_hash,
                                sigencode=sig_string_from_r_and_s,
                                sigdecode=get_r_and_s_from_sig_string)

lib/keystore.py

@@ -334,7 +334,10 @@ class BIP32_KeyStore(Deterministic_KeyStore, Xpub):
         pk = bip32_private_key(sequence, k, c)
         return pk, True
 
-
+    def get_keypair(self, sequence, password):
+        k, _ = self.get_private_key(sequence, password)
+        cK = ecc.ECPrivkey(k).get_public_key_bytes()
+        return cK, k
 
 class Old_KeyStore(Deterministic_KeyStore):
 

lib/lightning.json

@@ -0,0 +1,801 @@
+{
+ "init": {
+  "type": "16",
+  "payload": {
+   "gflen": {
+    "position": "0",
+    "length": "2"
+   },
+   "globalfeatures": {
+    "position": "2",
+    "length": "gflen"
+   },
+   "lflen": {
+    "position": "2+gflen",
+    "length": "2"
+   },
+   "localfeatures": {
+    "position": "4+gflen",
+    "length": "lflen"
+   }
+  }
+ },
+ "error": {
+  "type": "17",
+  "payload": {
+   "channel_id": {
+    "position": "0",
+    "length": "32"
+   },
+   "len": {
+    "position": "32",
+    "length": "2"
+   },
+   "data": {
+    "position": "34",
+    "length": "len"
+   }
+  }
+ },
+ "ping": {
+  "type": "18",
+  "payload": {
+   "num_pong_bytes": {
+    "position": "0",
+    "length": "2"
+   },
+   "byteslen": {
+    "position": "2",
+    "length": "2"
+   },
+   "ignored": {
+    "position": "4",
+    "length": "byteslen"
+   }
+  }
+ },
+ "pong": {
+  "type": "19",
+  "payload": {
+   "byteslen": {
+    "position": "0",
+    "length": "2"
+   },
+   "ignored": {
+    "position": "2",
+    "length": "byteslen"
+   }
+  }
+ },
+ "open_channel": {
+  "type": "32",
+  "payload": {
+   "chain_hash": {
+    "position": "0",
+    "length": "32"
+   },
+   "temporary_channel_id": {
+    "position": "32",
+    "length": "32"
+   },
+   "funding_satoshis": {
+    "position": "64",
+    "length": "8"
+   },
+   "push_msat": {
+    "position": "72",
+    "length": "8"
+   },
+   "dust_limit_satoshis": {
+    "position": "80",
+    "length": "8"
+   },
+   "max_htlc_value_in_flight_msat": {
+    "position": "88",
+    "length": "8"
+   },
+   "channel_reserve_satoshis": {
+    "position": "96",
+    "length": "8"
+   },
+   "htlc_minimum_msat": {
+    "position": "104",
+    "length": "8"
+   },
+   "feerate_per_kw": {
+    "position": "112",
+    "length": "4"
+   },
+   "to_self_delay": {
+    "position": "116",
+    "length": "2"
+   },
+   "max_accepted_htlcs": {
+    "position": "118",
+    "length": "2"
+   },
+   "funding_pubkey": {
+    "position": "120",
+    "length": "33"
+   },
+   "revocation_basepoint": {
+    "position": "153",
+    "length": "33"
+   },
+   "payment_basepoint": {
+    "position": "186",
+    "length": "33"
+   },
+   "delayed_payment_basepoint": {
+    "position": "219",
+    "length": "33"
+   },
+   "htlc_basepoint": {
+    "position": "252",
+    "length": "33"
+   },
+   "first_per_commitment_point": {
+    "position": "285",
+    "length": "33"
+   },
+   "channel_flags": {
+    "position": "318",
+    "length": "1"
+   },
+   "shutdown_len": {
+    "position": "319",
+    "length": "2",
+    "feature": "option_upfront_shutdown_script"
+   },
+   "shutdown_scriptpubkey": {
+    "position": "321",
+    "length": "shutdown_len",
+    "feature": "option_upfront_shutdown_script"
+   }
+  }
+ },
+ "accept_channel": {
+  "type": "33",
+  "payload": {
+   "temporary_channel_id": {
+    "position": "0",
+    "length": "32"
+   },
+   "dust_limit_satoshis": {
+    "position": "32",
+    "length": "8"
+   },
+   "max_htlc_value_in_flight_msat": {
+    "position": "40",
+    "length": "8"
+   },
+   "channel_reserve_satoshis": {
+    "position": "48",
+    "length": "8"
+   },
+   "htlc_minimum_msat": {
+    "position": "56",
+    "length": "8"
+   },
+   "minimum_depth": {
+    "position": "64",
+    "length": "4"
+   },
+   "to_self_delay": {
+    "position": "68",
+    "length": "2"
+   },
+   "max_accepted_htlcs": {
+    "position": "70",
+    "length": "2"
+   },
+   "funding_pubkey": {
+    "position": "72",
+    "length": "33"
+   },
+   "revocation_basepoint": {
+    "position": "105",
+    "length": "33"
+   },
+   "payment_basepoint": {
+    "position": "138",
+    "length": "33"
+   },
+   "delayed_payment_basepoint": {
+    "position": "171",
+    "length": "33"
+   },
+   "htlc_basepoint": {
+    "position": "204",
+    "length": "33"
+   },
+   "first_per_commitment_point": {
+    "position": "237",
+    "length": "33"
+   },
+   "shutdown_len": {
+    "position": "270",
+    "length": "2",
+    "feature": "option_upfront_shutdown_script"
+   },
+   "shutdown_scriptpubkey": {
+    "position": "272",
+    "length": "shutdown_len",
+    "feature": "option_upfront_shutdown_script"
+   }
+  }
+ },
+ "funding_created": {
+  "type": "34",
+  "payload": {
+   "temporary_channel_id": {
+    "position": "0",
+    "length": "32"
+   },
+   "funding_txid": {
+    "position": "32",
+    "length": "32"
+   },
+   "funding_output_index": {
+    "position": "64",
+    "length": "2"
+   },
+   "signature": {
+    "position": "66",
+    "length": "64"
+   }
+  }
+ },
+ "funding_signed": {
+  "type": "35",
+  "payload": {
+   "channel_id": {
+    "position": "0",
+    "length": "32"
+   },
+   "signature": {
+    "position": "32",
+    "length": "64"
+   }
+  }
+ },
+ "funding_locked": {
+  "type": "36",
+  "payload": {
+   "channel_id": {
+    "position": "0",
+    "length": "32"
+   },
+   "next_per_commitment_point": {
+    "position": "32",
+    "length": "33"
+   }
+  }
+ },
+ "shutdown": {
+  "type": "38",
+  "payload": {
+   "channel_id": {
+    "position": "0",
+    "length": "32"
+   },
+   "len": {
+    "position": "32",
+    "length": "2"
+   },
+   "scriptpubkey": {
+    "position": "34",
+    "length": "len"
+   }
+  }
+ },
+ "closing_signed": {
+  "type": "39",
+  "payload": {
+   "channel_id": {
+    "position": "0",
+    "length": "32"
+   },
+   "fee_satoshis": {
+    "position": "32",
+    "length": "8"
+   },
+   "signature": {
+    "position": "40",
+    "length": "64"
+   }
+  }
+ },
+ "update_add_htlc": {
+  "type": "128",
+  "payload": {
+   "channel_id": {
+    "position": "0",
+    "length": "32"
+   },
+   "id": {
+    "position": "32",
+    "length": "8"
+   },
+   "amount_msat": {
+    "position": "40",
+    "length": "8"
+   },
+   "payment_hash": {
+    "position": "48",
+    "length": "32"
+   },
+   "cltv_expiry": {
+    "position": "80",
+    "length": "4"
+   },
+   "onion_routing_packet": {
+    "position": "84",
+    "length": "1366"
+   }
+  }
+ },
+ "update_fulfill_htlc": {
+  "type": "130",
+  "payload": {
+   "channel_id": {
+    "position": "0",
+    "length": "32"
+   },
+   "id": {
+    "position": "32",
+    "length": "8"
+   },
+   "payment_preimage": {
+    "position": "40",
+    "length": "32"
+   }
+  }
+ },
+ "update_fail_htlc": {
+  "type": "131",
+  "payload": {
+   "channel_id": {
+    "position": "0",
+    "length": "32"
+   },
+   "id": {
+    "position": "32",
+    "length": "8"
+   },
+   "len": {
+    "position": "40",
+    "length": "2"
+   },
+   "reason": {
+    "position": "42",
+    "length": "len"
+   }
+  }
+ },
+ "update_fail_malformed_htlc": {
+  "type": "135",
+  "payload": {
+   "channel_id": {
+    "position": "0",
+    "length": "32"
+   },
+   "id": {
+    "position": "32",
+    "length": "8"
+   },
+   "sha256_of_onion": {
+    "position": "40",
+    "length": "32"
+   },
+   "failure_code": {
+    "position": "72",
+    "length": "2"
+   }
+  }
+ },
+ "commitment_signed": {
+  "type": "132",
+  "payload": {
+   "channel_id": {
+    "position": "0",
+    "length": "32"
+   },
+   "signature": {
+    "position": "32",
+    "length": "64"
+   },
+   "num_htlcs": {
+    "position": "96",
+    "length": "2"
+   },
+   "htlc_signature": {
+    "position": "98",
+    "length": "num_htlcs*64"
+   }
+  }
+ },
+ "revoke_and_ack": {
+  "type": "133",
+  "payload": {
+   "channel_id": {
+    "position": "0",
+    "length": "32"
+   },
+   "per_commitment_secret": {
+    "position": "32",
+    "length": "32"
+   },
+   "next_per_commitment_point": {
+    "position": "64",
+    "length": "33"
+   }
+  }
+ },
+ "update_fee": {
+  "type": "134",
+  "payload": {
+   "channel_id": {
+    "position": "0",
+    "length": "32"
+   },
+   "feerate_per_kw": {
+    "position": "32",
+    "length": "4"
+   }
+  }
+ },
+ "channel_reestablish": {
+  "type": "136",
+  "payload": {
+   "channel_id": {
+    "position": "0",
+    "length": "32"
+   },
+   "next_local_commitment_number": {
+    "position": "32",
+    "length": "8"
+   },
+   "next_remote_revocation_number": {
+    "position": "40",
+    "length": "8"
+   },
+   "your_last_per_commitment_secret": {
+    "position": "48",
+    "length": "32",
+    "feature": "option_data_loss_protect"
+   },
+   "my_current_per_commitment_point": {
+    "position": "80",
+    "length": "33",
+    "feature": "option_data_loss_protect"
+   }
+  }
+ },
+ "invalid_realm": {
+  "type": "PERM|1",
+  "payload": {}
+ },
+ "temporary_node_failure": {
+  "type": "NODE|2",
+  "payload": {}
+ },
+ "permanent_node_failure": {
+  "type": "PERM|NODE|2",
+  "payload": {}
+ },
+ "required_node_feature_missing": {
+  "type": "PERM|NODE|3",
+  "payload": {}
+ },
+ "invalid_onion_version": {
+  "type": "BADONION|PERM|4",
+  "payload": {
+   "sha256_of_onion": {
+    "position": "0",
+    "length": "32"
+   }
+  }
+ },
+ "invalid_onion_hmac": {
+  "type": "BADONION|PERM|5",
+  "payload": {
+   "sha256_of_onion": {
+    "position": "0",
+    "length": "32"
+   }
+  }
+ },
+ "invalid_onion_key": {
+  "type": "BADONION|PERM|6",
+  "payload": {
+   "sha256_of_onion": {
+    "position": "0",
+    "length": "32"
+   }
+  }
+ },
+ "temporary_channel_failure": {
+  "type": "UPDATE|7",
+  "payload": {
+   "len": {
+    "position": "0",
+    "length": "2"
+   },
+   "channel_update": {
+    "position": "2",
+    "length": "len"
+   }
+  }
+ },
+ "permanent_channel_failure": {
+  "type": "PERM|8",
+  "payload": {}
+ },
+ "required_channel_feature_missing": {
+  "type": "PERM|9",
+  "payload": {}
+ },
+ "unknown_next_peer": {
+  "type": "PERM|10",
+  "payload": {}
+ },
+ "amount_below_minimum": {
+  "type": "UPDATE|11",
+  "payload": {
+   "htlc_msat": {
+    "position": "0",
+    "length": "8"
+   },
+   "len": {
+    "position": "8",
+    "length": "2"
+   },
+   "channel_update": {
+    "position": "10",
+    "length": "len"
+   }
+  }
+ },
+ "fee_insufficient": {
+  "type": "UPDATE|12",
+  "payload": {
+   "htlc_msat": {
+    "position": "0",
+    "length": "8"
+   },
+   "len": {
+    "position": "8",
+    "length": "2"
+   },
+   "channel_update": {
+    "position": "10",
+    "length": "len"
+   }
+  }
+ },
+ "incorrect_cltv_expiry": {
+  "type": "UPDATE|13",
+  "payload": {
+   "cltv_expiry": {
+    "position": "0",
+    "length": "4"
+   },
+   "len": {
+    "position": "4",
+    "length": "2"
+   },
+   "channel_update": {
+    "position": "6",
+    "length": "len"
+   }
+  }
+ },
+ "expiry_too_soon": {
+  "type": "UPDATE|14",
+  "payload": {
+   "len": {
+    "position": "0",
+    "length": "2"
+   },
+   "channel_update": {
+    "position": "2",
+    "length": "len"
+   }
+  }
+ },
+ "unknown_payment_hash": {
+  "type": "PERM|15",
+  "payload": {}
+ },
+ "incorrect_payment_amount": {
+  "type": "PERM|16",
+  "payload": {}
+ },
+ "final_expiry_too_soon": {
+  "type": "17",
+  "payload": {}
+ },
+ "final_incorrect_cltv_expiry": {
+  "type": "18",
+  "payload": {
+   "cltv_expiry": {
+    "position": "0",
+    "length": "4"
+   }
+  }
+ },
+ "final_incorrect_htlc_amount": {
+  "type": "19",
+  "payload": {
+   "incoming_htlc_amt": {
+    "position": "0",
+    "length": "4"
+   }
+  }
+ },
+ "channel_disabled": {
+  "type": "UPDATE|20",
+  "payload": {}
+ },
+ "expiry_too_far": {
+  "type": "21",
+  "payload": {}
+ },
+ "announcement_signatures": {
+  "type": "259",
+  "payload": {
+   "channel_id": {
+    "position": "0",
+    "length": "32"
+   },
+   "short_channel_id": {
+    "position": "32",
+    "length": "8"
+   },
+   "node_signature": {
+    "position": "40",
+    "length": "64"
+   },
+   "bitcoin_signature": {
+    "position": "104",
+    "length": "64"
+   }
+  }
+ },
+ "channel_announcement": {
+  "type": "256",
+  "payload": {
+   "node_signature_1": {
+    "position": "0",
+    "length": "64"
+   },
+   "node_signature_2": {
+    "position": "64",
+    "length": "64"
+   },
+   "bitcoin_signature_1": {
+    "position": "128",
+    "length": "64"
+   },
+   "bitcoin_signature_2": {
+    "position": "192",
+    "length": "64"
+   },
+   "len": {
+    "position": "256",
+    "length": "2"
+   },
+   "features": {
+    "position": "258",
+    "length": "len"
+   },
+   "chain_hash": {
+    "position": "258+len",
+    "length": "32"
+   },
+   "short_channel_id": {
+    "position": "290+len",
+    "length": "8"
+   },
+   "node_id_1": {
+    "position": "298+len",
+    "length": "33"
+   },
+   "node_id_2": {
+    "position": "331+len",
+    "length": "33"
+   },
+   "bitcoin_key_1": {
+    "position": "364+len",
+    "length": "33"
+   },
+   "bitcoin_key_2": {
+    "position": "397+len",
+    "length": "33"
+   }
+  }
+ },
+ "node_announcement": {
+  "type": "257",
+  "payload": {
+   "signature": {
+    "position": "0",
+    "length": "64"
+   },
+   "flen": {
+    "position": "64",
+    "length": "2"
+   },
+   "features": {
+    "position": "66",
+    "length": "flen"
+   },
+   "timestamp": {
+    "position": "66+flen",
+    "length": "4"
+   },
+   "node_id": {
+    "position": "70+flen",
+    "length": "33"
+   },
+   "rgb_color": {
+    "position": "103+flen",
+    "length": "3"
+   },
+   "alias": {
+    "position": "106+flen",
+    "length": "32"
+   },
+   "addrlen": {
+    "position": "138+flen",
+    "length": "2"
+   },
+   "addresses": {
+    "position": "140+flen",
+    "length": "addrlen"
+   }
+  }
+ },
+ "channel_update": {
+  "type": "258",
+  "payload": {
+   "signature": {
+    "position": "0",
+    "length": "64"
+   },
+   "chain_hash": {
+    "position": "64",
+    "length": "32"
+   },
+   "short_channel_id": {
+    "position": "96",
+    "length": "8"
+   },
+   "timestamp": {
+    "position": "104",
+    "length": "4"
+   },
+   "flags": {
+    "position": "108",
+    "length": "2"
+   },
+   "cltv_expiry_delta": {
+    "position": "110",
+    "length": "2"
+   },
+   "htlc_minimum_msat": {
+    "position": "112",
+    "length": "8"
+   },
+   "fee_base_msat": {
+    "position": "120",
+    "length": "4"
+   },
+   "fee_proportional_millionths": {
+    "position": "124",
+    "length": "4"
+   }
+  }
+ }
+}

lib/lnaddr.py

@@ -0,0 +1,391 @@
+#! /usr/bin/env python3
+# This was forked from https://github.com/rustyrussell/lightning-payencode/tree/acc16ec13a3fa1dc16c07af6ec67c261bd8aff23
+
+from .bitcoin import hash160_to_b58_address, b58_address_to_hash160
+from hashlib import sha256
+from .segwit_addr import bech32_encode, bech32_decode, CHARSET
+from binascii import hexlify
+from decimal import Decimal
+from . import constants
+
+from . import ecc
+import bitstring
+import re
+import time
+
+
+# BOLT #11:
+#
+# A writer MUST encode `amount` as a positive decimal integer with no
+# leading zeroes, SHOULD use the shortest representation possible.
+def shorten_amount(amount):
+    """ Given an amount in bitcoin, shorten it
+    """
+    # Convert to pico initially
+    amount = int(amount * 10**12)
+    units = ['p', 'n', 'u', 'm', '']
+    for unit in units:
+        if amount % 1000 == 0:
+            amount //= 1000
+        else:
+            break
+    return str(amount) + unit
+
+def unshorten_amount(amount):
+    """ Given a shortened amount, convert it into a decimal
+    """
+    # BOLT #11:
+    # The following `multiplier` letters are defined:
+    #
+    #* `m` (milli): multiply by 0.001
+    #* `u` (micro): multiply by 0.000001
+    #* `n` (nano): multiply by 0.000000001
+    #* `p` (pico): multiply by 0.000000000001
+    units = {
+        'p': 10**12,
+        'n': 10**9,
+        'u': 10**6,
+        'm': 10**3,
+    }
+    unit = str(amount)[-1]
+    # BOLT #11:
+    # A reader SHOULD fail if `amount` contains a non-digit, or is followed by
+    # anything except a `multiplier` in the table above.
+    if not re.fullmatch("\d+[pnum]?", str(amount)):
+        raise ValueError("Invalid amount '{}'".format(amount))
+
+    if unit in units.keys():
+        return Decimal(amount[:-1]) / units[unit]
+    else:
+        return Decimal(amount)
+
+# Bech32 spits out array of 5-bit values.  Shim here.
+def u5_to_bitarray(arr):
+    ret = bitstring.BitArray()
+    for a in arr:
+        ret += bitstring.pack("uint:5", a)
+    return ret
+
+def bitarray_to_u5(barr):
+    assert barr.len % 5 == 0
+    ret = []
+    s = bitstring.ConstBitStream(barr)
+    while s.pos != s.len:
+        ret.append(s.read(5).uint)
+    return ret
+
+def encode_fallback(fallback, currency):
+    """ Encode all supported fallback addresses.
+    """
+    if currency == 'bc' or currency == 'tb':
+        fbhrp, witness = bech32_decode(fallback, ignore_long_length=True)
+        if fbhrp:
+            if fbhrp != currency:
+                raise ValueError("Not a bech32 address for this currency")
+            wver = witness[0]
+            if wver > 16:
+                raise ValueError("Invalid witness version {}".format(witness[0]))
+            wprog = u5_to_bitarray(witness[1:])
+        else:
+            addrtype, addr = b58_address_to_hash160(fallback)
+            if is_p2pkh(currency, addrtype):
+                wver = 17
+            elif is_p2sh(currency, addrtype):
+                wver = 18
+            else:
+                raise ValueError("Unknown address type for {}".format(currency))
+            wprog = addr
+        return tagged('f', bitstring.pack("uint:5", wver) + wprog)
+    else:
+        raise NotImplementedError("Support for currency {} not implemented".format(currency))
+
+def parse_fallback(fallback, currency):
+    if currency == 'bc' or currency == 'tb':
+        wver = fallback[0:5].uint
+        if wver == 17:
+            addr=hash160_to_b58_address(fallback[5:].tobytes(), base58_prefix_map[currency][0])
+        elif wver == 18:
+            addr=hash160_to_b58_address(fallback[5:].tobytes(), base58_prefix_map[currency][1])
+        elif wver <= 16:
+            addr=bech32_encode(currency, bitarray_to_u5(fallback))
+        else:
+            return None
+    else:
+        addr=fallback.tobytes()
+    return addr
+
+
+# Map of classical and witness address prefixes
+base58_prefix_map = {
+    'bc' : (0, 5),
+    'tb' : (111, 196)
+}
+
+def is_p2pkh(currency, prefix):
+    return prefix == base58_prefix_map[currency][0]
+
+def is_p2sh(currency, prefix):
+    return prefix == base58_prefix_map[currency][1]
+
+# Tagged field containing BitArray
+def tagged(char, l):
+    # Tagged fields need to be zero-padded to 5 bits.
+    while l.len % 5 != 0:
+        l.append('0b0')
+    return bitstring.pack("uint:5, uint:5, uint:5",
+                          CHARSET.find(char),
+                          (l.len / 5) / 32, (l.len / 5) % 32) + l
+
+# Tagged field containing bytes
+def tagged_bytes(char, l):
+    return tagged(char, bitstring.BitArray(l))
+
+# Discard trailing bits, convert to bytes.
+def trim_to_bytes(barr):
+    # Adds a byte if necessary.
+    b = barr.tobytes()
+    if barr.len % 8 != 0:
+        return b[:-1]
+    return b
+
+# Try to pull out tagged data: returns tag, tagged data and remainder.
+def pull_tagged(stream):
+    tag = stream.read(5).uint
+    length = stream.read(5).uint * 32 + stream.read(5).uint
+    return (CHARSET[tag], stream.read(length * 5), stream)
+
+def lnencode(addr, privkey):
+    if addr.amount:
+        amount = Decimal(str(addr.amount))
+        # We can only send down to millisatoshi.
+        if amount * 10**12 % 10:
+            raise ValueError("Cannot encode {}: too many decimal places".format(
+                addr.amount))
+
+        amount = addr.currency + shorten_amount(amount)
+    else:
+        amount = addr.currency if addr.currency else ''
+
+    hrp = 'ln' + amount
+
+    # Start with the timestamp
+    data = bitstring.pack('uint:35', addr.date)
+
+    # Payment hash
+    data += tagged_bytes('p', addr.paymenthash)
+    tags_set = set()
+
+    for k, v in addr.tags:
+
+        # BOLT #11:
+        #
+        # A writer MUST NOT include more than one `d`, `h`, `n` or `x` fields,
+        if k in ('d', 'h', 'n', 'x'):
+            if k in tags_set:
+                raise ValueError("Duplicate '{}' tag".format(k))
+
+        if k == 'r':
+            route = bitstring.BitArray()
+            for step in v:
+                pubkey, channel, feebase, feerate, cltv = step
+                route.append(bitstring.BitArray(pubkey) + bitstring.BitArray(channel) + bitstring.pack('intbe:32', feebase) + bitstring.pack('intbe:32', feerate) + bitstring.pack('intbe:16', cltv))
+            data += tagged('r', route)
+        elif k == 'f':
+            data += encode_fallback(v, addr.currency)
+        elif k == 'd':
+            data += tagged_bytes('d', v.encode())
+        elif k == 'x':
+            # Get minimal length by trimming leading 5 bits at a time.
+            expirybits = bitstring.pack('intbe:64', v)[4:64]
+            while expirybits.startswith('0b00000'):
+                expirybits = expirybits[5:]
+            data += tagged('x', expirybits)
+        elif k == 'h':
+            data += tagged_bytes('h', sha256(v.encode('utf-8')).digest())
+        elif k == 'n':
+            data += tagged_bytes('n', v)
+        else:
+            # FIXME: Support unknown tags?
+            raise ValueError("Unknown tag {}".format(k))
+
+        tags_set.add(k)
+
+    # BOLT #11:
+    #
+    # A writer MUST include either a `d` or `h` field, and MUST NOT include
+    # both.
+    if 'd' in tags_set and 'h' in tags_set:
+        raise ValueError("Cannot include both 'd' and 'h'")
+    if not 'd' in tags_set and not 'h' in tags_set:
+        raise ValueError("Must include either 'd' or 'h'")
+
+    # We actually sign the hrp, then data (padded to 8 bits with zeroes).
+    msg = hrp.encode("ascii") + data.tobytes()
+    privkey = ecc.ECPrivkey(privkey)
+    sig = privkey.sign_message(msg, is_compressed=False, algo=lambda x:sha256(x).digest())
+    recovery_flag = bytes([sig[0] - 27])
+    sig = bytes(sig[1:]) + recovery_flag
+    data += sig
+
+    return bech32_encode(hrp, bitarray_to_u5(data))
+
+class LnAddr(object):
+    def __init__(self, paymenthash=None, amount=None, currency=None, tags=None, date=None):
+        self.date = int(time.time()) if not date else int(date)
+        self.tags = [] if not tags else tags
+        self.unknown_tags = []
+        self.paymenthash=paymenthash
+        self.signature = None
+        self.pubkey = None
+        self.currency = constants.net.SEGWIT_HRP if currency is None else currency
+        self.amount = amount
+        self.min_final_cltv_expiry = 9
+
+    def __str__(self):
+        return "LnAddr[{}, amount={}{} tags=[{}]]".format(
+            hexlify(self.pubkey.serialize()).decode('utf-8'),
+            self.amount, self.currency,
+            ", ".join([k + '=' + str(v) for k, v in self.tags])
+        )
+
+def lndecode(a, verbose=False, expected_hrp=constants.net.SEGWIT_HRP):
+    hrp, data = bech32_decode(a, ignore_long_length=True)
+    if not hrp:
+        raise ValueError("Bad bech32 checksum")
+
+    # BOLT #11:
+    #
+    # A reader MUST fail if it does not understand the `prefix`.
+    if not hrp.startswith('ln'):
+        raise ValueError("Does not start with ln")
+
+    if not hrp[2:].startswith(expected_hrp):
+        raise ValueError("Wrong Lightning invoice HRP " + hrp[2:] + ", should be " + expected_hrp)
+
+    data = u5_to_bitarray(data)
+
+    # Final signature 65 bytes, split it off.
+    if len(data) < 65*8:
+        raise ValueError("Too short to contain signature")
+    sigdecoded = data[-65*8:].tobytes()
+    data = bitstring.ConstBitStream(data[:-65*8])
+
+    addr = LnAddr()
+    addr.pubkey = None
+
+    m = re.search("[^\d]+", hrp[2:])
+    if m:
+        addr.currency = m.group(0)
+        amountstr = hrp[2+m.end():]
+        # BOLT #11:
+        #
+        # A reader SHOULD indicate if amount is unspecified, otherwise it MUST
+        # multiply `amount` by the `multiplier` value (if any) to derive the
+        # amount required for payment.
+        if amountstr != '':
+            addr.amount = unshorten_amount(amountstr)
+
+    addr.date = data.read(35).uint
+
+    while data.pos != data.len:
+        tag, tagdata, data = pull_tagged(data)
+
+        # BOLT #11:
+        #
+        # A reader MUST skip over unknown fields, an `f` field with unknown
+        # `version`, or a `p`, `h`, or `n` field which does not have
+        # `data_length` 52, 52, or 53 respectively.
+        data_length = len(tagdata) / 5
+
+        if tag == 'r':
+            # BOLT #11:
+            #
+            # * `r` (3): `data_length` variable.  One or more entries
+            # containing extra routing information for a private route;
+            # there may be more than one `r` field, too.
+            #    * `pubkey` (264 bits)
+            #    * `short_channel_id` (64 bits)
+            #    * `feebase` (32 bits, big-endian)
+            #    * `feerate` (32 bits, big-endian)
+            #    * `cltv_expiry_delta` (16 bits, big-endian)
+            route=[]
+            s = bitstring.ConstBitStream(tagdata)
+            while s.pos + 264 + 64 + 32 + 32 + 16 < s.len:
+                route.append((s.read(264).tobytes(),
+                              s.read(64).tobytes(),
+                              s.read(32).intbe,
+                              s.read(32).intbe,
+                              s.read(16).intbe))
+            addr.tags.append(('r',route))
+        elif tag == 'f':
+            fallback = parse_fallback(tagdata, addr.currency)
+            if fallback:
+                addr.tags.append(('f', fallback))
+            else:
+                # Incorrect version.
+                addr.unknown_tags.append((tag, tagdata))
+                continue
+
+        elif tag == 'd':
+            addr.tags.append(('d', trim_to_bytes(tagdata).decode('utf-8')))
+
+        elif tag == 'h':
+            if data_length != 52:
+                addr.unknown_tags.append((tag, tagdata))
+                continue
+            addr.tags.append(('h', trim_to_bytes(tagdata)))
+
+        elif tag == 'x':
+            addr.tags.append(('x', tagdata.uint))
+
+        elif tag == 'p':
+            if data_length != 52:
+                addr.unknown_tags.append((tag, tagdata))
+                continue
+            addr.paymenthash = trim_to_bytes(tagdata)
+
+        elif tag == 'n':
+            if data_length != 53:
+                addr.unknown_tags.append((tag, tagdata))
+                continue
+            pubkeybytes = trim_to_bytes(tagdata)
+            addr.pubkey = pubkeybytes
+        elif tag == 'c':
+            addr.min_final_cltv_expiry = tagdata.int
+        else:
+            addr.unknown_tags.append((tag, tagdata))
+
+    if verbose:
+        print('hex of signature data (32 byte r, 32 byte s): {}'
+              .format(hexlify(sigdecoded[0:64])))
+        print('recovery flag: {}'.format(sigdecoded[64]))
+        print('hex of data for signing: {}'
+              .format(hexlify(hrp.encode("ascii") + data.tobytes())))
+        print('SHA256 of above: {}'.format(sha256(hrp.encode("ascii") + data.tobytes()).hexdigest()))
+
+    # BOLT #11:
+    #
+    # A reader MUST check that the `signature` is valid (see the `n` tagged
+    # field specified below).
+    addr.signature = sigdecoded[:65]
+    hrp_hash = sha256(hrp.encode("ascii") + data.tobytes()).digest()
+    if addr.pubkey: # Specified by `n`
+        # BOLT #11:
+        #
+        # A reader MUST use the `n` field to validate the signature instead of
+        # performing signature recovery if a valid `n` field is provided.
+        ecc.ECPubkey(addr.pubkey).verify_message_hash(sigdecoded[:64], hrp_hash)
+        pubkey_copy = addr.pubkey
+        class WrappedBytesKey:
+            serialize = lambda: pubkey_copy
+        addr.pubkey = WrappedBytesKey
+    else: # Recover pubkey from signature.
+        addr.pubkey = SerializableKey(ecc.ECPubkey.from_sig_string(sigdecoded[:64], sigdecoded[64], hrp_hash))
+
+    return addr
+
+class SerializableKey:
+    def __init__(self, pubkey):
+        self.pubkey = pubkey
+    def serialize(self):
+        return self.pubkey.get_public_key_bytes(True)

lib/lnhtlc.py

@@ -0,0 +1,545 @@
+# ported from lnd 42de4400bff5105352d0552155f73589166d162b
+from collections import namedtuple
+import binascii
+import json
+from .util import bfh, PrintError
+from .bitcoin import Hash
+from .crypto import sha256
+from . import ecc
+from .lnutil import Outpoint, ChannelConfig, LocalState, RemoteState, Keypair, OnlyPubkeyKeypair, ChannelConstraints, RevocationStore
+from .lnutil import get_per_commitment_secret_from_seed
+from .lnutil import secret_to_pubkey, derive_privkey, derive_pubkey, derive_blinded_pubkey
+from .lnutil import sign_and_get_sig_string
+from .lnutil import make_htlc_tx_with_open_channel, make_commitment, make_received_htlc, make_offered_htlc
+from .lnutil import HTLC_TIMEOUT_WEIGHT, HTLC_SUCCESS_WEIGHT
+
+SettleHtlc = namedtuple("SettleHtlc", ["htlc_id"])
+RevokeAndAck = namedtuple("RevokeAndAck", ["per_commitment_secret", "next_per_commitment_point"])
+
+class UpdateAddHtlc:
+    def __init__(self, amount_msat, payment_hash, cltv_expiry, total_fee):
+        self.amount_msat = amount_msat
+        self.payment_hash = payment_hash
+        self.cltv_expiry = cltv_expiry
+        self.total_fee = total_fee
+
+        # the height the htlc was locked in at, or None
+        self.r_locked_in = None
+        self.l_locked_in = None
+
+        self.htlc_id = None
+
+    def as_tuple(self):
+        return (self.htlc_id, self.amount_msat, self.payment_hash, self.cltv_expiry, self.r_locked_in, self.l_locked_in, self.total_fee)
+
+    def __hash__(self):
+        return hash(self.as_tuple())
+
+    def __eq__(self, o):
+        return type(o) is UpdateAddHtlc and self.as_tuple() == o.as_tuple()
+
+    def __repr__(self):
+        return "UpdateAddHtlc" + str(self.as_tuple())
+
+is_key = lambda k: k.endswith("_basepoint") or k.endswith("_key")
+
+def maybeDecode(k, v):
+    assert type(v) is not list
+    if k in ["node_id", "channel_id", "short_channel_id", "pubkey", "privkey", "current_per_commitment_point", "next_per_commitment_point", "per_commitment_secret_seed", "current_commitment_signature"] and v is not None:
+        return binascii.unhexlify(v)
+    return v
+
+def decodeAll(v):
+    return {i: maybeDecode(i, j) for i, j in v.items()} if isinstance(v, dict) else v
+
+def typeWrap(k, v, local):
+    if is_key(k):
+        if local:
+            return Keypair(**v)
+        else:
+            return OnlyPubkeyKeypair(**v)
+    return v
+
+class HTLCStateMachine(PrintError):
+    def lookup_htlc(self, log, htlc_id):
+        assert type(htlc_id) is int
+        for htlc in log:
+            if type(htlc) is not UpdateAddHtlc: continue
+            if htlc.htlc_id == htlc_id:
+                return htlc
+        assert False, self.diagnostic_name() + ": htlc_id {} not found in {}".format(htlc_id, log)
+
+    def diagnostic_name(self):
+        return str(self.name)
+
+    def __init__(self, state, name = None):
+        self.local_config = state["local_config"]
+        if type(self.local_config) is not ChannelConfig:
+            new_local_config = {k: typeWrap(k, decodeAll(v), True) for k, v in self.local_config.items()}
+            self.local_config = ChannelConfig(**new_local_config)
+
+        self.remote_config = state["remote_config"]
+        if type(self.remote_config) is not ChannelConfig:
+            new_remote_config = {k: typeWrap(k, decodeAll(v), False) for k, v in self.remote_config.items()}
+            self.remote_config = ChannelConfig(**new_remote_config)
+
+        self.local_state = state["local_state"]
+        if type(self.local_state) is not LocalState:
+            self.local_state = LocalState(**decodeAll(self.local_state))
+
+        self.remote_state = state["remote_state"]
+        if type(self.remote_state) is not RemoteState:
+            self.remote_state = RemoteState(**decodeAll(self.remote_state))
+
+        if type(self.remote_state.revocation_store) is not RevocationStore:
+            self.remote_state = self.remote_state._replace(revocation_store = RevocationStore.from_json_obj(self.remote_state.revocation_store))
+
+        self.channel_id = maybeDecode("channel_id", state["channel_id"]) if type(state["channel_id"]) is not bytes else state["channel_id"]
+        self.constraints = ChannelConstraints(**decodeAll(state["constraints"])) if type(state["constraints"]) is not ChannelConstraints else state["constraints"]
+        self.funding_outpoint = Outpoint(**decodeAll(state["funding_outpoint"])) if type(state["funding_outpoint"]) is not Outpoint else state["funding_outpoint"]
+        self.node_id = maybeDecode("node_id", state["node_id"]) if type(state["node_id"]) is not bytes else state["node_id"]
+        self.short_channel_id = maybeDecode("short_channel_id", state["short_channel_id"]) if type(state["short_channel_id"]) is not bytes else state["short_channel_id"]
+
+        self.local_update_log = []
+        self.remote_update_log = []
+
+        self.name = name
+
+        self.total_msat_sent = 0
+        self.total_msat_received = 0
+        self.pending_feerate = None
+
+    def add_htlc(self, htlc):
+        """
+        AddHTLC adds an HTLC to the state machine's local update log. This method
+        should be called when preparing to send an outgoing HTLC.
+        """
+        assert type(htlc) is UpdateAddHtlc
+        self.local_update_log.append(htlc)
+        self.print_error("add_htlc")
+        htlc_id = self.local_state.next_htlc_id
+        self.local_state=self.local_state._replace(next_htlc_id=htlc_id + 1)
+        htlc.htlc_id = htlc_id
+        return htlc_id
+
+    def receive_htlc(self, htlc):
+        """
+        ReceiveHTLC adds an HTLC to the state machine's remote update log. This
+        method should be called in response to receiving a new HTLC from the remote
+        party.
+        """
+        self.print_error("receive_htlc")
+        assert type(htlc) is UpdateAddHtlc
+        self.remote_update_log.append(htlc)
+        htlc_id = self.remote_state.next_htlc_id
+        self.remote_state=self.remote_state._replace(next_htlc_id=htlc_id + 1)
+        htlc.htlc_id = htlc_id
+        return htlc_id
+
+    def sign_next_commitment(self):
+        """
+        SignNextCommitment signs a new commitment which includes any previous
+        unsettled HTLCs, any new HTLCs, and any modifications to prior HTLCs
+        committed in previous commitment updates. Signing a new commitment
+        decrements the available revocation window by 1. After a successful method
+        call, the remote party's commitment chain is extended by a new commitment
+        which includes all updates to the HTLC log prior to this method invocation.
+        The first return parameter is the signature for the commitment transaction
+        itself, while the second parameter is a slice of all HTLC signatures (if
+        any). The HTLC signatures are sorted according to the BIP 69 order of the
+        HTLC's on the commitment transaction.
+        """
+        for htlc in self.local_update_log:
+            if not type(htlc) is UpdateAddHtlc: continue
+            if htlc.l_locked_in is None: htlc.l_locked_in = self.local_state.ctn
+        self.print_error("sign_next_commitment")
+
+        sig_64 = sign_and_get_sig_string(self.remote_commitment, self.local_config, self.remote_config)
+
+        their_remote_htlc_privkey_number = derive_privkey(
+            int.from_bytes(self.local_config.htlc_basepoint.privkey, 'big'),
+            self.remote_state.next_per_commitment_point)
+        their_remote_htlc_privkey = their_remote_htlc_privkey_number.to_bytes(32, 'big')
+
+        for_us = False
+
+        htlcsigs = []
+        for we_receive, htlcs in zip([True, False], [self.htlcs_in_remote, self.htlcs_in_local]):
+            assert len(htlcs) <= 1
+            for htlc in htlcs:
+                weight = HTLC_SUCCESS_WEIGHT if we_receive else HTLC_TIMEOUT_WEIGHT
+                if htlc.amount_msat // 1000 - weight * (self.constraints.feerate // 1000) < self.remote_config.dust_limit_sat:
+                    continue
+                original_htlc_output_index = 0
+                args = [self.remote_state.next_per_commitment_point, for_us, we_receive, htlc.amount_msat + htlc.total_fee, htlc.cltv_expiry, htlc.payment_hash, self.remote_commitment, original_htlc_output_index]
+                htlc_tx = make_htlc_tx_with_open_channel(self, *args)
+                sig = bfh(htlc_tx.sign_txin(0, their_remote_htlc_privkey))
+                htlc_sig = ecc.sig_string_from_der_sig(sig[:-1])
+                htlcsigs.append(htlc_sig)
+
+        return sig_64, htlcsigs
+
+    def receive_new_commitment(self, sig, htlc_sigs):
+        """
+        ReceiveNewCommitment process a signature for a new commitment state sent by
+        the remote party. This method should be called in response to the
+        remote party initiating a new change, or when the remote party sends a
+        signature fully accepting a new state we've initiated. If we are able to
+        successfully validate the signature, then the generated commitment is added
+        to our local commitment chain. Once we send a revocation for our prior
+        state, then this newly added commitment becomes our current accepted channel
+        state.
+        """
+
+        self.print_error("receive_new_commitment")
+        for htlc in self.remote_update_log:
+            if not type(htlc) is UpdateAddHtlc: continue
+            if htlc.r_locked_in is None: htlc.r_locked_in = self.remote_state.ctn
+        assert len(htlc_sigs) == 0 or type(htlc_sigs[0]) is bytes
+
+        preimage_hex = self.local_commitment.serialize_preimage(0)
+        pre_hash = Hash(bfh(preimage_hex))
+        if not ecc.verify_signature(self.remote_config.multisig_key.pubkey, sig, pre_hash):
+            raise Exception('failed verifying signature of our updated commitment transaction: ' + str(sig))
+
+        _, this_point, _ = self.points
+
+        if len(self.htlcs_in_remote) > 0 and len(self.local_commitment.outputs()) == 3:
+            print("CHECKING HTLC SIGS")
+            we_receive = True
+            payment_hash = self.htlcs_in_remote[0].payment_hash
+            amount_msat = self.htlcs_in_remote[0].amount_msat
+            cltv_expiry = self.htlcs_in_remote[0].cltv_expiry
+            htlc_tx = make_htlc_tx_with_open_channel(self, this_point, True, we_receive, amount_msat, cltv_expiry, payment_hash, self.local_commitment, 0)
+            pre_hash = Hash(bfh(htlc_tx.serialize_preimage(0)))
+            remote_htlc_pubkey = derive_pubkey(self.remote_config.htlc_basepoint.pubkey, this_point)
+            if not ecc.verify_signature(remote_htlc_pubkey, htlc_sigs[0], pre_hash):
+                raise Exception("failed verifying signature an HTLC tx spending from one of our commit tx'es HTLC outputs")
+
+        # TODO check htlc in htlcs_in_local
+
+    def revoke_current_commitment(self):
+        """
+        RevokeCurrentCommitment revokes the next lowest unrevoked commitment
+        transaction in the local commitment chain. As a result the edge of our
+        revocation window is extended by one, and the tail of our local commitment
+        chain is advanced by a single commitment. This now lowest unrevoked
+        commitment becomes our currently accepted state within the channel. This
+        method also returns the set of HTLC's currently active within the commitment
+        transaction. This return value allows callers to act once an HTLC has been
+        locked into our commitment transaction.
+        """
+        self.print_error("revoke_current_commitment")
+
+        last_secret, this_point, next_point = self.points
+
+        if self.pending_feerate is not None:
+            new_feerate = self.pending_feerate
+        else:
+            new_feerate = self.constraints.feerate
+
+        self.local_state=self.local_state._replace(
+            ctn=self.local_state.ctn + 1
+        )
+        self.constraints=self.constraints._replace(
+            feerate=new_feerate
+        )
+
+        return RevokeAndAck(last_secret, next_point), "current htlcs"
+
+    @property
+    def points(self):
+        last_small_num = self.local_state.ctn
+        next_small_num = last_small_num + 2
+        this_small_num = last_small_num + 1
+        last_secret = get_per_commitment_secret_from_seed(self.local_state.per_commitment_secret_seed, 2**48-last_small_num-1)
+        this_secret = get_per_commitment_secret_from_seed(self.local_state.per_commitment_secret_seed, 2**48-this_small_num-1)
+        this_point = secret_to_pubkey(int.from_bytes(this_secret, 'big'))
+        next_secret = get_per_commitment_secret_from_seed(self.local_state.per_commitment_secret_seed, 2**48-next_small_num-1)
+        next_point = secret_to_pubkey(int.from_bytes(next_secret, 'big'))
+        return last_secret, this_point, next_point
+
+    def receive_revocation(self, revocation):
+        """
+        ReceiveRevocation processes a revocation sent by the remote party for the
+        lowest unrevoked commitment within their commitment chain. We receive a
+        revocation either during the initial session negotiation wherein revocation
+        windows are extended, or in response to a state update that we initiate. If
+        successful, then the remote commitment chain is advanced by a single
+        commitment, and a log compaction is attempted.
+
+        Returns the forwarding package corresponding to the remote commitment height
+        that was revoked.
+        """
+        self.print_error("receive_revocation")
+
+        settle_fails2 = []
+        for x in self.remote_update_log:
+            if type(x) is not SettleHtlc:
+                continue
+            settle_fails2.append(x)
+
+        sent_this_batch, sent_fees = 0, 0
+
+        for x in settle_fails2:
+            htlc = self.lookup_htlc(self.local_update_log, x.htlc_id)
+            sent_this_batch += htlc.amount_msat
+            sent_fees += htlc.total_fee
+
+        self.total_msat_sent += sent_this_batch
+
+        # log compaction (remove entries relating to htlc's that have been settled)
+
+        to_remove = []
+        for x in filter(lambda x: type(x) is SettleHtlc, self.remote_update_log):
+            to_remove += [y for y in self.local_update_log if y.htlc_id == x.htlc_id]
+
+        # assert that we should have compacted the log earlier
+        assert len(to_remove) <= 1, to_remove
+        if len(to_remove) == 1:
+            self.remote_update_log = [x for x in self.remote_update_log if x.htlc_id != to_remove[0].htlc_id]
+            self.local_update_log = [x for x in self.local_update_log if x.htlc_id != to_remove[0].htlc_id]
+
+        to_remove = []
+        for x in filter(lambda x: type(x) is SettleHtlc, self.local_update_log):
+            to_remove += [y for y in self.remote_update_log if y.htlc_id == x.htlc_id]
+        if len(to_remove) == 1:
+            self.remote_update_log = [x for x in self.remote_update_log if x.htlc_id != to_remove[0].htlc_id]
+            self.local_update_log = [x for x in self.local_update_log if x.htlc_id != to_remove[0].htlc_id]
+        received_this_batch = sum(x.amount_msat for x in to_remove)
+
+        self.total_msat_received += received_this_batch
+
+        received_fees = sum(x.total_fee for x in to_remove)
+
+        self.remote_state.revocation_store.add_next_entry(revocation.per_commitment_secret)
+
+        next_point = self.remote_state.next_per_commitment_point
+
+        print("RECEIVED", received_this_batch)
+        print("SENT", sent_this_batch)
+        self.remote_state=self.remote_state._replace(
+            ctn=self.remote_state.ctn + 1,
+            current_per_commitment_point=next_point,
+            next_per_commitment_point=revocation.next_per_commitment_point,
+            amount_msat=self.remote_state.amount_msat + (sent_this_batch - received_this_batch) + sent_fees - received_fees
+        )
+        self.local_state=self.local_state._replace(
+            amount_msat = self.local_state.amount_msat + (received_this_batch - sent_this_batch) - sent_fees + received_fees
+        )
+
+    @staticmethod
+    def htlcsum(htlcs):
+        amount_unsettled = 0
+        fee = 0
+        for x in htlcs:
+            amount_unsettled += x.amount_msat
+            fee += x.total_fee
+        return amount_unsettled, fee
+
+    def amounts(self):
+        remote_settled_value, remote_settled_fee = self.htlcsum(self.gen_htlc_indices("remote", False))
+        local_settled_value, local_settled_fee = self.htlcsum(self.gen_htlc_indices("local", False))
+        htlc_value_local, total_fee_local = self.htlcsum(self.htlcs_in_local)
+        htlc_value_remote, total_fee_remote = self.htlcsum(self.htlcs_in_remote)
+        total_fee_local += local_settled_fee
+        total_fee_remote += remote_settled_fee
+        local_msat = self.local_state.amount_msat -\
+          htlc_value_local + remote_settled_value - local_settled_value
+        remote_msat = self.remote_state.amount_msat -\
+          htlc_value_remote + local_settled_value - remote_settled_value
+        return remote_msat, total_fee_remote, local_msat, total_fee_local
+
+    @property
+    def remote_commitment(self):
+        remote_msat, total_fee_remote, local_msat, total_fee_local = self.amounts()
+        assert local_msat >= 0
+        assert remote_msat >= 0
+
+        this_point = self.remote_state.next_per_commitment_point
+
+        remote_htlc_pubkey = derive_pubkey(self.remote_config.htlc_basepoint.pubkey, this_point)
+        local_htlc_pubkey = derive_pubkey(self.local_config.htlc_basepoint.pubkey, this_point)
+        local_revocation_pubkey = derive_blinded_pubkey(self.local_config.revocation_basepoint.pubkey, this_point)
+
+        trimmed = 0
+
+        htlcs_in_local = []
+        for htlc in self.htlcs_in_local:
+            if htlc.amount_msat // 1000 - HTLC_SUCCESS_WEIGHT * (self.constraints.feerate // 1000) < self.remote_config.dust_limit_sat:
+                trimmed += htlc.amount_msat // 1000
+                continue
+            htlcs_in_local.append(
+                ( make_received_htlc(local_revocation_pubkey, local_htlc_pubkey, remote_htlc_pubkey, htlc.payment_hash, htlc.cltv_expiry), htlc.amount_msat + htlc.total_fee))
+
+        htlcs_in_remote = []
+        for htlc in self.htlcs_in_remote:
+            if htlc.amount_msat // 1000 - HTLC_TIMEOUT_WEIGHT * (self.constraints.feerate // 1000) < self.remote_config.dust_limit_sat:
+                trimmed += htlc.amount_msat // 1000
+                continue
+            htlcs_in_remote.append(
+                ( make_offered_htlc(local_revocation_pubkey, local_htlc_pubkey, remote_htlc_pubkey, htlc.payment_hash), htlc.amount_msat + htlc.total_fee))
+
+        commit = self.make_commitment(self.remote_state.ctn + 1,
+            False, this_point,
+            remote_msat - total_fee_remote, local_msat - total_fee_local, htlcs_in_local + htlcs_in_remote, trimmed)
+        return commit
+
+    @property
+    def local_commitment(self):
+        remote_msat, total_fee_remote, local_msat, total_fee_local = self.amounts()
+        assert local_msat >= 0
+        assert remote_msat >= 0
+
+        _, this_point, _ = self.points
+
+        remote_htlc_pubkey = derive_pubkey(self.remote_config.htlc_basepoint.pubkey, this_point)
+        local_htlc_pubkey = derive_pubkey(self.local_config.htlc_basepoint.pubkey, this_point)
+        remote_revocation_pubkey = derive_blinded_pubkey(self.remote_config.revocation_basepoint.pubkey, this_point)
+
+        trimmed = 0
+
+        htlcs_in_local = []
+        for htlc in self.htlcs_in_local:
+            if htlc.amount_msat // 1000 - HTLC_TIMEOUT_WEIGHT * (self.constraints.feerate // 1000) < self.local_config.dust_limit_sat:
+                trimmed += htlc.amount_msat // 1000
+                continue
+            htlcs_in_local.append(
+                ( make_offered_htlc(remote_revocation_pubkey, remote_htlc_pubkey, local_htlc_pubkey, htlc.payment_hash), htlc.amount_msat + htlc.total_fee))
+
+        htlcs_in_remote = []
+        for htlc in self.htlcs_in_remote:
+            if htlc.amount_msat // 1000 - HTLC_SUCCESS_WEIGHT * (self.constraints.feerate // 1000) < self.local_config.dust_limit_sat:
+                trimmed += htlc.amount_msat // 1000
+                continue
+            htlcs_in_remote.append(
+                ( make_received_htlc(remote_revocation_pubkey, remote_htlc_pubkey, local_htlc_pubkey, htlc.payment_hash, htlc.cltv_expiry), htlc.amount_msat + htlc.total_fee))
+
+        commit = self.make_commitment(self.local_state.ctn + 1,
+            True, this_point,
+            local_msat - total_fee_local, remote_msat - total_fee_remote, htlcs_in_local + htlcs_in_remote, trimmed)
+        return commit
+
+    def gen_htlc_indices(self, subject, just_unsettled=True):
+        assert subject in ["local", "remote"]
+        update_log = (self.remote_update_log if subject == "remote" else self.local_update_log)
+        other_log = (self.remote_update_log if subject != "remote" else self.local_update_log)
+        res = []
+        for htlc in update_log:
+            if type(htlc) is not UpdateAddHtlc:
+                continue
+            height = (self.local_state.ctn if subject == "remote" else self.remote_state.ctn)
+            locked_in = (htlc.r_locked_in if subject == "remote" else htlc.l_locked_in)
+
+            if locked_in is None or just_unsettled == (SettleHtlc(htlc.htlc_id) in other_log):
+                continue
+            res.append(htlc)
+        return res
+
+    @property
+    def htlcs_in_local(self):
+        return self.gen_htlc_indices("local")
+
+    @property
+    def htlcs_in_remote(self):
+        return self.gen_htlc_indices("remote")
+
+    def settle_htlc(self, preimage, htlc_id):
+        """
+        SettleHTLC attempts to settle an existing outstanding received HTLC.
+        """
+        self.print_error("settle_htlc")
+        htlc = self.lookup_htlc(self.remote_update_log, htlc_id)
+        assert htlc.payment_hash == sha256(preimage)
+        self.local_update_log.append(SettleHtlc(htlc_id))
+
+    def receive_htlc_settle(self, preimage, htlc_index):
+        self.print_error("receive_htlc_settle")
+        htlc = self.lookup_htlc(self.local_update_log, htlc_index)
+        assert htlc.payment_hash == sha256(preimage)
+        assert len([x.htlc_id == htlc_index for x in self.local_update_log]) == 1
+        self.remote_update_log.append(SettleHtlc(htlc_index))
+
+    def fail_htlc(self, htlc):
+        # TODO
+        self.local_update_log = []
+        self.remote_update_log = []
+        self.print_error("fail_htlc (EMPTIED LOGS)")
+
+    @property
+    def l_current_height(self):
+        return self.local_state.ctn
+
+    @property
+    def r_current_height(self):
+        return self.remote_state.ctn
+
+    @property
+    def local_commit_fee(self):
+        return self.constraints.capacity - sum(x[2] for x in self.local_commitment.outputs())
+
+    def update_fee(self, fee):
+        self.pending_feerate = fee
+
+    def receive_update_fee(self, fee):
+        self.pending_feerate = fee
+
+    def to_save(self):
+        return {
+                "local_config": self.local_config,
+                "remote_config": self.remote_config,
+                "local_state": self.local_state,
+                "remote_state": self.remote_state,
+                "channel_id": self.channel_id,
+                "short_channel_id": self.short_channel_id,
+                "constraints": self.constraints,
+                "funding_outpoint": self.funding_outpoint,
+                "node_id": self.node_id,
+                "channel_id": self.channel_id
+        }
+
+    def serialize(self):
+        namedtuples_to_dict = lambda v: {i: j._asdict() if isinstance(j, tuple) else j for i, j in v._asdict().items()}
+        serialized_channel = {k: namedtuples_to_dict(v) if isinstance(v, tuple) else v for k, v in self.to_save().items()}
+        class MyJsonEncoder(json.JSONEncoder):
+            def default(self, o):
+                if isinstance(o, bytes):
+                    return binascii.hexlify(o).decode("ascii")
+                if isinstance(o, RevocationStore):
+                    return o.serialize()
+                return super(MyJsonEncoder, self)
+        dumped = MyJsonEncoder().encode(serialized_channel)
+        roundtripped = json.loads(dumped)
+        reconstructed = HTLCStateMachine(roundtripped)
+        if reconstructed.to_save() != self.to_save():
+            raise Exception("Channels did not roundtrip serialization without changes:\n" + repr(reconstructed.to_save()) + "\n" + repr(self.to_save()))
+        return roundtripped
+
+    def __str__(self):
+        return self.serialize()
+
+    def make_commitment(chan, ctn, for_us, pcp, local_msat, remote_msat, htlcs=[], trimmed=0):
+        conf = chan.local_config if for_us else chan.remote_config
+        other_conf = chan.local_config if not for_us else chan.remote_config
+        payment_pubkey = derive_pubkey(other_conf.payment_basepoint.pubkey, pcp)
+        remote_revocation_pubkey = derive_blinded_pubkey(other_conf.revocation_basepoint.pubkey, pcp)
+        return make_commitment(
+            ctn,
+            conf.multisig_key.pubkey,
+            other_conf.multisig_key.pubkey,
+            payment_pubkey,
+            chan.local_config.payment_basepoint.pubkey,
+            chan.remote_config.payment_basepoint.pubkey,
+            remote_revocation_pubkey,
+            derive_pubkey(conf.delayed_basepoint.pubkey, pcp),
+            other_conf.to_self_delay,
+            *chan.funding_outpoint,
+            chan.constraints.capacity,
+            local_msat,
+            remote_msat,
+            chan.local_config.dust_limit_sat,
+            chan.constraints.feerate,
+            for_us,
+            chan.constraints.is_initiator,
+            htlcs=htlcs,
+            trimmed=trimmed)

lib/lnrouter.py

@@ -0,0 +1,525 @@
+# -*- coding: utf-8 -*-
+#
+# Electrum - lightweight Bitcoin client
+# Copyright (C) 2018 The Electrum developers
+#
+# Permission is hereby granted, free of charge, to any person
+# obtaining a copy of this software and associated documentation files
+# (the "Software"), to deal in the Software without restriction,
+# including without limitation the rights to use, copy, modify, merge,
+# publish, distribute, sublicense, and/or sell copies of the Software,
+# and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+
+import queue
+import traceback
+import sys
+import binascii
+import hashlib
+import hmac
+from collections import namedtuple, defaultdict
+from typing import Sequence, Union, Tuple
+
+from cryptography.hazmat.primitives.ciphers import Cipher, algorithms
+from cryptography.hazmat.backends import default_backend
+
+from . import bitcoin
+from . import ecc
+from . import crypto
+from .crypto import sha256
+from .util import PrintError, bh2u, profiler, xor_bytes
+from .lnutil import get_ecdh
+
+
+class ChannelInfo(PrintError):
+
+    def __init__(self, channel_announcement_payload):
+        self.channel_id = channel_announcement_payload['short_channel_id']
+        self.node_id_1 = channel_announcement_payload['node_id_1']
+        self.node_id_2 = channel_announcement_payload['node_id_2']
+        assert type(self.node_id_1) is bytes
+        assert type(self.node_id_2) is bytes
+        assert list(sorted([self.node_id_1, self.node_id_2])) == [self.node_id_1, self.node_id_2]
+
+        self.capacity_sat = None
+        self.policy_node1 = None
+        self.policy_node2 = None
+
+    def set_capacity(self, capacity):
+        # TODO call this after looking up UTXO for funding txn on chain
+        self.capacity_sat = capacity
+
+    def on_channel_update(self, msg_payload):
+        assert self.channel_id == msg_payload['short_channel_id']
+        flags = int.from_bytes(msg_payload['flags'], 'big')
+        direction = flags & 1
+        if direction == 0:
+            self.policy_node1 = ChannelInfoDirectedPolicy(msg_payload)
+        else:
+            self.policy_node2 = ChannelInfoDirectedPolicy(msg_payload)
+        self.print_error('channel update', binascii.hexlify(self.channel_id).decode("ascii"), flags)
+
+    def get_policy_for_node(self, node_id):
+        if node_id == self.node_id_1:
+            return self.policy_node1
+        elif node_id == self.node_id_2:
+            return self.policy_node2
+        else:
+            raise Exception('node_id {} not in channel {}'.format(node_id, self.channel_id))
+
+
+class ChannelInfoDirectedPolicy:
+
+    def __init__(self, channel_update_payload):
+        self.cltv_expiry_delta           = channel_update_payload['cltv_expiry_delta']
+        self.htlc_minimum_msat           = channel_update_payload['htlc_minimum_msat']
+        self.fee_base_msat               = channel_update_payload['fee_base_msat']
+        self.fee_proportional_millionths = channel_update_payload['fee_proportional_millionths']
+        self.cltv_expiry_delta = int.from_bytes(self.cltv_expiry_delta, "big")
+        self.htlc_minimum_msat = int.from_bytes(self.htlc_minimum_msat, "big")
+        self.fee_base_msat = int.from_bytes(self.fee_base_msat, "big")
+        self.fee_proportional_millionths = int.from_bytes(self.fee_proportional_millionths, "big")
+
+
+class ChannelDB(PrintError):
+
+    def __init__(self):
+        self._id_to_channel_info = {}
+        self._channels_for_node = defaultdict(set)  # node -> set(short_channel_id)
+
+    def get_channel_info(self, channel_id):
+        return self._id_to_channel_info.get(channel_id, None)
+
+    def get_channels_for_node(self, node_id):
+        """Returns the set of channels that have node_id as one of the endpoints."""
+        return self._channels_for_node[node_id]
+
+    def on_channel_announcement(self, msg_payload):
+        short_channel_id = msg_payload['short_channel_id']
+        self.print_error('channel announcement', binascii.hexlify(short_channel_id).decode("ascii"))
+        channel_info = ChannelInfo(msg_payload)
+        self._id_to_channel_info[short_channel_id] = channel_info
+        self._channels_for_node[channel_info.node_id_1].add(short_channel_id)
+        self._channels_for_node[channel_info.node_id_2].add(short_channel_id)
+
+    def on_channel_update(self, msg_payload):
+        short_channel_id = msg_payload['short_channel_id']
+        try:
+            channel_info = self._id_to_channel_info[short_channel_id]
+        except KeyError:
+            self.print_error("could not find", short_channel_id)
+        else:
+            channel_info.on_channel_update(msg_payload)
+
+    def remove_channel(self, short_channel_id):
+        try:
+            channel_info = self._id_to_channel_info[short_channel_id]
+        except KeyError:
+            self.print_error('cannot find channel {}'.format(short_channel_id))
+            return
+        self._id_to_channel_info.pop(short_channel_id, None)
+        for node in (channel_info.node_id_1, channel_info.node_id_2):
+            try:
+                self._channels_for_node[node].remove(short_channel_id)
+            except KeyError:
+                pass
+
+
+class RouteEdge:
+
+    def __init__(self, node_id: bytes, short_channel_id: bytes,
+                 channel_policy: ChannelInfoDirectedPolicy):
+        # "if you travel through short_channel_id, you will reach node_id"
+        self.node_id = node_id
+        self.short_channel_id = short_channel_id
+        self.channel_policy = channel_policy
+
+
+class LNPathFinder(PrintError):
+
+    def __init__(self, channel_db):
+        self.channel_db = channel_db
+        self.blacklist = set()
+
+    def _edge_cost(self, short_channel_id: bytes, start_node: bytes, payment_amt_msat: int,
+                   ignore_cltv=False) -> float:
+        """Heuristic cost of going through a channel.
+        direction: 0 or 1. --- 0 means node_id_1 -> node_id_2
+        """
+        channel_info = self.channel_db.get_channel_info(short_channel_id)
+        if channel_info is None:
+            return float('inf')
+
+        channel_policy = channel_info.get_policy_for_node(start_node)
+        if channel_policy is None: return float('inf')
+        cltv_expiry_delta           = channel_policy.cltv_expiry_delta
+        htlc_minimum_msat           = channel_policy.htlc_minimum_msat
+        fee_base_msat               = channel_policy.fee_base_msat
+        fee_proportional_millionths = channel_policy.fee_proportional_millionths
+        if payment_amt_msat is not None:
+            if payment_amt_msat < htlc_minimum_msat:
+                return float('inf')  # payment amount too little
+            if channel_info.capacity_sat is not None and \
+                    payment_amt_msat // 1000 > channel_info.capacity_sat:
+                return float('inf')  # payment amount too large
+        amt = payment_amt_msat or 50000 * 1000  # guess for typical payment amount
+        fee_msat = fee_base_msat + amt * fee_proportional_millionths / 1000000
+        # TODO revise
+        # paying 10 more satoshis ~ waiting one more block
+        fee_cost = fee_msat / 1000 / 10
+        cltv_cost = cltv_expiry_delta if not ignore_cltv else 0
+        return cltv_cost + fee_cost + 1
+
+    @profiler
+    def find_path_for_payment(self, from_node_id: bytes, to_node_id: bytes,
+                              amount_msat: int=None) -> Sequence[Tuple[bytes, bytes]]:
+        """Return a path between from_node_id and to_node_id.
+
+        Returns a list of (node_id, short_channel_id) representing a path.
+        To get from node ret[n][0] to ret[n+1][0], use channel ret[n+1][1];
+        i.e. an element reads as, "to get to node_id, travel through short_channel_id"
+        """
+        if amount_msat is not None: assert type(amount_msat) is int
+        # TODO find multiple paths??
+
+        # run Dijkstra
+        distance_from_start = defaultdict(lambda: float('inf'))
+        distance_from_start[from_node_id] = 0
+        prev_node = {}
+        nodes_to_explore = queue.PriorityQueue()
+        nodes_to_explore.put((0, from_node_id))
+
+        while nodes_to_explore.qsize() > 0:
+            dist_to_cur_node, cur_node = nodes_to_explore.get()
+            if cur_node == to_node_id:
+                break
+            if dist_to_cur_node != distance_from_start[cur_node]:
+                # queue.PriorityQueue does not implement decrease_priority,
+                # so instead of decreasing priorities, we add items again into the queue.
+                # so there are duplicates in the queue, that we discard now:
+                continue
+            for edge_channel_id in self.channel_db.get_channels_for_node(cur_node):
+                if edge_channel_id in self.blacklist: continue
+                channel_info = self.channel_db.get_channel_info(edge_channel_id)
+                node1, node2 = channel_info.node_id_1, channel_info.node_id_2
+                neighbour = node2 if node1 == cur_node else node1
+                ignore_cltv_delta_in_edge_cost = cur_node == from_node_id
+                edge_cost = self._edge_cost(edge_channel_id, cur_node, amount_msat,
+                                            ignore_cltv=ignore_cltv_delta_in_edge_cost)
+                alt_dist_to_neighbour = distance_from_start[cur_node] + edge_cost
+                if alt_dist_to_neighbour < distance_from_start[neighbour]:
+                    distance_from_start[neighbour] = alt_dist_to_neighbour
+                    prev_node[neighbour] = cur_node, edge_channel_id
+                    nodes_to_explore.put((alt_dist_to_neighbour, neighbour))
+        else:
+            return None  # no path found
+
+        # backtrack from end to start
+        cur_node = to_node_id
+        path = []
+        while cur_node != from_node_id:
+            prev_node_id, edge_taken = prev_node[cur_node]
+            path += [(cur_node, edge_taken)]
+            cur_node = prev_node_id
+        path.reverse()
+        return path
+
+    def create_route_from_path(self, path, from_node_id: bytes) -> Sequence[RouteEdge]:
+        assert type(from_node_id) is bytes
+        if path is None:
+            raise Exception('cannot create route from None path')
+        route = []
+        prev_node_id = from_node_id
+        for node_id, short_channel_id in path:
+            channel_info = self.channel_db.get_channel_info(short_channel_id)
+            if channel_info is None:
+                raise Exception('cannot find channel info for short_channel_id: {}'.format(bh2u(short_channel_id)))
+            channel_policy = channel_info.get_policy_for_node(prev_node_id)
+            if channel_policy is None:
+                raise Exception('cannot find channel policy for short_channel_id: {}'.format(bh2u(short_channel_id)))
+            route.append(RouteEdge(node_id, short_channel_id, channel_policy))
+            prev_node_id = node_id
+        return route
+
+
+# bolt 04, "onion"  ----->
+
+NUM_MAX_HOPS_IN_PATH = 20
+HOPS_DATA_SIZE = 1300      # also sometimes called routingInfoSize in bolt-04
+PER_HOP_FULL_SIZE = 65     # HOPS_DATA_SIZE / 20
+NUM_STREAM_BYTES = HOPS_DATA_SIZE + PER_HOP_FULL_SIZE
+PER_HOP_HMAC_SIZE = 32
+
+
+class UnsupportedOnionPacketVersion(Exception): pass
+class InvalidOnionMac(Exception): pass
+
+
+class OnionPerHop:
+
+    def __init__(self, short_channel_id: bytes, amt_to_forward: bytes, outgoing_cltv_value: bytes):
+        self.short_channel_id = short_channel_id
+        self.amt_to_forward = amt_to_forward
+        self.outgoing_cltv_value = outgoing_cltv_value
+
+    def to_bytes(self) -> bytes:
+        ret = self.short_channel_id
+        ret += self.amt_to_forward
+        ret += self.outgoing_cltv_value
+        ret += bytes(12)  # padding
+        if len(ret) != 32:
+            raise Exception('unexpected length {}'.format(len(ret)))
+        return ret
+
+    @classmethod
+    def from_bytes(cls, b: bytes):
+        if len(b) != 32:
+            raise Exception('unexpected length {}'.format(len(b)))
+        return OnionPerHop(
+            short_channel_id=b[:8],
+            amt_to_forward=b[8:16],
+            outgoing_cltv_value=b[16:20]
+        )
+
+
+class OnionHopsDataSingle:  # called HopData in lnd
+
+    def __init__(self, per_hop: OnionPerHop = None):
+        self.realm = 0
+        self.per_hop = per_hop
+        self.hmac = None
+
+    def to_bytes(self) -> bytes:
+        ret = bytes([self.realm])
+        ret += self.per_hop.to_bytes()
+        ret += self.hmac if self.hmac is not None else bytes(PER_HOP_HMAC_SIZE)
+        if len(ret) != PER_HOP_FULL_SIZE:
+            raise Exception('unexpected length {}'.format(len(ret)))
+        return ret
+
+    @classmethod
+    def from_bytes(cls, b: bytes):
+        if len(b) != PER_HOP_FULL_SIZE:
+            raise Exception('unexpected length {}'.format(len(b)))
+        ret = OnionHopsDataSingle()
+        ret.realm = b[0]
+        if ret.realm != 0:
+            raise Exception('only realm 0 is supported')
+        ret.per_hop = OnionPerHop.from_bytes(b[1:33])
+        ret.hmac = b[33:]
+        return ret
+
+
+class OnionPacket:
+
+    def __init__(self, public_key: bytes, hops_data: bytes, hmac: bytes):
+        self.version = 0
+        self.public_key = public_key
+        self.hops_data = hops_data  # also called RoutingInfo in bolt-04
+        self.hmac = hmac
+
+    def to_bytes(self) -> bytes:
+        ret = bytes([self.version])
+        ret += self.public_key
+        ret += self.hops_data
+        ret += self.hmac
+        if len(ret) != 1366:
+            raise Exception('unexpected length {}'.format(len(ret)))
+        return ret
+
+    @classmethod
+    def from_bytes(cls, b: bytes):
+        if len(b) != 1366:
+            raise Exception('unexpected length {}'.format(len(b)))
+        version = b[0]
+        if version != 0:
+            raise UnsupportedOnionPacketVersion('version {} is not supported'.format(version))
+        return OnionPacket(
+            public_key=b[1:34],
+            hops_data=b[34:1334],
+            hmac=b[1334:]
+        )
+
+
+def get_bolt04_onion_key(key_type: bytes, secret: bytes) -> bytes:
+    if key_type not in (b'rho', b'mu', b'um', b'ammag'):
+        raise Exception('invalid key_type {}'.format(key_type))
+    key = hmac.new(key_type, msg=secret, digestmod=hashlib.sha256).digest()
+    return key
+
+
+def get_shared_secrets_along_route(payment_path_pubkeys: Sequence[bytes],
+                                   session_key: bytes) -> Sequence[bytes]:
+    num_hops = len(payment_path_pubkeys)
+    hop_shared_secrets = num_hops * [b'']
+    ephemeral_key = session_key
+    # compute shared key for each hop
+    for i in range(0, num_hops):
+        hop_shared_secrets[i] = get_ecdh(ephemeral_key, payment_path_pubkeys[i])
+        ephemeral_pubkey = ecc.ECPrivkey(ephemeral_key).get_public_key_bytes()
+        blinding_factor = sha256(ephemeral_pubkey + hop_shared_secrets[i])
+        blinding_factor_int = int.from_bytes(blinding_factor, byteorder="big")
+        ephemeral_key_int = int.from_bytes(ephemeral_key, byteorder="big")
+        ephemeral_key_int = ephemeral_key_int * blinding_factor_int % ecc.CURVE_ORDER
+        ephemeral_key = ephemeral_key_int.to_bytes(32, byteorder="big")
+    return hop_shared_secrets
+
+
+def new_onion_packet(payment_path_pubkeys: Sequence[bytes], session_key: bytes,
+                     hops_data: Sequence[OnionHopsDataSingle], associated_data: bytes) -> OnionPacket:
+    num_hops = len(payment_path_pubkeys)
+    hop_shared_secrets = get_shared_secrets_along_route(payment_path_pubkeys, session_key)
+
+    filler = generate_filler(b'rho', num_hops, PER_HOP_FULL_SIZE, hop_shared_secrets)
+    mix_header = bytes(HOPS_DATA_SIZE)
+    next_hmac = bytes(PER_HOP_HMAC_SIZE)
+
+    # compute routing info and MAC for each hop
+    for i in range(num_hops-1, -1, -1):
+        rho_key = get_bolt04_onion_key(b'rho', hop_shared_secrets[i])
+        mu_key = get_bolt04_onion_key(b'mu', hop_shared_secrets[i])
+        hops_data[i].hmac = next_hmac
+        stream_bytes = generate_cipher_stream(rho_key, NUM_STREAM_BYTES)
+        mix_header = mix_header[:-PER_HOP_FULL_SIZE]
+        mix_header = hops_data[i].to_bytes() + mix_header
+        mix_header = xor_bytes(mix_header, stream_bytes)
+        if i == num_hops - 1 and len(filler) != 0:
+            mix_header = mix_header[:-len(filler)] + filler
+        packet = mix_header + associated_data
+        next_hmac = hmac.new(mu_key, msg=packet, digestmod=hashlib.sha256).digest()
+
+    return OnionPacket(
+        public_key=ecc.ECPrivkey(session_key).get_public_key_bytes(),
+        hops_data=mix_header,
+        hmac=next_hmac)
+
+
+def generate_filler(key_type: bytes, num_hops: int, hop_size: int,
+                    shared_secrets: Sequence[bytes]) -> bytes:
+    filler_size = (NUM_MAX_HOPS_IN_PATH + 1) * hop_size
+    filler = bytearray(filler_size)
+
+    for i in range(0, num_hops-1):  # -1, as last hop does not obfuscate
+        filler = filler[hop_size:]
+        filler += bytearray(hop_size)
+        stream_key = get_bolt04_onion_key(key_type, shared_secrets[i])
+        stream_bytes = generate_cipher_stream(stream_key, filler_size)
+        filler = xor_bytes(filler, stream_bytes)
+
+    return filler[(NUM_MAX_HOPS_IN_PATH-num_hops+2)*hop_size:]
+
+
+def generate_cipher_stream(stream_key: bytes, num_bytes: int) -> bytes:
+    algo = algorithms.ChaCha20(stream_key, nonce=bytes(16))
+    cipher = Cipher(algo, mode=None, backend=default_backend())
+    encryptor = cipher.encryptor()
+    return encryptor.update(bytes(num_bytes))
+
+
+ProcessedOnionPacket = namedtuple("ProcessedOnionPacket", ["are_we_final", "hop_data", "next_packet"])
+
+
+# TODO replay protection
+def process_onion_packet(onion_packet: OnionPacket, associated_data: bytes,
+                         our_onion_private_key: bytes) -> ProcessedOnionPacket:
+    shared_secret = get_ecdh(our_onion_private_key, onion_packet.public_key)
+
+    # check message integrity
+    mu_key = get_bolt04_onion_key(b'mu', shared_secret)
+    calculated_mac = hmac.new(mu_key, msg=onion_packet.hops_data+associated_data,
+                              digestmod=hashlib.sha256).digest()
+    if onion_packet.hmac != calculated_mac:
+        raise InvalidOnionMac()
+
+    # peel an onion layer off
+    rho_key = get_bolt04_onion_key(b'rho', shared_secret)
+    stream_bytes = generate_cipher_stream(rho_key, NUM_STREAM_BYTES)
+    padded_header = onion_packet.hops_data + bytes(PER_HOP_FULL_SIZE)
+    next_hops_data = xor_bytes(padded_header, stream_bytes)
+
+    # calc next ephemeral key
+    blinding_factor = sha256(onion_packet.public_key + shared_secret)
+    blinding_factor_int = int.from_bytes(blinding_factor, byteorder="big")
+    next_public_key_int = ecc.ECPubkey(onion_packet.public_key) * blinding_factor_int
+    next_public_key = next_public_key_int.get_public_key_bytes()
+
+    hop_data = OnionHopsDataSingle.from_bytes(next_hops_data[:PER_HOP_FULL_SIZE])
+    next_onion_packet = OnionPacket(
+        public_key=next_public_key,
+        hops_data=next_hops_data[PER_HOP_FULL_SIZE:],
+        hmac=hop_data.hmac
+    )
+    if hop_data.hmac == bytes(PER_HOP_HMAC_SIZE):
+        # we are the destination / exit node
+        are_we_final = True
+    else:
+        # we are an intermediate node; forwarding
+        are_we_final = False
+    return ProcessedOnionPacket(are_we_final, hop_data, next_onion_packet)
+
+
+class FailedToDecodeOnionError(Exception): pass
+
+
+class OnionRoutingFailureMessage:
+
+    def __init__(self, code: int, data: bytes):
+        self.code = code
+        self.data = data
+
+    def __repr__(self):
+        return repr((self.code, self.data))
+
+
+def _decode_onion_error(error_packet: bytes, payment_path_pubkeys: Sequence[bytes],
+                        session_key: bytes) -> (bytes, int):
+    """Returns the decoded error bytes, and the index of the sender of the error."""
+    num_hops = len(payment_path_pubkeys)
+    hop_shared_secrets = get_shared_secrets_along_route(payment_path_pubkeys, session_key)
+    for i in range(num_hops):
+        ammag_key = get_bolt04_onion_key(b'ammag', hop_shared_secrets[i])
+        um_key = get_bolt04_onion_key(b'um', hop_shared_secrets[i])
+        stream_bytes = generate_cipher_stream(ammag_key, len(error_packet))
+        error_packet = xor_bytes(error_packet, stream_bytes)
+        hmac_computed = hmac.new(um_key, msg=error_packet[32:], digestmod=hashlib.sha256).digest()
+        hmac_found = error_packet[:32]
+        if hmac_computed == hmac_found:
+            return error_packet, i
+    raise FailedToDecodeOnionError()
+
+
+def decode_onion_error(error_packet: bytes, payment_path_pubkeys: Sequence[bytes],
+                       session_key: bytes) -> (OnionRoutingFailureMessage, int):
+    """Returns the failure message, and the index of the sender of the error."""
+    decrypted_error, sender_index = _decode_onion_error(error_packet, payment_path_pubkeys, session_key)
+    failure_msg = get_failure_msg_from_onion_error(decrypted_error)
+    return failure_msg, sender_index
+
+
+def get_failure_msg_from_onion_error(decrypted_error_packet: bytes) -> OnionRoutingFailureMessage:
+    # get failure_msg bytes from error packet
+    failure_len = int.from_bytes(decrypted_error_packet[32:34], byteorder='big')
+    failure_msg = decrypted_error_packet[34:34+failure_len]
+    # create failure message object
+    failure_code = int.from_bytes(failure_msg[:2], byteorder='big')
+    failure_data = failure_msg[2:]
+    return OnionRoutingFailureMessage(failure_code, failure_data)
+
+
+
+
+# <----- bolt 04, "onion"
+

lib/lnutil.py

@@ -0,0 +1,315 @@
+from .util import bfh, bh2u
+from .crypto import sha256
+import json
+from collections import namedtuple
+from .transaction import Transaction
+from .ecc import CURVE_ORDER, sig_string_from_der_sig, ECPubkey, string_to_number
+from . import ecc, bitcoin, crypto, transaction
+from .transaction import opcodes
+from .bitcoin import push_script
+
+HTLC_TIMEOUT_WEIGHT = 663
+HTLC_SUCCESS_WEIGHT = 703
+
+Keypair = namedtuple("Keypair", ["pubkey", "privkey"])
+Outpoint = namedtuple("Outpoint", ["txid", "output_index"])
+ChannelConfig = namedtuple("ChannelConfig", [
+    "payment_basepoint", "multisig_key", "htlc_basepoint", "delayed_basepoint", "revocation_basepoint",
+    "to_self_delay", "dust_limit_sat", "max_htlc_value_in_flight_msat", "max_accepted_htlcs"])
+OnlyPubkeyKeypair = namedtuple("OnlyPubkeyKeypair", ["pubkey"])
+RemoteState = namedtuple("RemoteState", ["ctn", "next_per_commitment_point", "amount_msat", "revocation_store", "current_per_commitment_point", "next_htlc_id"])
+LocalState = namedtuple("LocalState", ["ctn", "per_commitment_secret_seed", "amount_msat", "next_htlc_id", "funding_locked_received", "was_announced", "current_commitment_signature"])
+ChannelConstraints = namedtuple("ChannelConstraints", ["feerate", "capacity", "is_initiator", "funding_txn_minimum_depth"])
+#OpenChannel = namedtuple("OpenChannel", ["channel_id", "short_channel_id", "funding_outpoint", "local_config", "remote_config", "remote_state", "local_state", "constraints", "node_id"])
+
+class RevocationStore:
+    """ taken from lnd """
+    def __init__(self):
+        self.buckets = [None] * 48
+        self.index = 2**48 - 1
+    def add_next_entry(self, hsh):
+        new_element = ShachainElement(index=self.index, secret=hsh)
+        bucket = count_trailing_zeros(self.index)
+        for i in range(0, bucket):
+            this_bucket = self.buckets[i]
+            e = shachain_derive(new_element, this_bucket.index)
+
+            if e != this_bucket:
+                raise Exception("hash is not derivable: {} {} {}".format(bh2u(e.secret), bh2u(this_bucket.secret), this_bucket.index))
+        self.buckets[bucket] = new_element
+        self.index -= 1
+    def serialize(self):
+        return {"index": self.index, "buckets": [[bh2u(k.secret), k.index] if k is not None else None for k in self.buckets]}
+    @staticmethod
+    def from_json_obj(decoded_json_obj):
+        store = RevocationStore()
+        decode = lambda to_decode: ShachainElement(bfh(to_decode[0]), int(to_decode[1]))
+        store.buckets = [k if k is None else decode(k) for k in decoded_json_obj["buckets"]]
+        store.index = decoded_json_obj["index"]
+        return store
+    def __eq__(self, o):
+        return type(o) is RevocationStore and self.serialize() == o.serialize()
+    def __hash__(self):
+        return hash(json.dumps(self.serialize(), sort_keys=True))
+
+def count_trailing_zeros(index):
+    """ BOLT-03 (where_to_put_secret) """
+    try:
+        return list(reversed(bin(index)[2:])).index("1")
+    except ValueError:
+        return 48
+
+def shachain_derive(element, toIndex):
+    return ShachainElement(get_per_commitment_secret_from_seed(element.secret, toIndex, count_trailing_zeros(element.index)), toIndex)
+
+ShachainElement = namedtuple("ShachainElement", ["secret", "index"])
+ShachainElement.__str__ = lambda self: "ShachainElement(" + bh2u(self.secret) + "," + str(self.index) + ")"
+
+def get_per_commitment_secret_from_seed(seed: bytes, i: int, bits: int = 48) -> bytes:
+    """Generate per commitment secret."""
+    per_commitment_secret = bytearray(seed)
+    for bitindex in range(bits - 1, -1, -1):
+        mask = 1 << bitindex
+        if i & mask:
+            per_commitment_secret[bitindex // 8] ^= 1 << (bitindex % 8)
+            per_commitment_secret = bytearray(sha256(per_commitment_secret))
+    bajts = bytes(per_commitment_secret)
+    return bajts
+
+def secret_to_pubkey(secret):
+    assert type(secret) is int
+    return (ecc.generator() * secret).get_public_key_bytes()
+
+def derive_pubkey(basepoint, per_commitment_point):
+    p = ecc.ECPubkey(basepoint) + ecc.generator() * ecc.string_to_number(sha256(per_commitment_point + basepoint))
+    return p.get_public_key_bytes()
+
+def derive_privkey(secret, per_commitment_point):
+    assert type(secret) is int
+    basepoint = secret_to_pubkey(secret)
+    basepoint = secret + ecc.string_to_number(sha256(per_commitment_point + basepoint))
+    basepoint %= CURVE_ORDER
+    return basepoint
+
+def derive_blinded_pubkey(basepoint, per_commitment_point):
+    k1 = ecc.ECPubkey(basepoint) * ecc.string_to_number(sha256(basepoint + per_commitment_point))
+    k2 = ecc.ECPubkey(per_commitment_point) * ecc.string_to_number(sha256(per_commitment_point + basepoint))
+    return (k1 + k2).get_public_key_bytes()
+
+def make_htlc_tx_output(amount_msat, local_feerate, revocationpubkey, local_delayedpubkey, success, to_self_delay):
+    assert type(amount_msat) is int
+    assert type(local_feerate) is int
+    assert type(revocationpubkey) is bytes
+    assert type(local_delayedpubkey) is bytes
+    script = bytes([opcodes.OP_IF]) \
+        + bfh(push_script(bh2u(revocationpubkey))) \
+        + bytes([opcodes.OP_ELSE]) \
+        + bitcoin.add_number_to_script(to_self_delay) \
+        + bytes([opcodes.OP_CSV, opcodes.OP_DROP]) \
+        + bfh(push_script(bh2u(local_delayedpubkey))) \
+        + bytes([opcodes.OP_ENDIF, opcodes.OP_CHECKSIG])
+
+    p2wsh = bitcoin.redeem_script_to_address('p2wsh', bh2u(script))
+    weight = HTLC_SUCCESS_WEIGHT if success else HTLC_TIMEOUT_WEIGHT
+    fee = local_feerate * weight
+    final_amount_sat = (amount_msat - fee) // 1000
+    assert final_amount_sat > 0, final_amount_sat
+    output = (bitcoin.TYPE_ADDRESS, p2wsh, final_amount_sat)
+    return output
+
+def make_htlc_tx_witness(remotehtlcsig, localhtlcsig, payment_preimage, witness_script):
+    assert type(remotehtlcsig) is bytes
+    assert type(localhtlcsig) is bytes
+    assert type(payment_preimage) is bytes
+    assert type(witness_script) is bytes
+    return bfh(transaction.construct_witness([0, remotehtlcsig, localhtlcsig, payment_preimage, witness_script]))
+
+def make_htlc_tx_inputs(htlc_output_txid, htlc_output_index, revocationpubkey, local_delayedpubkey, amount_msat, witness_script):
+    assert type(htlc_output_txid) is str
+    assert type(htlc_output_index) is int
+    assert type(revocationpubkey) is bytes
+    assert type(local_delayedpubkey) is bytes
+    assert type(amount_msat) is int
+    assert type(witness_script) is str
+    c_inputs = [{
+        'scriptSig': '',
+        'type': 'p2wsh',
+        'signatures': [],
+        'num_sig': 0,
+        'prevout_n': htlc_output_index,
+        'prevout_hash': htlc_output_txid,
+        'value': amount_msat // 1000,
+        'coinbase': False,
+        'sequence': 0x0,
+        'preimage_script': witness_script,
+    }]
+    return c_inputs
+
+def make_htlc_tx(cltv_timeout, inputs, output):
+    assert type(cltv_timeout) is int
+    c_outputs = [output]
+    tx = Transaction.from_io(inputs, c_outputs, locktime=cltv_timeout, version=2)
+    tx.BIP_LI01_sort()
+    return tx
+
+def make_offered_htlc(revocation_pubkey, remote_htlcpubkey, local_htlcpubkey, payment_hash):
+    assert type(revocation_pubkey) is bytes
+    assert type(remote_htlcpubkey) is bytes
+    assert type(local_htlcpubkey) is bytes
+    assert type(payment_hash) is bytes
+    return bytes([opcodes.OP_DUP, opcodes.OP_HASH160]) + bfh(push_script(bh2u(bitcoin.hash_160(revocation_pubkey))))\
+        + bytes([opcodes.OP_EQUAL, opcodes.OP_IF, opcodes.OP_CHECKSIG, opcodes.OP_ELSE]) \
+        + bfh(push_script(bh2u(remote_htlcpubkey)))\
+        + bytes([opcodes.OP_SWAP, opcodes.OP_SIZE]) + bitcoin.add_number_to_script(32) + bytes([opcodes.OP_EQUAL, opcodes.OP_NOTIF, opcodes.OP_DROP])\
+        + bitcoin.add_number_to_script(2) + bytes([opcodes.OP_SWAP]) + bfh(push_script(bh2u(local_htlcpubkey))) + bitcoin.add_number_to_script(2)\
+        + bytes([opcodes.OP_CHECKMULTISIG, opcodes.OP_ELSE, opcodes.OP_HASH160])\
+        + bfh(push_script(bh2u(crypto.ripemd(payment_hash)))) + bytes([opcodes.OP_EQUALVERIFY, opcodes.OP_CHECKSIG, opcodes.OP_ENDIF, opcodes.OP_ENDIF])
+
+def make_received_htlc(revocation_pubkey, remote_htlcpubkey, local_htlcpubkey, payment_hash, cltv_expiry):
+    for i in [revocation_pubkey, remote_htlcpubkey, local_htlcpubkey, payment_hash]:
+        assert type(i) is bytes
+    assert type(cltv_expiry) is int
+
+    return bytes([opcodes.OP_DUP, opcodes.OP_HASH160]) \
+        + bfh(push_script(bh2u(bitcoin.hash_160(revocation_pubkey)))) \
+        + bytes([opcodes.OP_EQUAL, opcodes.OP_IF, opcodes.OP_CHECKSIG, opcodes.OP_ELSE]) \
+        + bfh(push_script(bh2u(remote_htlcpubkey))) \
+        + bytes([opcodes.OP_SWAP, opcodes.OP_SIZE]) \
+        + bitcoin.add_number_to_script(32) \
+        + bytes([opcodes.OP_EQUAL, opcodes.OP_IF, opcodes.OP_HASH160]) \
+        + bfh(push_script(bh2u(crypto.ripemd(payment_hash)))) \
+        + bytes([opcodes.OP_EQUALVERIFY]) \
+        + bitcoin.add_number_to_script(2) \
+        + bytes([opcodes.OP_SWAP]) \
+        + bfh(push_script(bh2u(local_htlcpubkey))) \
+        + bitcoin.add_number_to_script(2) \
+        + bytes([opcodes.OP_CHECKMULTISIG, opcodes.OP_ELSE, opcodes.OP_DROP]) \
+        + bitcoin.add_number_to_script(cltv_expiry) \
+        + bytes([opcodes.OP_CLTV, opcodes.OP_DROP, opcodes.OP_CHECKSIG, opcodes.OP_ENDIF, opcodes.OP_ENDIF])
+
+def make_htlc_tx_with_open_channel(chan, pcp, for_us, we_receive, amount_msat, cltv_expiry, payment_hash, commit, original_htlc_output_index):
+    conf = chan.local_config if for_us else chan.remote_config
+    other_conf = chan.local_config if not for_us else chan.remote_config
+
+    revocation_pubkey = derive_blinded_pubkey(other_conf.revocation_basepoint.pubkey, pcp)
+    delayedpubkey = derive_pubkey(conf.delayed_basepoint.pubkey, pcp)
+    other_revocation_pubkey = derive_blinded_pubkey(other_conf.revocation_basepoint.pubkey, pcp)
+    other_htlc_pubkey = derive_pubkey(other_conf.htlc_basepoint.pubkey, pcp)
+    htlc_pubkey = derive_pubkey(conf.htlc_basepoint.pubkey, pcp)
+    # HTLC-success for the HTLC spending from a received HTLC output
+    # if we do not receive, and the commitment tx is not for us, they receive, so it is also an HTLC-success
+    is_htlc_success = for_us == we_receive
+    htlc_tx_output = make_htlc_tx_output(
+        amount_msat = amount_msat,
+        local_feerate = chan.constraints.feerate,
+        revocationpubkey=revocation_pubkey,
+        local_delayedpubkey=delayedpubkey,
+        success = is_htlc_success,
+        to_self_delay = other_conf.to_self_delay)
+    if is_htlc_success:
+        preimage_script = make_received_htlc(other_revocation_pubkey, other_htlc_pubkey, htlc_pubkey, payment_hash, cltv_expiry)
+    else:
+        preimage_script = make_offered_htlc(other_revocation_pubkey, other_htlc_pubkey, htlc_pubkey, payment_hash)
+    htlc_tx_inputs = make_htlc_tx_inputs(
+        commit.txid(), commit.htlc_output_indices[original_htlc_output_index],
+        revocationpubkey=revocation_pubkey,
+        local_delayedpubkey=delayedpubkey,
+        amount_msat=amount_msat,
+        witness_script=bh2u(preimage_script))
+    if is_htlc_success:
+        cltv_expiry = 0
+    htlc_tx = make_htlc_tx(cltv_expiry, inputs=htlc_tx_inputs, output=htlc_tx_output)
+    return htlc_tx
+
+
+def make_commitment(ctn, local_funding_pubkey, remote_funding_pubkey,
+                    remote_payment_pubkey, payment_basepoint,
+                    remote_payment_basepoint, revocation_pubkey,
+                    delayed_pubkey, to_self_delay, funding_txid,
+                    funding_pos, funding_sat, local_amount, remote_amount,
+                    dust_limit_sat, local_feerate, for_us, we_are_initiator,
+                    htlcs, trimmed=0):
+
+    pubkeys = sorted([bh2u(local_funding_pubkey), bh2u(remote_funding_pubkey)])
+    payments = [payment_basepoint, remote_payment_basepoint]
+    if not we_are_initiator:
+        payments.reverse()
+    obs = get_obscured_ctn(ctn, *payments)
+    locktime = (0x20 << 24) + (obs & 0xffffff)
+    sequence = (0x80 << 24) + (obs >> 24)
+    # commitment tx input
+    c_inputs = [{
+        'type': 'p2wsh',
+        'x_pubkeys': pubkeys,
+        'signatures': [None, None],
+        'num_sig': 2,
+        'prevout_n': funding_pos,
+        'prevout_hash': funding_txid,
+        'value': funding_sat,
+        'coinbase': False,
+        'sequence': sequence
+    }]
+    # commitment tx outputs
+    local_script = bytes([opcodes.OP_IF]) + bfh(push_script(bh2u(revocation_pubkey))) + bytes([opcodes.OP_ELSE]) + bitcoin.add_number_to_script(to_self_delay) \
+                   + bytes([opcodes.OP_CSV, opcodes.OP_DROP]) + bfh(push_script(bh2u(delayed_pubkey))) + bytes([opcodes.OP_ENDIF, opcodes.OP_CHECKSIG])
+    local_address = bitcoin.redeem_script_to_address('p2wsh', bh2u(local_script))
+    remote_address = bitcoin.pubkey_to_address('p2wpkh', bh2u(remote_payment_pubkey))
+    # TODO trim htlc outputs here while also considering 2nd stage htlc transactions
+    fee = local_feerate * overall_weight(len(htlcs))
+    fee -= trimmed * 1000
+    assert type(fee) is int
+    we_pay_fee = for_us == we_are_initiator
+    to_local_amt = local_amount - (fee if we_pay_fee else 0)
+    assert type(to_local_amt) is int
+    to_local = (bitcoin.TYPE_ADDRESS, local_address, to_local_amt // 1000)
+    to_remote_amt = remote_amount - (fee if not we_pay_fee else 0)
+    assert type(to_remote_amt) is int
+    to_remote = (bitcoin.TYPE_ADDRESS, remote_address, to_remote_amt // 1000)
+    c_outputs = [to_local, to_remote]
+    for script, msat_amount in htlcs:
+        c_outputs += [(bitcoin.TYPE_ADDRESS, bitcoin.redeem_script_to_address('p2wsh', bh2u(script)), msat_amount // 1000)]
+
+    # trim outputs
+    c_outputs_filtered = list(filter(lambda x:x[2]>= dust_limit_sat, c_outputs))
+    assert sum(x[2] for x in c_outputs) <= funding_sat
+
+    # create commitment tx
+    tx = Transaction.from_io(c_inputs, c_outputs_filtered, locktime=locktime, version=2)
+    tx.BIP_LI01_sort()
+
+    tx.htlc_output_indices = {}
+    for idx, output in enumerate(c_outputs):
+        if output in tx.outputs():
+            # minus the first two outputs (to_local, to_remote)
+            tx.htlc_output_indices[idx - 2] = tx.outputs().index(output)
+
+    return tx
+
+def sign_and_get_sig_string(tx, local_config, remote_config):
+    pubkeys = sorted([bh2u(local_config.multisig_key.pubkey), bh2u(remote_config.multisig_key.pubkey)])
+    tx.sign({bh2u(local_config.multisig_key.pubkey): (local_config.multisig_key.privkey, True)})
+    sig_index = pubkeys.index(bh2u(local_config.multisig_key.pubkey))
+    sig = bytes.fromhex(tx.inputs()[0]["signatures"][sig_index])
+    sig_64 = sig_string_from_der_sig(sig[:-1])
+    return sig_64
+
+def funding_output_script(local_config, remote_config):
+    pubkeys = sorted([bh2u(local_config.multisig_key.pubkey), bh2u(remote_config.multisig_key.pubkey)])
+    return transaction.multisig_script(pubkeys, 2)
+
+def calc_short_channel_id(block_height: int, tx_pos_in_block: int, output_index: int) -> bytes:
+    bh = block_height.to_bytes(3, byteorder='big')
+    tpos = tx_pos_in_block.to_bytes(3, byteorder='big')
+    oi = output_index.to_bytes(2, byteorder='big')
+    return bh + tpos + oi
+
+def get_obscured_ctn(ctn, local, remote):
+    mask = int.from_bytes(sha256(local + remote)[-6:], 'big')
+    return ctn ^ mask
+
+def overall_weight(num_htlc):
+    return 500 + 172 * num_htlc + 224
+
+def get_ecdh(priv: bytes, pub: bytes) -> bytes:
+    pt = ECPubkey(pub) * string_to_number(priv)
+    return sha256(pt.get_public_key_bytes())

lib/lnwatcher.py

@@ -0,0 +1,36 @@
+from .util import PrintError
+from .lnutil import funding_output_script
+from .bitcoin import redeem_script_to_address
+
+class LNWatcher(PrintError):
+
+    def __init__(self, network):
+        self.network = network
+        self.watched_channels = {}
+
+    def parse_response(self, response):
+        if response.get('error'):
+            self.print_error("response error:", response)
+            return None, None
+        return response['params'], response['result']
+
+    def watch_channel(self, chan, callback):
+        script = funding_output_script(chan.local_config, chan.remote_config)
+        funding_address = redeem_script_to_address('p2wsh', script)
+        self.watched_channels[funding_address] = chan, callback
+        self.network.subscribe_to_addresses([funding_address], self.on_address_status)
+
+    def on_address_status(self, response):
+        params, result = self.parse_response(response)
+        if not params:
+            return
+        addr = params[0]
+        self.network.request_address_utxos(addr, self.on_utxos)
+
+    def on_utxos(self, response):
+        params, result = self.parse_response(response)
+        if not params:
+            return
+        addr = params[0]
+        chan, callback = self.watched_channels[addr]
+        callback(chan, result)

lib/lnworker.py

@@ -0,0 +1,190 @@
+import json
+import binascii
+import asyncio
+import os
+from decimal import Decimal
+import threading
+from collections import defaultdict
+
+from . import constants
+from .bitcoin import sha256, COIN
+from .util import bh2u, bfh, PrintError
+from .constants import set_testnet, set_simnet
+from .lnbase import Peer, privkey_to_pubkey
+from .lnaddr import lnencode, LnAddr, lndecode
+from .ecc import der_sig_from_sig_string
+from .transaction import Transaction
+from .lnhtlc import HTLCStateMachine
+from .lnutil import Outpoint, calc_short_channel_id
+
+# hardcoded nodes
+node_list = [
+    ('ecdsa.net', '9735', '038370f0e7a03eded3e1d41dc081084a87f0afa1c5b22090b4f3abb391eb15d8ff'),
+]
+
+class LNWorker(PrintError):
+
+    def __init__(self, wallet, network):
+        self.wallet = wallet
+        self.network = network
+        pk = wallet.storage.get('lightning_privkey')
+        if pk is None:
+            pk = bh2u(os.urandom(32))
+            wallet.storage.put('lightning_privkey', pk)
+            wallet.storage.write()
+        self.privkey = bfh(pk)
+        self.pubkey = privkey_to_pubkey(self.privkey)
+        self.config = network.config
+        self.peers = {}
+        self.channels = {x.channel_id: x for x in map(HTLCStateMachine, wallet.storage.get("channels", []))}
+        self.invoices = wallet.storage.get('lightning_invoices', {})
+        peer_list = network.config.get('lightning_peers', node_list)
+        self.channel_state = {chan.channel_id: "DISCONNECTED" for chan in self.channels.values()}
+        for chan_id, chan in self.channels.items():
+            self.network.lnwatcher.watch_channel(chan, self.on_channel_utxos)
+        for host, port, pubkey in peer_list:
+            self.add_peer(host, int(port), bfh(pubkey))
+        # wait until we see confirmations
+        self.network.register_callback(self.on_network_update, ['updated', 'verified']) # thread safe
+        self.on_network_update('updated') # shortcut (don't block) if funding tx locked and verified
+
+    def channels_for_peer(self, node_id):
+        assert type(node_id) is bytes
+        return {x: y for (x, y) in self.channels.items() if y.node_id == node_id}
+
+    def add_peer(self, host, port, node_id, aiosafe=True):
+        peer = Peer(self, host, int(port), node_id, request_initial_sync=self.config.get("request_initial_sync", True))
+        if not aiosafe:
+            method = peer._main_loop
+        else:
+            method = peer.main_loop
+        coro = asyncio.run_coroutine_threadsafe(method(), asyncio.get_event_loop())
+        self.network.futures.append(coro)
+        self.peers[node_id] = peer
+        return peer, coro
+
+    def save_channel(self, openchannel):
+        assert type(openchannel) is HTLCStateMachine
+        if openchannel.channel_id not in self.channel_state:
+            self.channel_state[openchannel.channel_id] = "OPENING"
+        self.channels[openchannel.channel_id] = openchannel
+        for node_id, peer in self.peers.items():
+            peer.channels = self.channels_for_peer(node_id)
+        if openchannel.remote_state.next_per_commitment_point == openchannel.remote_state.current_per_commitment_point:
+            raise Exception("Tried to save channel with next_point == current_point, this should not happen")
+        dumped = [x.serialize() for x in self.channels.values()]
+        self.wallet.storage.put("channels", dumped)
+        self.wallet.storage.write()
+        self.network.trigger_callback('channel', openchannel)
+
+    def save_short_chan_id(self, chan):
+        """
+        Checks if the Funding TX has been mined. If it has save the short channel ID to disk and return the new OpenChannel.
+
+        If the Funding TX has not been mined, return None
+        """
+        assert self.channel_state[chan.channel_id] in ["OPEN", "OPENING"]
+        peer = self.peers[chan.node_id]
+        conf = self.wallet.get_tx_height(chan.funding_outpoint.txid)[1]
+        if conf >= chan.constraints.funding_txn_minimum_depth:
+            block_height, tx_pos = self.wallet.get_txpos(chan.funding_outpoint.txid)
+            if tx_pos == -1:
+                self.print_error('funding tx is not yet SPV verified.. but there are '
+                                 'already enough confirmations (currently {})'.format(conf))
+                return False
+            chan.short_channel_id = calc_short_channel_id(block_height, tx_pos, chan.funding_outpoint.output_index)
+            self.save_channel(chan)
+            return True
+        return False
+
+    def on_channel_utxos(self, chan, utxos):
+        outpoints = [Outpoint(x["tx_hash"], x["tx_pos"]) for x in utxos]
+        if chan.funding_outpoint not in outpoints:
+            self.channel_state[chan.channel_id] = "CLOSED"
+        elif self.channel_state[chan.channel_id] == 'DISCONNECTED':
+            peer = self.peers[chan.node_id]
+            coro = peer.reestablish_channel(chan)
+            asyncio.run_coroutine_threadsafe(coro, self.network.asyncio_loop)
+
+    def on_network_update(self, event, *args):
+        for chan in self.channels.values():
+            peer = self.peers[chan.node_id]
+            if self.channel_state[chan.channel_id] == "OPENING":
+                res = self.save_short_chan_id(chan)
+                if not res:
+                    self.print_error("network update but funding tx is still not at sufficient depth")
+                    continue
+                # this results in the channel being marked OPEN
+                peer.funding_locked(chan)
+            elif self.channel_state[chan.channel_id] == "OPEN":
+                conf = self.wallet.get_tx_height(chan.funding_outpoint.txid)[1]
+                peer.on_network_update(chan, conf)
+
+    async def _open_channel_coroutine(self, node_id, local_amount_sat, push_sat, password):
+
+        peer = self.peers[node_id]
+        openingchannel = await peer.channel_establishment_flow(self.wallet, self.config, password, local_amount_sat + push_sat, push_sat * 1000, temp_channel_id=os.urandom(32))
+        self.save_channel(openingchannel)
+        self.on_channels_updated()
+
+    def on_channels_updated(self):
+        self.network.trigger_callback('channels')
+
+    def open_channel(self, node_id, local_amt_sat, push_amt_sat, pw):
+        coro = self._open_channel_coroutine(node_id, local_amt_sat, push_amt_sat, None if pw == "" else pw)
+        return asyncio.run_coroutine_threadsafe(coro, self.network.asyncio_loop)
+
+    def pay(self, invoice):
+        addr = lndecode(invoice, expected_hrp=constants.net.SEGWIT_HRP)
+        payment_hash = addr.paymenthash
+        invoice_pubkey = addr.pubkey.serialize()
+        amount_msat = int(addr.amount * COIN * 1000)
+        path = self.network.path_finder.find_path_for_payment(self.pubkey, invoice_pubkey, amount_msat)
+        if path is None:
+            raise Exception("No path found")
+        node_id, short_channel_id = path[0]
+        peer = self.peers[node_id]
+        for chan in self.channels.values():
+            if chan.short_channel_id == short_channel_id:
+                break
+        else:
+            raise Exception("ChannelDB returned path with short_channel_id that is not in channel list")
+        coro = peer.pay(path, chan, amount_msat, payment_hash, invoice_pubkey, addr.min_final_cltv_expiry)
+        return asyncio.run_coroutine_threadsafe(coro, self.network.asyncio_loop)
+
+    def add_invoice(self, amount_sat, message):
+        payment_preimage = os.urandom(32)
+        RHASH = sha256(payment_preimage)
+        amount_btc = amount_sat/Decimal(COIN) if amount_sat else None
+        pay_req = lnencode(LnAddr(RHASH, amount_btc, tags=[('d', message)]), self.privkey)
+        self.invoices[bh2u(payment_preimage)] = pay_req
+        self.wallet.storage.put('lightning_invoices', self.invoices)
+        self.wallet.storage.write()
+        return pay_req
+
+    def delete_invoice(self, payreq_key):
+        try:
+            del self.invoices[payreq_key]
+        except KeyError:
+            return
+        self.wallet.storage.put('lightning_invoices', self.invoices)
+        self.wallet.storage.write()
+
+    def list_channels(self):
+        return [str(x) for x in self.channels]
+
+    def close_channel(self, chan_id):
+        chan = self.channels[chan_id]
+        # local_commitment always gives back the next expected local_commitment,
+        # but in this case, we want the current one. So substract one ctn number
+        old_local_state = chan.local_state
+        chan.local_state=chan.local_state._replace(ctn=chan.local_state.ctn - 1)
+        tx = chan.local_commitment
+        chan.local_state = old_local_state
+        tx.sign({bh2u(chan.local_config.multisig_key.pubkey): (chan.local_config.multisig_key.privkey, True)})
+        remote_sig = chan.local_state.current_commitment_signature
+        remote_sig = der_sig_from_sig_string(remote_sig) + b"\x01"
+        none_idx = tx._inputs[0]["signatures"].index(None)
+        tx.add_signature_to_txin(0, none_idx, bh2u(remote_sig))
+        assert tx.is_complete()
+        return self.network.broadcast_transaction(tx)

lib/network.py

@@ -20,6 +20,7 @@
 # ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
+import asyncio
 import time
 import queue
 import os
@@ -27,7 +28,7 @@ import errno
 import random
 import re
 import select
-from collections import defaultdict
+from collections import defaultdict, deque
 import threading
 import socket
 import json
@@ -46,6 +47,10 @@ from . import blockchain
 from .version import ELECTRUM_VERSION, PROTOCOL_VERSION
 from .i18n import _
 
+# lightning network
+from . import lnwatcher
+from . import lnrouter
+
 
 NODES_RETRY_INTERVAL = 60
 SERVER_RETRY_INTERVAL = 10
@@ -151,7 +156,6 @@ def deserialize_server(server_str):
 def serialize_server(host, port, protocol):
     return str(':'.join([host, port, protocol]))
 
-
 class Network(util.DaemonThread):
     """The Network class manages a set of connections to remote electrum
     servers, each connected socket is handled by an Interface() object.
@@ -233,6 +237,13 @@ class Network(util.DaemonThread):
         self.socket_queue = queue.Queue()
         self.start_network(deserialize_server(self.default_server)[2],
                            deserialize_proxy(self.config.get('proxy')))
+        self.asyncio_loop = loop = asyncio.new_event_loop()
+        self.futures = []
+        # lightning network
+        self.lightning_nodes = {}
+        self.channel_db = lnrouter.ChannelDB()
+        self.path_finder = lnrouter.LNPathFinder(self.channel_db)
+        self.lnwatcher = lnwatcher.LNWatcher(self)
 
     def with_interface_lock(func):
         def func_wrapper(self, *args, **kwargs):
@@ -1030,13 +1041,28 @@ class Network(util.DaemonThread):
             b.update_size()
 
     def run(self):
+        asyncio.set_event_loop(self.asyncio_loop)
         self.init_headers_file()
+        def asyncioThread():
+            self.asyncio_loop.run_forever()
+        threading.Thread(target=asyncioThread).start()
         while self.is_running():
             self.maintain_sockets()
             self.wait_on_sockets()
             self.maintain_requests()
             self.run_jobs()    # Synchronizer and Verifier
             self.process_pending_sends()
+        # cancel tasks
+        [f.cancel() for f in self.futures]
+        async def loopstop():
+            self.asyncio_loop.stop()
+        asyncio.run_coroutine_threadsafe(loopstop(), self.asyncio_loop)
+        while self.asyncio_loop.is_running():
+            time.sleep(0.1)
+        try:
+            self.asyncio_loop.close()
+        except:
+            pass
         self.stop_network()
         self.on_stop()
 
@@ -1178,6 +1204,11 @@ class Network(util.DaemonThread):
         self.h2addr.update({h: address})
         self.send([('blockchain.scripthash.get_history', [h])], self.map_scripthash_to_address(callback))
 
+    def request_address_utxos(self, address, callback):
+        h = bitcoin.address_to_scripthash(address)
+        self.h2addr.update({h: address})
+        self.send([('blockchain.scripthash.listunspent', [h])], self.map_scripthash_to_address(callback))
+
     # NOTE this method handles exceptions and a special edge case, counter to
     # what the other ElectrumX methods do. This is unexpected.
     def broadcast_transaction(self, transaction, callback=None):

lib/segwit_addr.py

@@ -59,14 +59,14 @@ def bech32_encode(hrp, data):
     return hrp + '1' + ''.join([CHARSET[d] for d in combined])
 
 
-def bech32_decode(bech):
+def bech32_decode(bech, ignore_long_length=False):
     """Validate a Bech32 string, and determine HRP and data."""
     if ((any(ord(x) < 33 or ord(x) > 126 for x in bech)) or
             (bech.lower() != bech and bech.upper() != bech)):
         return (None, None)
     bech = bech.lower()
     pos = bech.rfind('1')
-    if pos < 1 or pos + 7 > len(bech) or len(bech) > 90:
+    if pos < 1 or pos + 7 > len(bech) or (not ignore_long_length and len(bech) > 90):
         return (None, None)
     if not all(x in CHARSET for x in bech[pos+1:]):
         return (None, None)

lib/simple_config.py

@@ -109,13 +109,11 @@ class SimpleConfig(PrintError):
         make_dir(path, allow_symlink=False)
         if self.get('testnet'):
             path = os.path.join(path, 'testnet')
-            make_dir(path, allow_symlink=False)
         elif self.get('regtest'):
             path = os.path.join(path, 'regtest')
-            make_dir(path, allow_symlink=False)
         elif self.get('simnet'):
             path = os.path.join(path, 'simnet')
-            make_dir(path, allow_symlink=False)
+        make_dir(path, allow_symlink=False)
 
         self.print_error("electrum directory", path)
         return path

lib/tests/test_bolt11.py

@@ -0,0 +1,97 @@
+from hashlib import sha256
+from lib.lightning_payencode.lnaddr import shorten_amount, unshorten_amount, LnAddr, lnencode, lndecode, u5_to_bitarray, bitarray_to_u5
+from decimal import Decimal
+from binascii import unhexlify, hexlify
+from lib.segwit_addr import bech32_encode, bech32_decode
+import pprint
+import unittest
+
+RHASH=unhexlify('0001020304050607080900010203040506070809000102030405060708090102')
+CONVERSION_RATE=1200
+PRIVKEY=unhexlify('e126f68f7eafcc8b74f54d269fe206be715000f94dac067d1c04a8ca3b2db734')
+PUBKEY=unhexlify('03e7156ae33b0a208d0744199163177e909e80176e55d97a2f221ede0f934dd9ad')
+
+class TestBolt11(unittest.TestCase):
+    def test_shorten_amount(self):
+        tests = {
+            Decimal(10)/10**12: '10p',
+            Decimal(1000)/10**12: '1n',
+            Decimal(1200)/10**12: '1200p',
+            Decimal(123)/10**6: '123u',
+            Decimal(123)/1000: '123m',
+            Decimal(3): '3',
+        }
+
+        for i, o in tests.items():
+            assert shorten_amount(i) == o
+            assert unshorten_amount(shorten_amount(i)) == i
+
+    @staticmethod
+    def compare(a, b):
+
+        if len([t[1] for t in a.tags if t[0] == 'h']) == 1:
+            h1 = sha256([t[1] for t in a.tags if t[0] == 'h'][0].encode('utf-8')).digest()
+            h2 = [t[1] for t in b.tags if t[0] == 'h'][0]
+            assert h1 == h2
+
+        # Need to filter out these, since they are being modified during
+        # encoding, i.e., hashed
+        a.tags = [t for t in a.tags if t[0] != 'h' and t[0] != 'n']
+        b.tags = [t for t in b.tags if t[0] != 'h' and t[0] != 'n']
+
+        assert b.pubkey.serialize() == PUBKEY, (hexlify(b.pubkey.serialize()), hexlify(PUBKEY))
+        assert b.signature != None
+
+        # Unset these, they are generated during encoding/decoding
+        b.pubkey = None
+        b.signature = None
+
+        assert a.__dict__ == b.__dict__, (pprint.pformat([a.__dict__, b.__dict__]))
+
+    def test_roundtrip(self):
+        longdescription = ('One piece of chocolate cake, one icecream cone, one'
+                          ' pickle, one slice of swiss cheese, one slice of salami,'
+                          ' one lollypop, one piece of cherry pie, one sausage, one'
+                          ' cupcake, and one slice of watermelon')
+
+
+        tests = [
+            LnAddr(RHASH, tags=[('d', '')]),
+            LnAddr(RHASH, amount=Decimal('0.001'),
+                   tags=[('d', '1 cup coffee'), ('x', 60)]),
+            LnAddr(RHASH, amount=Decimal('1'), tags=[('h', longdescription)]),
+            LnAddr(RHASH, currency='tb', tags=[('f', 'mk2QpYatsKicvFVuTAQLBryyccRXMUaGHP'), ('h', longdescription)]),
+            LnAddr(RHASH, amount=24, tags=[
+                ('r', [(unhexlify('029e03a901b85534ff1e92c43c74431f7ce72046060fcf7a95c37e148f78c77255'), unhexlify('0102030405060708'), 1, 20, 3), (unhexlify('039e03a901b85534ff1e92c43c74431f7ce72046060fcf7a95c37e148f78c77255'), unhexlify('030405060708090a'), 2, 30, 4)]), ('f', '1RustyRX2oai4EYYDpQGWvEL62BBGqN9T'), ('h', longdescription)]),
+            LnAddr(RHASH, amount=24, tags=[('f', '3EktnHQD7RiAE6uzMj2ZifT9YgRrkSgzQX'), ('h', longdescription)]),
+            LnAddr(RHASH, amount=24, tags=[('f', 'bc1qw508d6qejxtdg4y5r3zarvary0c5xw7kv8f3t4'), ('h', longdescription)]),
+            LnAddr(RHASH, amount=24, tags=[('f', 'bc1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3qccfmv3'), ('h', longdescription)]),
+            LnAddr(RHASH, amount=24, tags=[('n', PUBKEY), ('h', longdescription)]),
+        ]
+
+        # Roundtrip
+        for t in tests:
+            o = lndecode(lnencode(t, PRIVKEY), False, t.currency)
+            self.compare(t, o)
+
+    def test_n_decoding(self):
+        # We flip the signature recovery bit, which would normally give a different
+        # pubkey.
+        hrp, data = bech32_decode(lnencode(LnAddr(RHASH, amount=24, tags=[('d', '')]), PRIVKEY), True)
+        databits = u5_to_bitarray(data)
+        databits.invert(-1)
+        lnaddr = lndecode(bech32_encode(hrp, bitarray_to_u5(databits)), True)
+        assert lnaddr.pubkey.serialize() != PUBKEY
+
+        # But not if we supply expliciy `n` specifier!
+        hrp, data = bech32_decode(lnencode(LnAddr(RHASH, amount=24,
+                                                  tags=[('d', ''),
+                                                        ('n', PUBKEY)]),
+                                           PRIVKEY), True)
+        databits = u5_to_bitarray(data)
+        databits.invert(-1)
+        lnaddr = lndecode(bech32_encode(hrp, bitarray_to_u5(databits)), True)
+        assert lnaddr.pubkey.serialize() == PUBKEY
+
+    def test_min_final_cltv_expiry(self):
+        self.assertEquals(lndecode("lnsb500u1pdsgyf3pp5nmrqejdsdgs4n9ukgxcp2kcq265yhrxd4k5dyue58rxtp5y83s3qdqqcqzystrggccm9yvkr5yqx83jxll0qjpmgfg9ywmcd8g33msfgmqgyfyvqhku80qmqm8q6v35zvck2y5ccxsz5avtrauz8hgjj3uahppyq20qp6dvwxe", expected_hrp="sb").min_final_cltv_expiry, 144)

lib/tests/test_lnhtlc.py

@@ -0,0 +1,307 @@
+# ported from lnd 42de4400bff5105352d0552155f73589166d162b
+
+import unittest
+import lib.bitcoin as bitcoin
+import lib.lnbase as lnbase
+import lib.lnhtlc as lnhtlc
+import lib.lnutil as lnutil
+import lib.util as util
+import os
+import binascii
+
+def create_channel_state(funding_txid, funding_index, funding_sat, local_feerate, is_initiator, local_amount, remote_amount, privkeys, other_pubkeys, seed, cur, nex, other_node_id, l_dust, r_dust, l_csv, r_csv):
+    assert local_amount > 0
+    assert remote_amount > 0
+    channel_id, _ = lnbase.channel_id_from_funding_tx(funding_txid, funding_index)
+    their_revocation_store = lnbase.RevocationStore()
+    local_config=lnbase.ChannelConfig(
+        payment_basepoint=privkeys[0],
+        multisig_key=privkeys[1],
+        htlc_basepoint=privkeys[2],
+        delayed_basepoint=privkeys[3],
+        revocation_basepoint=privkeys[4],
+        to_self_delay=l_csv,
+        dust_limit_sat=l_dust,
+        max_htlc_value_in_flight_msat=500000 * 1000,
+        max_accepted_htlcs=5
+    )
+    remote_config=lnbase.ChannelConfig(
+        payment_basepoint=other_pubkeys[0],
+        multisig_key=other_pubkeys[1],
+        htlc_basepoint=other_pubkeys[2],
+        delayed_basepoint=other_pubkeys[3],
+        revocation_basepoint=other_pubkeys[4],
+        to_self_delay=r_csv,
+        dust_limit_sat=r_dust,
+        max_htlc_value_in_flight_msat=500000 * 1000,
+        max_accepted_htlcs=5
+    )
+
+    return {
+            "channel_id":channel_id,
+            "short_channel_id":channel_id[:8],
+            "funding_outpoint":lnbase.Outpoint(funding_txid, funding_index),
+            "local_config":local_config,
+            "remote_config":remote_config,
+            "remote_state":lnbase.RemoteState(
+                ctn = 0,
+                next_per_commitment_point=nex,
+                current_per_commitment_point=cur,
+                amount_msat=remote_amount,
+                revocation_store=their_revocation_store,
+                next_htlc_id = 0
+            ),
+            "local_state":lnbase.LocalState(
+                ctn = 0,
+                per_commitment_secret_seed=seed,
+                amount_msat=local_amount,
+                next_htlc_id = 0,
+                funding_locked_received=True,
+                was_announced=False,
+                current_commitment_signature=None
+            ),
+            "constraints":lnbase.ChannelConstraints(capacity=funding_sat, feerate=local_feerate, is_initiator=is_initiator, funding_txn_minimum_depth=3),
+            "node_id":other_node_id
+    }
+
+def bip32(sequence):
+    xprv, xpub = bitcoin.bip32_root(b"9dk", 'standard')
+    xprv, xpub = bitcoin.bip32_private_derivation(xprv, "m/", sequence)
+    xtype, depth, fingerprint, child_number, c, k = bitcoin.deserialize_xprv(xprv)
+    assert len(k) == 32
+    assert type(k) is bytes
+    return k
+
+def create_test_channels():
+    funding_txid = binascii.hexlify(os.urandom(32)).decode("ascii")
+    funding_index = 0
+    funding_sat = bitcoin.COIN * 10
+    local_amount = (funding_sat * 1000) // 2
+    remote_amount = (funding_sat * 1000) // 2
+    alice_raw = [ bip32("m/" + str(i)) for i in range(5) ]
+    bob_raw = [ bip32("m/" + str(i)) for i in range(5,11) ]
+    alice_privkeys = [lnbase.Keypair(lnbase.privkey_to_pubkey(x), x) for x in alice_raw]
+    bob_privkeys = [lnbase.Keypair(lnbase.privkey_to_pubkey(x), x) for x in bob_raw]
+    alice_pubkeys = [lnbase.OnlyPubkeyKeypair(x.pubkey) for x in alice_privkeys]
+    bob_pubkeys = [lnbase.OnlyPubkeyKeypair(x.pubkey) for x in bob_privkeys]
+
+    alice_seed = os.urandom(32)
+    bob_seed = os.urandom(32)
+
+    alice_cur = lnutil.secret_to_pubkey(int.from_bytes(lnutil.get_per_commitment_secret_from_seed(alice_seed, 2**48 - 1), "big"))
+    alice_next = lnutil.secret_to_pubkey(int.from_bytes(lnutil.get_per_commitment_secret_from_seed(alice_seed, 2**48 - 2), "big"))
+    bob_cur = lnutil.secret_to_pubkey(int.from_bytes(lnutil.get_per_commitment_secret_from_seed(bob_seed, 2**48 - 1), "big"))
+    bob_next = lnutil.secret_to_pubkey(int.from_bytes(lnutil.get_per_commitment_secret_from_seed(bob_seed, 2**48 - 2), "big"))
+
+    return \
+        lnhtlc.HTLCStateMachine(
+            create_channel_state(funding_txid, funding_index, funding_sat, 6000, True, local_amount, remote_amount, alice_privkeys, bob_pubkeys, alice_seed, bob_cur, bob_next, b"\x02"*33, l_dust=200, r_dust=1300, l_csv=5, r_csv=4), "alice"), \
+        lnhtlc.HTLCStateMachine(
+            create_channel_state(funding_txid, funding_index, funding_sat, 6000, False, remote_amount, local_amount, bob_privkeys, alice_pubkeys, bob_seed, alice_cur, alice_next, b"\x01"*33, l_dust=1300, r_dust=200, l_csv=4, r_csv=5), "bob")
+
+one_bitcoin_in_msat = bitcoin.COIN * 1000
+
+class TestLNBaseHTLCStateMachine(unittest.TestCase):
+    def assertOutputExistsByValue(self, tx, amt_sat):
+        for typ, scr, val in tx.outputs():
+            if val == amt_sat:
+                break
+        else:
+            self.assertFalse()
+
+    def test_SimpleAddSettleWorkflow(self):
+
+        # Create a test channel which will be used for the duration of this
+        # unittest. The channel will be funded evenly with Alice having 5 BTC,
+        # and Bob having 5 BTC.
+        alice_channel, bob_channel = create_test_channels()
+
+        paymentPreimage = b"\x01" * 32
+        paymentHash = bitcoin.sha256(paymentPreimage)
+        htlcAmt = one_bitcoin_in_msat
+        htlc = lnhtlc.UpdateAddHtlc(
+            payment_hash = paymentHash,
+            amount_msat =  htlcAmt,
+            cltv_expiry =  5,
+            total_fee = 0
+        )
+
+        # First Alice adds the outgoing HTLC to her local channel's state
+        # update log. Then Alice sends this wire message over to Bob who adds
+        # this htlc to his remote state update log.
+        aliceHtlcIndex = alice_channel.add_htlc(htlc)
+
+        bobHtlcIndex = bob_channel.receive_htlc(htlc)
+
+        # Next alice commits this change by sending a signature message. Since
+        # we expect the messages to be ordered, Bob will receive the HTLC we
+        # just sent before he receives this signature, so the signature will
+        # cover the HTLC.
+        aliceSig, aliceHtlcSigs = alice_channel.sign_next_commitment()
+
+        self.assertEqual(len(aliceHtlcSigs), 1, "alice should generate one htlc signature")
+
+        # Bob receives this signature message, and checks that this covers the
+        # state he has in his remote log. This includes the HTLC just sent
+        # from Alice.
+        bob_channel.receive_new_commitment(aliceSig, aliceHtlcSigs)
+
+        # Bob revokes his prior commitment given to him by Alice, since he now
+        # has a valid signature for a newer commitment.
+        bobRevocation, _ = bob_channel.revoke_current_commitment()
+
+        # Bob finally send a signature for Alice's commitment transaction.
+        # This signature will cover the HTLC, since Bob will first send the
+        # revocation just created. The revocation also acks every received
+        # HTLC up to the point where Alice sent here signature.
+        bobSig, bobHtlcSigs = bob_channel.sign_next_commitment()
+
+        # Alice then processes this revocation, sending her own revocation for
+        # her prior commitment transaction. Alice shouldn't have any HTLCs to
+        # forward since she's sending an outgoing HTLC.
+        alice_channel.receive_revocation(bobRevocation)
+
+        # Alice then processes bob's signature, and since she just received
+        # the revocation, she expect this signature to cover everything up to
+        # the point where she sent her signature, including the HTLC.
+        alice_channel.receive_new_commitment(bobSig, bobHtlcSigs)
+
+        # Alice then generates a revocation for bob.
+        aliceRevocation, _ = alice_channel.revoke_current_commitment()
+
+        # Finally Bob processes Alice's revocation, at this point the new HTLC
+        # is fully locked in within both commitment transactions. Bob should
+        # also be able to forward an HTLC now that the HTLC has been locked
+        # into both commitment transactions.
+        bob_channel.receive_revocation(aliceRevocation)
+
+        # At this point, both sides should have the proper number of satoshis
+        # sent, and commitment height updated within their local channel
+        # state.
+        aliceSent = 0
+        bobSent = 0
+
+        self.assertEqual(alice_channel.total_msat_sent, aliceSent, "alice has incorrect milli-satoshis sent")
+        self.assertEqual(alice_channel.total_msat_received, bobSent, "alice has incorrect milli-satoshis received")
+        self.assertEqual(bob_channel.total_msat_sent, bobSent, "bob has incorrect milli-satoshis sent")
+        self.assertEqual(bob_channel.total_msat_received, aliceSent, "bob has incorrect milli-satoshis received")
+        self.assertEqual(bob_channel.local_state.ctn, 1, "bob has incorrect commitment height")
+        self.assertEqual(alice_channel.local_state.ctn, 1, "alice has incorrect commitment height")
+
+        # Both commitment transactions should have three outputs, and one of
+        # them should be exactly the amount of the HTLC.
+        self.assertEqual(len(alice_channel.local_commitment.outputs()), 3, "alice should have three commitment outputs, instead have %s"% len(alice_channel.local_commitment.outputs()))
+        self.assertEqual(len(bob_channel.local_commitment.outputs()), 3, "bob should have three commitment outputs, instead have %s"% len(bob_channel.local_commitment.outputs()))
+        self.assertOutputExistsByValue(alice_channel.local_commitment, htlcAmt // 1000)
+        self.assertOutputExistsByValue(bob_channel.local_commitment, htlcAmt // 1000)
+
+        # Now we'll repeat a similar exchange, this time with Bob settling the
+        # HTLC once he learns of the preimage.
+        preimage = paymentPreimage
+        bob_channel.settle_htlc(preimage, bobHtlcIndex)
+
+        alice_channel.receive_htlc_settle(preimage, aliceHtlcIndex)
+
+        bobSig2, bobHtlcSigs2 = bob_channel.sign_next_commitment()
+        alice_channel.receive_new_commitment(bobSig2, bobHtlcSigs2)
+
+        aliceRevocation2, _ = alice_channel.revoke_current_commitment()
+        aliceSig2, aliceHtlcSigs2 = alice_channel.sign_next_commitment()
+        self.assertEqual(aliceHtlcSigs2, [], "alice should generate no htlc signatures")
+
+        bob_channel.receive_revocation(aliceRevocation2)
+
+        bob_channel.receive_new_commitment(aliceSig2, aliceHtlcSigs2)
+
+        bobRevocation2, _ = bob_channel.revoke_current_commitment()
+        alice_channel.receive_revocation(bobRevocation2)
+
+        # At this point, Bob should have 6 BTC settled, with Alice still having
+        # 4 BTC. Alice's channel should show 1 BTC sent and Bob's channel
+        # should show 1 BTC received. They should also be at commitment height
+        # two, with the revocation window extended by 1 (5).
+        mSatTransferred = one_bitcoin_in_msat
+        self.assertEqual(alice_channel.total_msat_sent, mSatTransferred, "alice satoshis sent incorrect %s vs %s expected"% (alice_channel.total_msat_sent, mSatTransferred))
+        self.assertEqual(alice_channel.total_msat_received, 0, "alice satoshis received incorrect %s vs %s expected"% (alice_channel.total_msat_received, 0))
+        self.assertEqual(bob_channel.total_msat_received, mSatTransferred, "bob satoshis received incorrect %s vs %s expected"% (bob_channel.total_msat_received, mSatTransferred))
+        self.assertEqual(bob_channel.total_msat_sent, 0, "bob satoshis sent incorrect %s vs %s expected"% (bob_channel.total_msat_sent, 0))
+        self.assertEqual(bob_channel.l_current_height, 2, "bob has incorrect commitment height, %s vs %s"% (bob_channel.l_current_height, 2))
+        self.assertEqual(alice_channel.l_current_height, 2, "alice has incorrect commitment height, %s vs %s"% (alice_channel.l_current_height, 2))
+
+        # The logs of both sides should now be cleared since the entry adding
+        # the HTLC should have been removed once both sides receive the
+        # revocation.
+        self.assertEqual(alice_channel.local_update_log, [], "alice's local not updated, should be empty, has %s entries instead"% len(alice_channel.local_update_log))
+        self.assertEqual(alice_channel.remote_update_log, [], "alice's remote not updated, should be empty, has %s entries instead"% len(alice_channel.remote_update_log))
+
+    def test_HTLCDustLimit(self):
+        alice_channel, bob_channel = create_test_channels()
+
+        paymentPreimage = b"\x01" * 32
+        paymentHash = bitcoin.sha256(paymentPreimage)
+        fee_per_kw = alice_channel.constraints.feerate
+        self.assertEqual(fee_per_kw, 6000)
+        htlcAmt = 500 + lnutil.HTLC_TIMEOUT_WEIGHT * (fee_per_kw // 1000)
+        self.assertEqual(htlcAmt, 4478)
+        htlc = lnhtlc.UpdateAddHtlc(
+            payment_hash = paymentHash,
+            amount_msat =  1000 * htlcAmt,
+            cltv_expiry =  5, # also in create_test_channels
+            total_fee = 0
+        )
+
+        aliceHtlcIndex = alice_channel.add_htlc(htlc)
+        bobHtlcIndex = bob_channel.receive_htlc(htlc)
+        force_state_transition(alice_channel, bob_channel)
+        self.assertEqual(len(alice_channel.local_commitment.outputs()), 3)
+        self.assertEqual(len(bob_channel.local_commitment.outputs()), 2)
+        default_fee = calc_static_fee(0)
+        self.assertEqual(bob_channel.local_commit_fee, default_fee)
+        bob_channel.settle_htlc(paymentPreimage, htlc.htlc_id)
+        alice_channel.receive_htlc_settle(paymentPreimage, aliceHtlcIndex)
+        force_state_transition(bob_channel, alice_channel)
+        self.assertEqual(len(alice_channel.local_commitment.outputs()), 2)
+        self.assertEqual(alice_channel.total_msat_sent // 1000, htlcAmt)
+
+    def test_UpdateFeeSenderCommits(self):
+        alice_channel, bob_channel = create_test_channels()
+
+        paymentPreimage = b"\x01" * 32
+        paymentHash = bitcoin.sha256(paymentPreimage)
+        htlc = lnhtlc.UpdateAddHtlc(
+            payment_hash = paymentHash,
+            amount_msat =  one_bitcoin_in_msat,
+            cltv_expiry =  5, # also in create_test_channels
+            total_fee = 0
+        )
+
+        aliceHtlcIndex = alice_channel.add_htlc(htlc)
+        bobHtlcIndex = bob_channel.receive_htlc(htlc)
+
+        fee = 111
+        alice_channel.update_fee(fee)
+        bob_channel.receive_update_fee(fee)
+
+        alice_sig, alice_htlc_sigs = alice_channel.sign_next_commitment()
+        bob_channel.receive_new_commitment(alice_sig, alice_htlc_sigs)
+        self.assertNotEqual(fee, alice_channel.constraints.feerate)
+        rev, _ = alice_channel.revoke_current_commitment()
+        self.assertEqual(fee, alice_channel.constraints.feerate)
+        bob_channel.receive_revocation(rev)
+
+def force_state_transition(chanA, chanB):
+    chanB.receive_new_commitment(*chanA.sign_next_commitment())
+    rev, _ = chanB.revoke_current_commitment()
+    bob_sig, bob_htlc_sigs = chanB.sign_next_commitment()
+    chanA.receive_revocation(rev)
+    chanA.receive_new_commitment(bob_sig, bob_htlc_sigs)
+    chanB.receive_revocation(chanA.revoke_current_commitment()[0])
+
+# calcStaticFee calculates appropriate fees for commitment transactions.  This
+# function provides a simple way to allow test balance assertions to take fee
+# calculations into account.
+def calc_static_fee(numHTLCs):
+  commitWeight = 724
+  htlcWeight   = 172
+  feePerKw     = 24//4 * 1000
+  return feePerKw * (commitWeight + htlcWeight*numHTLCs) // 1000

lib/tests/test_lnrouter.py

@@ -0,0 +1,150 @@
+import unittest
+
+from lib.util import bh2u, bfh
+from lib.lnbase import Peer
+from lib.lnrouter import OnionHopsDataSingle, new_onion_packet, OnionPerHop
+from lib import bitcoin, lnrouter
+
+class Test_LNRouter(unittest.TestCase):
+
+    #@staticmethod
+    #def parse_witness_list(witness_bytes):
+    #    amount_witnesses = witness_bytes[0]
+    #    witness_bytes = witness_bytes[1:]
+    #    res = []
+    #    for i in range(amount_witnesses):
+    #        witness_length = witness_bytes[0]
+    #        this_witness = witness_bytes[1:witness_length+1]
+    #        assert len(this_witness) == witness_length
+    #        witness_bytes = witness_bytes[witness_length+1:]
+    #        res += [bytes(this_witness)]
+    #    assert witness_bytes == b"", witness_bytes
+    #    return res
+
+
+
+    def test_find_path_for_payment(self):
+        class fake_network:
+            channel_db = lnrouter.ChannelDB()
+        class fake_ln_worker:
+            path_finder = lnrouter.LNPathFinder(fake_network.channel_db)
+            privkey = bitcoin.sha256('privkeyseed')
+            network = fake_network
+            channel_state = {}
+            channels = []
+            invoices = {}
+        p = Peer(fake_ln_worker, '', 0, 'a')
+        p.on_channel_announcement({'node_id_1': b'b', 'node_id_2': b'c', 'short_channel_id': bfh('0000000000000001')})
+        p.on_channel_announcement({'node_id_1': b'b', 'node_id_2': b'e', 'short_channel_id': bfh('0000000000000002')})
+        p.on_channel_announcement({'node_id_1': b'a', 'node_id_2': b'b', 'short_channel_id': bfh('0000000000000003')})
+        p.on_channel_announcement({'node_id_1': b'c', 'node_id_2': b'd', 'short_channel_id': bfh('0000000000000004')})
+        p.on_channel_announcement({'node_id_1': b'd', 'node_id_2': b'e', 'short_channel_id': bfh('0000000000000005')})
+        p.on_channel_announcement({'node_id_1': b'a', 'node_id_2': b'd', 'short_channel_id': bfh('0000000000000006')})
+        o = lambda i: i.to_bytes(8, "big")
+        p.on_channel_update({'short_channel_id': bfh('0000000000000001'), 'flags': b'\x00', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150)})
+        p.on_channel_update({'short_channel_id': bfh('0000000000000001'), 'flags': b'\x01', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150)})
+        p.on_channel_update({'short_channel_id': bfh('0000000000000002'), 'flags': b'\x00', 'cltv_expiry_delta': o(99), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150)})
+        p.on_channel_update({'short_channel_id': bfh('0000000000000002'), 'flags': b'\x01', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150)})
+        p.on_channel_update({'short_channel_id': bfh('0000000000000003'), 'flags': b'\x01', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150)})
+        p.on_channel_update({'short_channel_id': bfh('0000000000000003'), 'flags': b'\x00', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150)})
+        p.on_channel_update({'short_channel_id': bfh('0000000000000004'), 'flags': b'\x01', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150)})
+        p.on_channel_update({'short_channel_id': bfh('0000000000000004'), 'flags': b'\x00', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150)})
+        p.on_channel_update({'short_channel_id': bfh('0000000000000005'), 'flags': b'\x01', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150)})
+        p.on_channel_update({'short_channel_id': bfh('0000000000000005'), 'flags': b'\x00', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(999)})
+        p.on_channel_update({'short_channel_id': bfh('0000000000000006'), 'flags': b'\x00', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(99999999)})
+        p.on_channel_update({'short_channel_id': bfh('0000000000000006'), 'flags': b'\x01', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150)})
+        self.assertNotEqual(None, fake_ln_worker.path_finder.find_path_for_payment(b'a', b'e', 100000))
+
+
+
+    def test_new_onion_packet(self):
+        # test vector from bolt-04
+        payment_path_pubkeys = [
+            bfh('02eec7245d6b7d2ccb30380bfbe2a3648cd7a942653f5aa340edcea1f283686619'),
+            bfh('0324653eac434488002cc06bbfb7f10fe18991e35f9fe4302dbea6d2353dc0ab1c'),
+            bfh('027f31ebc5462c1fdce1b737ecff52d37d75dea43ce11c74d25aa297165faa2007'),
+            bfh('032c0b7cf95324a07d05398b240174dc0c2be444d96b159aa6c7f7b1e668680991'),
+            bfh('02edabbd16b41c8371b92ef2f04c1185b4f03b6dcd52ba9b78d9d7c89c8f221145'),
+        ]
+        session_key = bfh('4141414141414141414141414141414141414141414141414141414141414141')
+        associated_data = bfh('4242424242424242424242424242424242424242424242424242424242424242')
+        hops_data = [
+            OnionHopsDataSingle(OnionPerHop(
+                bfh('0000000000000000'), bfh('0000000000000000'), bfh('00000000')
+            )),
+            OnionHopsDataSingle(OnionPerHop(
+                bfh('0101010101010101'), bfh('0000000000000001'), bfh('00000001')
+            )),
+            OnionHopsDataSingle(OnionPerHop(
+                bfh('0202020202020202'), bfh('0000000000000002'), bfh('00000002')
+            )),
+            OnionHopsDataSingle(OnionPerHop(
+                bfh('0303030303030303'), bfh('0000000000000003'), bfh('00000003')
+            )),
+            OnionHopsDataSingle(OnionPerHop(
+                bfh('0404040404040404'), bfh('0000000000000004'), bfh('00000004')
+            )),
+        ]
+        packet = new_onion_packet(payment_path_pubkeys, session_key, hops_data, associated_data)
+        self.assertEqual(bfh('0002eec7245d6b7d2ccb30380bfbe2a3648cd7a942653f5aa340edcea1f283686619e5f14350c2a76fc232b5e46d421e9615471ab9e0bc887beff8c95fdb878f7b3a71da571226458c510bbadd1276f045c21c520a07d35da256ef75b4367962437b0dd10f7d61ab590531cf08000178a333a347f8b4072e216400406bdf3bf038659793a86cae5f52d32f3438527b47a1cfc54285a8afec3a4c9f3323db0c946f5d4cb2ce721caad69320c3a469a202f3e468c67eaf7a7cda226d0fd32f7b48084dca885d15222e60826d5d971f64172d98e0760154400958f00e86697aa1aa9d41bee8119a1ec866abe044a9ad635778ba61fc0776dc832b39451bd5d35072d2269cf9b040d6ba38b54ec35f81d7fc67678c3be47274f3c4cc472aff005c3469eb3bc140769ed4c7f0218ff8c6c7dd7221d189c65b3b9aaa71a01484b122846c7c7b57e02e679ea8469b70e14fe4f70fee4d87b910cf144be6fe48eef24da475c0b0bcc6565ae82cd3f4e3b24c76eaa5616c6111343306ab35c1fe5ca4a77c0e314ed7dba39d6f1e0de791719c241a939cc493bea2bae1c1e932679ea94d29084278513c77b899cc98059d06a27d171b0dbdf6bee13ddc4fc17a0c4d2827d488436b57baa167544138ca2e64a11b43ac8a06cd0c2fba2d4d900ed2d9205305e2d7383cc98dacb078133de5f6fb6bed2ef26ba92cea28aafc3b9948dd9ae5559e8bd6920b8cea462aa445ca6a95e0e7ba52961b181c79e73bd581821df2b10173727a810c92b83b5ba4a0403eb710d2ca10689a35bec6c3a708e9e92f7d78ff3c5d9989574b00c6736f84c199256e76e19e78f0c98a9d580b4a658c84fc8f2096c2fbea8f5f8c59d0fdacb3be2802ef802abbecb3aba4acaac69a0e965abd8981e9896b1f6ef9d60f7a164b371af869fd0e48073742825e9434fc54da837e120266d53302954843538ea7c6c3dbfb4ff3b2fdbe244437f2a153ccf7bdb4c92aa08102d4f3cff2ae5ef86fab4653595e6a5837fa2f3e29f27a9cde5966843fb847a4a61f1e76c281fe8bb2b0a181d096100db5a1a5ce7a910238251a43ca556712eaadea167fb4d7d75825e440f3ecd782036d7574df8bceacb397abefc5f5254d2722215c53ff54af8299aaaad642c6d72a14d27882d9bbd539e1cc7a527526ba89b8c037ad09120e98ab042d3e8652b31ae0e478516bfaf88efca9f3676ffe99d2819dcaeb7610a626695f53117665d267d3f7abebd6bbd6733f645c72c389f03855bdf1e4b8075b516569b118233a0f0971d24b83113c0b096f5216a207ca99a7cddc81c130923fe3d91e7508c9ac5f2e914ff5dccab9e558566fa14efb34ac98d878580814b94b73acbfde9072f30b881f7f0fff42d4045d1ace6322d86a97d164aa84d93a60498065cc7c20e636f5862dc81531a88c60305a2e59a985be327a6902e4bed986dbf4a0b50c217af0ea7fdf9ab37f9ea1a1aaa72f54cf40154ea9b269f1a7c09f9f43245109431a175d50e2db0132337baa0ef97eed0fcf20489da36b79a1172faccc2f7ded7c60e00694282d93359c4682135642bc81f433574aa8ef0c97b4ade7ca372c5ffc23c7eddd839bab4e0f14d6df15c9dbeab176bec8b5701cf054eb3072f6dadc98f88819042bf10c407516ee58bce33fbe3b3d86a54255e577db4598e30a135361528c101683a5fcde7e8ba53f3456254be8f45fe3a56120ae96ea3773631fcb3873aa3abd91bcff00bd38bd43697a2e789e00da6077482e7b1b1a677b5afae4c54e6cbdf7377b694eb7d7a5b913476a5be923322d3de06060fd5e819635232a2cf4f0731da13b8546d1d6d4f8d75b9fce6c2341a71b0ea6f780df54bfdb0dd5cd9855179f602f917265f21f9190c70217774a6fbaaa7d63ad64199f4664813b955cff954949076dcf'),
+                         packet.to_bytes())
+
+    def test_process_onion_packet(self):
+        # this test is not from bolt-04, but is based on the one there;
+        # except here we have the privkeys for these pubkeys
+        payment_path_pubkeys = [
+            bfh('03d75c0ee70f68d73d7d13aeb6261d8ace11416800860c7e59407afe4e2e2d42bb'),
+            bfh('03960a0b830c7b8e76de745b819f252c62508346196b916f5e813cdb0773283cce'),
+            bfh('0385620e0a571cbc3552620f8bf1bdcdab2d1a4a59c36fa10b8249114ccbdda40d'),
+            bfh('02ee242cf6c38b7285f0152c33804ff777f5c51fd352ca8132e845e2cf23b3d8ba'),
+            bfh('025c585fd2e174bf8245b2b4a119e52a417688904228643ea3edaa1728bf2a258e'),
+        ]
+        payment_path_privkeys = [
+            bfh('3463a278617b3dd83f79bda7f97673f12609c54386e1f0d2b67b1c6354fda14e'),
+            bfh('7e1255fddb52db1729fc3ceb21a46f95b8d9fe94cc83425e936a6c5223bb679d'),
+            bfh('c7ce8c1462c311eec24dff9e2532ac6241e50ae57e7d1833af21942136972f23'),
+            bfh('3d885f374d79a5e777459b083f7818cdc9493e5c4994ac9c7b843de8b70be661'),
+            bfh('dd72ab44729527b7942e195e7a835e7c71f9c0ff61844eb21274d9c26166a8f8'),
+        ]
+        session_key = bfh('4141414141414141414141414141414141414141414141414141414141414141')
+        associated_data = bfh('4242424242424242424242424242424242424242424242424242424242424242')
+        hops_data = [
+            OnionHopsDataSingle(OnionPerHop(
+                bfh('0000000000000000'), bfh('0000000000000000'), bfh('00000000')
+            )),
+            OnionHopsDataSingle(OnionPerHop(
+                bfh('0101010101010101'), bfh('0000000000000001'), bfh('00000001')
+            )),
+            OnionHopsDataSingle(OnionPerHop(
+                bfh('0202020202020202'), bfh('0000000000000002'), bfh('00000002')
+            )),
+            OnionHopsDataSingle(OnionPerHop(
+                bfh('0303030303030303'), bfh('0000000000000003'), bfh('00000003')
+            )),
+            OnionHopsDataSingle(OnionPerHop(
+                bfh('0404040404040404'), bfh('0000000000000004'), bfh('00000004')
+            )),
+        ]
+        packet = new_onion_packet(payment_path_pubkeys, session_key, hops_data, associated_data)
+        self.assertEqual(bfh('0002eec7245d6b7d2ccb30380bfbe2a3648cd7a942653f5aa340edcea1f28368661954176cd9869da33d713aa219fcef1e5c806fef11e696bcc66844de8271c27974a0fd57c2dbcb2c6dd4e8ef35d96db28d5a0e49b6ab3d6de31af65950723b8cddc108390bebf8d149002e31bdc283056477ba27c8054c248ad7306de31663a7c99ec659da15d0f6fbc7e1687485b39e9be0ec3b70164cb3618a9b546317e7c2d62ae9f0f840704535729262d30c6132d1b390f073edec8fa057176c6268b6ad06a82ff0d16d4c662194873e8b4ecf46eb2c9d4d58d2ee2021adb19840605ac5afd8bd942dd71e8244c83e28b2ed5a3b09e9e7df5c8c747e5765ba366a4f7407a6c6b0a32f74bc5e428f7fa4c3cf70e13ed91563177d94190d5149aa4b9c96d00e40d2ac35ab9c4a621ce0f6f5df7d64a9c8d435db19de192d9db522c7f7b4e201fc1b61a9bd3efd062ae24455d463818b01e2756c7d0691bc3ac4c017be34c9a8b2913bb1b937e31e0ae40f650a7cd820bcb4996825b1cbad1ff7ccc2b513b1104524c34f6573e1b59201c005a632ee5dccd3711a32e3ba1ff00fcffbe636e4b3a84bbe491b836a57ccec138b8cc2ec733846904d872f305d538d51db8e56232ec6e07877075328874cb7b09c7e799100a9ff085dead253886b174fc408a0ea7b48bce2c5d8992285011960af088f7e006ef60089d46ac9aa15acfac6c87c3cf6904764dd785419292fbafa9cca09c8ade24a6cd63f12d1cfc83fa35cf2f1cf503c39cbf78293f06c68a3cece7177169cd872bb49bf69d933a27a887dd9daefa9239fca9f0c3e309ec61d9df947211da98cf11a6e0fb77252629cdf9f2226dd69ca73fa51be4df224592f8d471b69a1aebbdaa2f3a798b3581253d97feb0a12e6606043ca0fc5efc0f49b8061d6796eff31cd8638499e2f25ffb96eec32837438ed7ebebbe587886648f63e35d80f41869f4c308f2e6970bd65fead5e8544e3239a6acc9d996b08d1546455bcafbe88ed3ed547714841946fe2e77180e4d7bf1452414e4b1745a7897184a2c4cbc3ac46f83342a55a48e29dc8f17cf595dd28f51e297ba89fd25ed0dbd1c0081a810beaab09758a36fbfd16fbdc3daa9fe05c8a73195f244ef2743a5df761f01ee6e693eb6c7f1a7834fab3671391e5ddebf611e119a2ae4456e2cee7a6d4f27a2246cdb1f8ef35f0b3d7044b3799d8d0ed0a6470557fd807c065d6d83acba07e96e10770ada8c0b4d4921522944188d5f30086a6ee0a4795331273f32beaaa43363fc58208a257e5c5c434c7325b583642219d81c7d67b908d5263b42ac1991edc69a777da60f38eff138c844af9e549374e8b29b166211bfded24587a29394e33828b784da7e7b62ab7e49ea2693fcdd17fa96186a5ef11ef1a8adffa50f93a3119e95e6c09014f3e3b0709183fa08a826ced6deb4608b7d986ebbcf99ad58e25451d4d9d38d0059734d8501467b97182cd11e0c07c91ca50f61cc31255a3147ade654976a5989097281892aafd8df595c63bd14f1e03f5955a9398d2dd6368bbcae833ae1cc2df31eb0980b4817dfd130020ffb275743fcc01df40e3ecda1c5988e8e1bde965353b0b1bf34ea05f095000c45b6249618d275905a24d3eb58c600aeab4fb552fbf1ccdb2a5c80ace220310f89829d7e53f78c126037b6d8d500220c7a118d9621b4d6bd5379edd7e24bcf540e87aba6b88862db16fa4ee00b009fda80577be67ab94910fd8a7807dfe4ebe66b8fdcd040aa2dc17ec22639298be56b2a2c9d8940647b75f2f6d81746df16e1cb2f05e23397a8c63baea0803441ff4b7d517ff172980a056726235e2f6af85e8aa9b91ba85f14532272d6170df3166b91169dc09d4f4a251610f57ff0885a93364cfaf650bdf436c89795efed5ca934bc7ffc0a4'),
+                         packet.to_bytes())
+        for i, privkey in enumerate(payment_path_privkeys):
+            processed_packet = lnrouter.process_onion_packet(packet, associated_data, privkey)
+            self.assertEqual(hops_data[i].per_hop.to_bytes(), processed_packet.hop_data.per_hop.to_bytes())
+            packet = processed_packet.next_packet
+
+    def test_decode_onion_error(self):
+        # test vector from bolt-04
+        payment_path_pubkeys = [
+            bfh('02eec7245d6b7d2ccb30380bfbe2a3648cd7a942653f5aa340edcea1f283686619'),
+            bfh('0324653eac434488002cc06bbfb7f10fe18991e35f9fe4302dbea6d2353dc0ab1c'),
+            bfh('027f31ebc5462c1fdce1b737ecff52d37d75dea43ce11c74d25aa297165faa2007'),
+            bfh('032c0b7cf95324a07d05398b240174dc0c2be444d96b159aa6c7f7b1e668680991'),
+            bfh('02edabbd16b41c8371b92ef2f04c1185b4f03b6dcd52ba9b78d9d7c89c8f221145'),
+        ]
+        session_key = bfh('4141414141414141414141414141414141414141414141414141414141414141')
+        error_packet_for_node_0 = bfh('9c5add3963fc7f6ed7f148623c84134b5647e1306419dbe2174e523fa9e2fbed3a06a19f899145610741c83ad40b7712aefaddec8c6baf7325d92ea4ca4d1df8bce517f7e54554608bf2bd8071a4f52a7a2f7ffbb1413edad81eeea5785aa9d990f2865dc23b4bc3c301a94eec4eabebca66be5cf638f693ec256aec514620cc28ee4a94bd9565bc4d4962b9d3641d4278fb319ed2b84de5b665f307a2db0f7fbb757366067d88c50f7e829138fde4f78d39b5b5802f1b92a8a820865af5cc79f9f30bc3f461c66af95d13e5e1f0381c184572a91dee1c849048a647a1158cf884064deddbf1b0b88dfe2f791428d0ba0f6fb2f04e14081f69165ae66d9297c118f0907705c9c4954a199bae0bb96fad763d690e7daa6cfda59ba7f2c8d11448b604d12d')
+        decoded_error, index_of_sender = lnrouter._decode_onion_error(error_packet_for_node_0, payment_path_pubkeys, session_key)
+        self.assertEqual(bfh('4c2fc8bc08510334b6833ad9c3e79cd1b52ae59dfe5c2a4b23ead50f09f7ee0b0002200200fe0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000'),
+                             decoded_error)
+        self.assertEqual(4, index_of_sender)

lib/tests/test_lnutil.py

@@ -0,0 +1,670 @@
+import unittest
+import json
+from lib import bitcoin
+from lib.lnutil import RevocationStore, get_per_commitment_secret_from_seed, make_offered_htlc, make_received_htlc, make_commitment, make_htlc_tx_witness, make_htlc_tx_output, make_htlc_tx_inputs, secret_to_pubkey, derive_blinded_pubkey, derive_privkey, derive_pubkey, make_htlc_tx
+from lib.util import bh2u, bfh
+
+funding_tx_id = '8984484a580b825b9972d7adb15050b3ab624ccd731946b3eeddb92f4e7ef6be'
+funding_output_index = 0
+funding_amount_satoshi = 10000000
+commitment_number = 42
+local_delay = 144
+local_dust_limit_satoshi = 546
+
+local_payment_basepoint = bytes.fromhex('034f355bdcb7cc0af728ef3cceb9615d90684bb5b2ca5f859ab0f0b704075871aa')
+remote_payment_basepoint = bytes.fromhex('032c0b7cf95324a07d05398b240174dc0c2be444d96b159aa6c7f7b1e668680991')
+# obs = get_obscured_ctn(42, local_payment_basepoint, remote_payment_basepoint)
+local_funding_privkey = bytes.fromhex('30ff4956bbdd3222d44cc5e8a1261dab1e07957bdac5ae88fe3261ef321f374901')
+local_funding_pubkey = bytes.fromhex('023da092f6980e58d2c037173180e9a465476026ee50f96695963e8efe436f54eb')
+remote_funding_pubkey = bytes.fromhex('030e9f7b623d2ccc7c9bd44d66d5ce21ce504c0acf6385a132cec6d3c39fa711c1')
+local_privkey = bytes.fromhex('bb13b121cdc357cd2e608b0aea294afca36e2b34cf958e2e6451a2f27469449101')
+localpubkey = bytes.fromhex('030d417a46946384f88d5f3337267c5e579765875dc4daca813e21734b140639e7')
+remotepubkey = bytes.fromhex('0394854aa6eab5b2a8122cc726e9dded053a2184d88256816826d6231c068d4a5b')
+local_delayedpubkey = bytes.fromhex('03fd5960528dc152014952efdb702a88f71e3c1653b2314431701ec77e57fde83c')
+local_revocation_pubkey = bytes.fromhex('0212a140cd0c6539d07cd08dfe09984dec3251ea808b892efeac3ede9402bf2b19')
+# funding wscript = 5221023da092f6980e58d2c037173180e9a465476026ee50f96695963e8efe436f54eb21030e9f7b623d2ccc7c9bd44d66d5ce21ce504c0acf6385a132cec6d3c39fa711c152ae
+
+class TestLNUtil(unittest.TestCase):
+    def test_shachain_store(self):
+        tests = [
+            {
+                "name": "insert_secret correct sequence",
+                "inserts": [
+                    {
+                        "index": 281474976710655,
+                        "secret": "7cc854b54e3e0dcdb010d7a3fee464a9687b" +\
+                            "e6e8db3be6854c475621e007a5dc",
+                        "successful": True
+                    },
+                    {
+                        "index": 281474976710654,
+                        "secret": "c7518c8ae4660ed02894df8976fa1a3659c1" +\
+                            "a8b4b5bec0c4b872abeba4cb8964",
+                        "successful": True
+                    },
+                    {
+                        "index": 281474976710653,
+                        "secret": "2273e227a5b7449b6e70f1fb4652864038b1" +\
+                            "cbf9cd7c043a7d6456b7fc275ad8",
+                        "successful": True
+                    },
+                    {
+                        "index": 281474976710652,
+                        "secret": "27cddaa5624534cb6cb9d7da077cf2b22ab2" +\
+                            "1e9b506fd4998a51d54502e99116",
+                        "successful": True
+                    },
+                    {
+                        "index": 281474976710651,
+                        "secret": "c65716add7aa98ba7acb236352d665cab173" +\
+                            "45fe45b55fb879ff80e6bd0c41dd",
+                        "successful": True
+                    },
+                    {
+                        "index": 281474976710650,
+                        "secret": "969660042a28f32d9be17344e09374b37996" +\
+                            "2d03db1574df5a8a5a47e19ce3f2",
+                        "successful": True
+                    },
+                    {
+                        "index": 281474976710649,
+                        "secret": "a5a64476122ca0925fb344bdc1854c1c0a59" +\
+                            "fc614298e50a33e331980a220f32",
+                        "successful": True
+                    },
+                    {
+                        "index": 281474976710648,
+                        "secret": "05cde6323d949933f7f7b78776bcc1ea6d9b" +\
+                            "31447732e3802e1f7ac44b650e17",
+                        "successful": True
+                    }
+                ]
+            },
+            {
+                "name": "insert_secret #1 incorrect",
+                "inserts": [
+                    {
+                        "index": 281474976710655,
+                        "secret": "02a40c85b6f28da08dfdbe0926c53fab2d" +\
+                            "e6d28c10301f8f7c4073d5e42e3148",
+                        "successful": True
+                    },
+                    {
+                        "index": 281474976710654,
+                        "secret": "c7518c8ae4660ed02894df8976fa1a3659" +\
+                            "c1a8b4b5bec0c4b872abeba4cb8964",
+                        "successful": False
+                    }
+                ]
+            },
+            {
+                "name": "insert_secret #2 incorrect (#1 derived from incorrect)",
+                "inserts": [
+                    {
+                        "index": 281474976710655,
+                        "secret": "02a40c85b6f28da08dfdbe0926c53fab2de6" +\
+                            "d28c10301f8f7c4073d5e42e3148",
+                        "successful": True
+                    },
+                    {
+                        "index": 281474976710654,
+                        "secret": "dddc3a8d14fddf2b68fa8c7fbad274827493" +\
+                            "7479dd0f8930d5ebb4ab6bd866a3",
+                        "successful": True
+                    },
+                    {
+                        "index": 281474976710653,
+                        "secret": "2273e227a5b7449b6e70f1fb4652864038b1" +\
+                            "cbf9cd7c043a7d6456b7fc275ad8",
+                        "successful": True
+                    },
+                    {
+                        "index": 281474976710652,
+                        "secret": "27cddaa5624534cb6cb9d7da077cf2b22a" +\
+                            "b21e9b506fd4998a51d54502e99116",
+                        "successful": False
+                    }
+                ]
+            },
+            {
+                "name": "insert_secret #3 incorrect",
+                "inserts": [
+                    {
+                        "index": 281474976710655,
+                        "secret": "7cc854b54e3e0dcdb010d7a3fee464a9687b" +\
+                            "e6e8db3be6854c475621e007a5dc",
+                        "successful": True
+                    },
+                    {
+                        "index": 281474976710654,
+                        "secret": "c7518c8ae4660ed02894df8976fa1a3659c1" +\
+                            "a8b4b5bec0c4b872abeba4cb8964",
+                        "successful": True
+                    },
+                    {
+                        "index": 281474976710653,
+                        "secret": "c51a18b13e8527e579ec56365482c62f180b" +\
+                            "7d5760b46e9477dae59e87ed423a",
+                        "successful": True
+                    },
+                    {
+                        "index": 281474976710652,
+                        "secret": "27cddaa5624534cb6cb9d7da077cf2b22ab2" +\
+                            "1e9b506fd4998a51d54502e99116",
+                        "successful": False
+                    }
+                ]
+            },
+            {
+                "name": "insert_secret #4 incorrect (1,2,3 derived from incorrect)",
+                "inserts": [
+                    {
+                        "index": 281474976710655,
+                        "secret": "02a40c85b6f28da08dfdbe0926c53fab2de6" +\
+                            "d28c10301f8f7c4073d5e42e3148",
+                        "successful": True
+                    },
+                    {
+                        "index": 281474976710654,
+                        "secret": "dddc3a8d14fddf2b68fa8c7fbad274827493" +\
+                            "7479dd0f8930d5ebb4ab6bd866a3",
+                        "successful": True
+                    },
+                    {
+                        "index": 281474976710653,
+                        "secret": "c51a18b13e8527e579ec56365482c62f18" +\
+                            "0b7d5760b46e9477dae59e87ed423a",
+                        "successful": True
+                    },
+                    {
+                        "index": 281474976710652,
+                        "secret": "ba65d7b0ef55a3ba300d4e87af29868f39" +\
+                            "4f8f138d78a7011669c79b37b936f4",
+                        "successful": True
+                    },
+                    {
+                        "index": 281474976710651,
+                        "secret": "c65716add7aa98ba7acb236352d665cab1" +\
+                            "7345fe45b55fb879ff80e6bd0c41dd",
+                        "successful": True
+                    },
+                    {
+                        "index": 281474976710650,
+                        "secret": "969660042a28f32d9be17344e09374b379" +\
+                            "962d03db1574df5a8a5a47e19ce3f2",
+                        "successful": True
+                    },
+                    {
+                        "index": 281474976710649,
+                        "secret": "a5a64476122ca0925fb344bdc1854c1c0a" +\
+                            "59fc614298e50a33e331980a220f32",
+                        "successful": True
+                    },
+                    {
+                        "index": 281474976710649,
+                        "secret": "05cde6323d949933f7f7b78776bcc1ea6d9b" +\
+                            "31447732e3802e1f7ac44b650e17",
+                        "successful": False
+                    }
+                ]
+            },
+            {
+                "name": "insert_secret #5 incorrect",
+                "inserts": [
+                    {
+                        "index": 281474976710655,
+                        "secret": "7cc854b54e3e0dcdb010d7a3fee464a9687b" +\
+                            "e6e8db3be6854c475621e007a5dc",
+                        "successful": True
+                    },
+                    {
+                        "index": 281474976710654,
+                        "secret": "c7518c8ae4660ed02894df8976fa1a3659c1a" +\
+                            "8b4b5bec0c4b872abeba4cb8964",
+                        "successful": True
+                    },
+                    {
+                        "index": 281474976710653,
+                        "secret": "2273e227a5b7449b6e70f1fb4652864038b1" +\
+                            "cbf9cd7c043a7d6456b7fc275ad8",
+                        "successful": True
+                    },
+                    {
+                        "index": 281474976710652,
+                        "secret": "27cddaa5624534cb6cb9d7da077cf2b22ab21" +\
+                            "e9b506fd4998a51d54502e99116",
+                        "successful": True
+                    },
+                    {
+                        "index": 281474976710651,
+                        "secret": "631373ad5f9ef654bb3dade742d09504c567" +\
+                            "edd24320d2fcd68e3cc47e2ff6a6",
+                        "successful": True
+                    },
+                    {
+                        "index": 281474976710650,
+                        "secret": "969660042a28f32d9be17344e09374b37996" +\
+                            "2d03db1574df5a8a5a47e19ce3f2",
+                        "successful": False
+                    }
+                ]
+            },
+            {
+                "name": "insert_secret #6 incorrect (5 derived from incorrect)",
+                "inserts": [
+                    {
+                        "index": 281474976710655,
+                        "secret": "7cc854b54e3e0dcdb010d7a3fee464a9687b" +\
+                            "e6e8db3be6854c475621e007a5dc",
+                        "successful": True
+                    },
+                    {
+                        "index": 281474976710654,
+                        "secret": "c7518c8ae4660ed02894df8976fa1a3659c1a" +\
+                            "8b4b5bec0c4b872abeba4cb8964",
+                        "successful": True
+                    },
+                    {
+                        "index": 281474976710653,
+                        "secret": "2273e227a5b7449b6e70f1fb4652864038b1" +\
+                            "cbf9cd7c043a7d6456b7fc275ad8",
+                        "successful": True
+                    },
+                    {
+                        "index": 281474976710652,
+                        "secret": "27cddaa5624534cb6cb9d7da077cf2b22ab21" +\
+                            "e9b506fd4998a51d54502e99116",
+                        "successful": True
+                    },
+                    {
+                        "index": 281474976710651,
+                        "secret": "631373ad5f9ef654bb3dade742d09504c567" +\
+                            "edd24320d2fcd68e3cc47e2ff6a6",
+                        "successful": True
+                    },
+                    {
+                        "index": 281474976710650,
+                        "secret": "b7e76a83668bde38b373970155c868a65330" +\
+                            "4308f9896692f904a23731224bb1",
+                        "successful": True
+                    },
+                    {
+                        "index": 281474976710649,
+                        "secret": "a5a64476122ca0925fb344bdc1854c1c0a59f" +\
+                            "c614298e50a33e331980a220f32",
+                        "successful": True
+                    },
+                    {
+                        "index": 281474976710648,
+                        "secret": "05cde6323d949933f7f7b78776bcc1ea6d9b" +\
+                            "31447732e3802e1f7ac44b650e17",
+                        "successful": False
+                    }
+                ]
+            },
+            {
+                "name": "insert_secret #7 incorrect",
+                "inserts": [
+                    {
+                        "index": 281474976710655,
+                        "secret": "7cc854b54e3e0dcdb010d7a3fee464a9687b" +\
+                            "e6e8db3be6854c475621e007a5dc",
+                        "successful": True
+                    },
+                    {
+                        "index": 281474976710654,
+                        "secret": "c7518c8ae4660ed02894df8976fa1a3659c1a" +\
+                            "8b4b5bec0c4b872abeba4cb8964",
+                        "successful": True
+                    },
+                    {
+                        "index": 281474976710653,
+                        "secret": "2273e227a5b7449b6e70f1fb4652864038b1" +\
+                            "cbf9cd7c043a7d6456b7fc275ad8",
+                        "successful": True
+                    },
+                    {
+                        "index": 281474976710652,
+                        "secret": "27cddaa5624534cb6cb9d7da077cf2b22ab21" +\
+                            "e9b506fd4998a51d54502e99116",
+                        "successful": True
+                    },
+                    {
+                        "index": 281474976710651,
+                        "secret": "c65716add7aa98ba7acb236352d665cab173" +\
+                            "45fe45b55fb879ff80e6bd0c41dd",
+                        "successful": True
+                    },
+                    {
+                        "index": 281474976710650,
+                        "secret": "969660042a28f32d9be17344e09374b37996" +\
+                            "2d03db1574df5a8a5a47e19ce3f2",
+                        "successful": True
+                    },
+                    {
+                        "index": 281474976710649,
+                        "secret": "e7971de736e01da8ed58b94c2fc216cb1d" +\
+                            "ca9e326f3a96e7194fe8ea8af6c0a3",
+                        "successful": True
+                    },
+                    {
+                        "index": 281474976710648,
+                        "secret": "05cde6323d949933f7f7b78776bcc1ea6d" +\
+                            "9b31447732e3802e1f7ac44b650e17",
+                        "successful": False
+                    }
+                ]
+            },
+            {
+                "name": "insert_secret #8 incorrect",
+                "inserts": [
+                    {
+                        "index": 281474976710655,
+                        "secret": "7cc854b54e3e0dcdb010d7a3fee464a9687b" +\
+                            "e6e8db3be6854c475621e007a5dc",
+                        "successful": True
+                    },
+                    {
+                        "index": 281474976710654,
+                        "secret": "c7518c8ae4660ed02894df8976fa1a3659c1a" +\
+                            "8b4b5bec0c4b872abeba4cb8964",
+                        "successful": True
+                    },
+                    {
+                        "index": 281474976710653,
+                        "secret": "2273e227a5b7449b6e70f1fb4652864038b1" +\
+                            "cbf9cd7c043a7d6456b7fc275ad8",
+                        "successful": True
+                    },
+                    {
+                        "index": 281474976710652,
+                        "secret": "27cddaa5624534cb6cb9d7da077cf2b22ab21" +\
+                            "e9b506fd4998a51d54502e99116",
+                        "successful": True
+                    },
+                    {
+                        "index": 281474976710651,
+                        "secret": "c65716add7aa98ba7acb236352d665cab173" +\
+                            "45fe45b55fb879ff80e6bd0c41dd",
+                        "successful": True
+                    },
+                    {
+                        "index": 281474976710650,
+                        "secret": "969660042a28f32d9be17344e09374b37996" +\
+                            "2d03db1574df5a8a5a47e19ce3f2",
+                        "successful": True
+                    },
+                    {
+                        "index": 281474976710649,
+                        "secret": "a5a64476122ca0925fb344bdc1854c1c0a" +\
+                            "59fc614298e50a33e331980a220f32",
+                        "successful": True
+                    },
+                    {
+                        "index": 281474976710648,
+                        "secret": "a7efbc61aac46d34f77778bac22c8a20c6" +\
+                            "a46ca460addc49009bda875ec88fa4",
+                        "successful": False
+                    }
+                ]
+            }
+        ]
+
+        for test in tests:
+            receiver = RevocationStore()
+            for insert in test["inserts"]:
+                secret = bytes.fromhex(insert["secret"])
+
+                try:
+                    receiver.add_next_entry(secret)
+                except Exception as e:
+                    if insert["successful"]:
+                        raise Exception("Failed ({}): error was received but it shouldn't: {}".format(test["name"], e))
+                else:
+                    if not insert["successful"]:
+                        raise Exception("Failed ({}): error wasn't received".format(test["name"]))
+
+            print("Passed ({})".format(test["name"]))
+
+    def test_shachain_produce_consume(self):
+        seed = bitcoin.sha256(b"shachaintest")
+        consumer = RevocationStore()
+        for i in range(10000):
+            secret = get_per_commitment_secret_from_seed(seed, 2**48 - i - 1)
+            try:
+                consumer.add_next_entry(secret)
+            except Exception as e:
+                raise Exception("iteration " + str(i) + ": " + str(e))
+            if i % 1000 == 0: self.assertEqual(consumer.serialize(), RevocationStore.from_json_obj(json.loads(json.dumps(consumer.serialize()))).serialize())
+
+    def test_commitment_tx_with_all_five_HTLCs_untrimmed_minimum_feerate(self):
+        to_local_msat = 6988000000
+        to_remote_msat = 3000000000
+        local_feerate_per_kw = 0
+        # base commitment transaction fee = 0
+        # actual commitment transaction fee = 0
+
+        per_commitment_secret = 0x1f1e1d1c1b1a191817161514131211100f0e0d0c0b0a09080706050403020100
+        per_commitment_point = secret_to_pubkey(per_commitment_secret)
+
+        remote_htlcpubkey = remotepubkey
+        local_htlcpubkey = localpubkey
+
+        htlc2_cltv_timeout = 502
+        htlc2_payment_preimage = b"\x02" * 32
+        htlc2 = make_offered_htlc(local_revocation_pubkey, remote_htlcpubkey, local_htlcpubkey, bitcoin.sha256(htlc2_payment_preimage))
+        # HTLC 2 offered amount 2000
+        ref_htlc2_wscript = "76a91414011f7254d96b819c76986c277d115efce6f7b58763ac67210394854aa6eab5b2a8122cc726e9dded053a2184d88256816826d6231c068d4a5b7c820120876475527c21030d417a46946384f88d5f3337267c5e579765875dc4daca813e21734b140639e752ae67a914b43e1b38138a41b37f7cd9a1d274bc63e3a9b5d188ac6868"
+        self.assertEqual(htlc2, bfh(ref_htlc2_wscript))
+
+        htlc3_cltv_timeout = 503
+        htlc3_payment_preimage = b"\x03" * 32
+        htlc3 = make_offered_htlc(local_revocation_pubkey, remote_htlcpubkey, local_htlcpubkey, bitcoin.sha256(htlc3_payment_preimage))
+        # HTLC 3 offered amount 3000
+        ref_htlc3_wscript = "76a91414011f7254d96b819c76986c277d115efce6f7b58763ac67210394854aa6eab5b2a8122cc726e9dded053a2184d88256816826d6231c068d4a5b7c820120876475527c21030d417a46946384f88d5f3337267c5e579765875dc4daca813e21734b140639e752ae67a9148a486ff2e31d6158bf39e2608864d63fefd09d5b88ac6868"
+        self.assertEqual(htlc3, bfh(ref_htlc3_wscript))
+
+        htlc0_cltv_timeout = 500
+        htlc0_payment_preimage = b"\x00" * 32
+        htlc0 = make_received_htlc(local_revocation_pubkey, remote_htlcpubkey, local_htlcpubkey, bitcoin.sha256(htlc0_payment_preimage), htlc0_cltv_timeout)
+        # HTLC 0 received amount 1000
+        ref_htlc0_wscript = "76a91414011f7254d96b819c76986c277d115efce6f7b58763ac67210394854aa6eab5b2a8122cc726e9dded053a2184d88256816826d6231c068d4a5b7c8201208763a914b8bcb07f6344b42ab04250c86a6e8b75d3fdbbc688527c21030d417a46946384f88d5f3337267c5e579765875dc4daca813e21734b140639e752ae677502f401b175ac6868"
+        self.assertEqual(htlc0, bfh(ref_htlc0_wscript))
+
+        htlc1_cltv_timeout = 501
+        htlc1_payment_preimage = b"\x01" * 32
+        htlc1 = make_received_htlc(local_revocation_pubkey, remote_htlcpubkey, local_htlcpubkey, bitcoin.sha256(htlc1_payment_preimage), htlc1_cltv_timeout)
+        # HTLC 1 received amount 2000
+        ref_htlc1_wscript = "76a91414011f7254d96b819c76986c277d115efce6f7b58763ac67210394854aa6eab5b2a8122cc726e9dded053a2184d88256816826d6231c068d4a5b7c8201208763a9144b6b2e5444c2639cc0fb7bcea5afba3f3cdce23988527c21030d417a46946384f88d5f3337267c5e579765875dc4daca813e21734b140639e752ae677502f501b175ac6868"
+        self.assertEqual(htlc1, bfh(ref_htlc1_wscript))
+
+        htlc4_cltv_timeout = 504
+        htlc4_payment_preimage = b"\x04" * 32
+        htlc4 = make_received_htlc(local_revocation_pubkey, remote_htlcpubkey, local_htlcpubkey, bitcoin.sha256(htlc4_payment_preimage), htlc4_cltv_timeout)
+        # HTLC 4 received amount 4000
+        ref_htlc4_wscript = "76a91414011f7254d96b819c76986c277d115efce6f7b58763ac67210394854aa6eab5b2a8122cc726e9dded053a2184d88256816826d6231c068d4a5b7c8201208763a91418bc1a114ccf9c052d3d23e28d3b0a9d1227434288527c21030d417a46946384f88d5f3337267c5e579765875dc4daca813e21734b140639e752ae677502f801b175ac6868"
+        self.assertEqual(htlc4, bfh(ref_htlc4_wscript))
+
+        # to_local amount 6988000 wscript 63210212a140cd0c6539d07cd08dfe09984dec3251ea808b892efeac3ede9402bf2b1967029000b2752103fd5960528dc152014952efdb702a88f71e3c1653b2314431701ec77e57fde83c68ac
+        # to_remote amount 3000000 P2WPKH(0394854aa6eab5b2a8122cc726e9dded053a2184d88256816826d6231c068d4a5b)
+        remote_signature = "304402204fd4928835db1ccdfc40f5c78ce9bd65249b16348df81f0c44328dcdefc97d630220194d3869c38bc732dd87d13d2958015e2fc16829e74cd4377f84d215c0b70606"
+        # local_signature = 30440220275b0c325a5e9355650dc30c0eccfbc7efb23987c24b556b9dfdd40effca18d202206caceb2c067836c51f296740c7ae807ffcbfbf1dd3a0d56b6de9a5b247985f06
+        output_commit_tx = "02000000000101bef67e4e2fb9ddeeb3461973cd4c62abb35050b1add772995b820b584a488489000000000038b02b8007e80300000000000022002052bfef0479d7b293c27e0f1eb294bea154c63a3294ef092c19af51409bce0e2ad007000000000000220020403d394747cae42e98ff01734ad5c08f82ba123d3d9a620abda88989651e2ab5d007000000000000220020748eba944fedc8827f6b06bc44678f93c0f9e6078b35c6331ed31e75f8ce0c2db80b000000000000220020c20b5d1f8584fd90443e7b7b720136174fa4b9333c261d04dbbd012635c0f419a00f0000000000002200208c48d15160397c9731df9bc3b236656efb6665fbfe92b4a6878e88a499f741c4c0c62d0000000000160014ccf1af2f2aabee14bb40fa3851ab2301de843110e0a06a00000000002200204adb4e2f00643db396dd120d4e7dc17625f5f2c11a40d857accc862d6b7dd80e04004730440220275b0c325a5e9355650dc30c0eccfbc7efb23987c24b556b9dfdd40effca18d202206caceb2c067836c51f296740c7ae807ffcbfbf1dd3a0d56b6de9a5b247985f060147304402204fd4928835db1ccdfc40f5c78ce9bd65249b16348df81f0c44328dcdefc97d630220194d3869c38bc732dd87d13d2958015e2fc16829e74cd4377f84d215c0b7060601475221023da092f6980e58d2c037173180e9a465476026ee50f96695963e8efe436f54eb21030e9f7b623d2ccc7c9bd44d66d5ce21ce504c0acf6385a132cec6d3c39fa711c152ae3e195220"
+
+        htlc0_msat = 1000 * 1000
+        htlc2_msat = 2000 * 1000
+        htlc3_msat = 3000 * 1000
+        htlc1_msat = 2000 * 1000
+        htlc4_msat = 4000 * 1000
+        htlcs = [(htlc2, htlc2_msat), (htlc3, htlc3_msat), (htlc0, htlc0_msat), (htlc1, htlc1_msat), (htlc4, htlc4_msat)]
+
+        our_commit_tx = make_commitment(
+            commitment_number,
+            local_funding_pubkey, remote_funding_pubkey, remotepubkey,
+            local_payment_basepoint, remote_payment_basepoint,
+            local_revocation_pubkey, local_delayedpubkey, local_delay,
+            funding_tx_id, funding_output_index, funding_amount_satoshi,
+            to_local_msat, to_remote_msat, local_dust_limit_satoshi,
+            local_feerate_per_kw, True, we_are_initiator=True, htlcs=htlcs)
+        self.sign_and_insert_remote_sig(our_commit_tx, remote_funding_pubkey, remote_signature, local_funding_pubkey, local_funding_privkey)
+        self.assertEqual(str(our_commit_tx), output_commit_tx)
+
+        # (HTLC 0)
+        signature_for_output_remote_htlc_0 = "304402206a6e59f18764a5bf8d4fa45eebc591566689441229c918b480fb2af8cc6a4aeb02205248f273be447684b33e3c8d1d85a8e0ca9fa0bae9ae33f0527ada9c162919a6"
+        # (HTLC 2)
+        signature_for_output_remote_htlc_2 = "3045022100d5275b3619953cb0c3b5aa577f04bc512380e60fa551762ce3d7a1bb7401cff9022037237ab0dac3fe100cde094e82e2bed9ba0ed1bb40154b48e56aa70f259e608b"
+        # (HTLC 1)
+        signature_for_output_remote_htlc_1 = "304402201b63ec807771baf4fdff523c644080de17f1da478989308ad13a58b51db91d360220568939d38c9ce295adba15665fa68f51d967e8ed14a007b751540a80b325f202"
+        # (HTLC 3)
+        signature_for_output_remote_htlc_3 = "3045022100daee1808f9861b6c3ecd14f7b707eca02dd6bdfc714ba2f33bc8cdba507bb182022026654bf8863af77d74f51f4e0b62d461a019561bb12acb120d3f7195d148a554"
+        # (HTLC 4)
+        signature_for_output_remote_htlc_4 = "304402207e0410e45454b0978a623f36a10626ef17b27d9ad44e2760f98cfa3efb37924f0220220bd8acd43ecaa916a80bd4f919c495a2c58982ce7c8625153f8596692a801d"
+
+
+        local_signature_htlc0 = "304402207cb324fa0de88f452ffa9389678127ebcf4cabe1dd848b8e076c1a1962bf34720220116ed922b12311bd602d67e60d2529917f21c5b82f25ff6506c0f87886b4dfd5" # derive ourselves
+        output_htlc_success_tx_0 = "020000000001018154ecccf11a5fb56c39654c4deb4d2296f83c69268280b94d021370c94e219700000000000000000001e8030000000000002200204adb4e2f00643db396dd120d4e7dc17625f5f2c11a40d857accc862d6b7dd80e050047304402206a6e59f18764a5bf8d4fa45eebc591566689441229c918b480fb2af8cc6a4aeb02205248f273be447684b33e3c8d1d85a8e0ca9fa0bae9ae33f0527ada9c162919a60147304402207cb324fa0de88f452ffa9389678127ebcf4cabe1dd848b8e076c1a1962bf34720220116ed922b12311bd602d67e60d2529917f21c5b82f25ff6506c0f87886b4dfd5012000000000000000000000000000000000000000000000000000000000000000008a76a91414011f7254d96b819c76986c277d115efce6f7b58763ac67210394854aa6eab5b2a8122cc726e9dded053a2184d88256816826d6231c068d4a5b7c8201208763a914b8bcb07f6344b42ab04250c86a6e8b75d3fdbbc688527c21030d417a46946384f88d5f3337267c5e579765875dc4daca813e21734b140639e752ae677502f401b175ac686800000000"
+
+        local_signature_htlc2 = "3045022100c89172099507ff50f4c925e6c5150e871fb6e83dd73ff9fbb72f6ce829a9633f02203a63821d9162e99f9be712a68f9e589483994feae2661e4546cd5b6cec007be5"
+        output_htlc_timeout_tx_2 = "020000000001018154ecccf11a5fb56c39654c4deb4d2296f83c69268280b94d021370c94e219701000000000000000001d0070000000000002200204adb4e2f00643db396dd120d4e7dc17625f5f2c11a40d857accc862d6b7dd80e0500483045022100d5275b3619953cb0c3b5aa577f04bc512380e60fa551762ce3d7a1bb7401cff9022037237ab0dac3fe100cde094e82e2bed9ba0ed1bb40154b48e56aa70f259e608b01483045022100c89172099507ff50f4c925e6c5150e871fb6e83dd73ff9fbb72f6ce829a9633f02203a63821d9162e99f9be712a68f9e589483994feae2661e4546cd5b6cec007be501008576a91414011f7254d96b819c76986c277d115efce6f7b58763ac67210394854aa6eab5b2a8122cc726e9dded053a2184d88256816826d6231c068d4a5b7c820120876475527c21030d417a46946384f88d5f3337267c5e579765875dc4daca813e21734b140639e752ae67a914b43e1b38138a41b37f7cd9a1d274bc63e3a9b5d188ac6868f6010000"
+
+        local_signature_htlc1 = "3045022100def389deab09cee69eaa1ec14d9428770e45bcbe9feb46468ecf481371165c2f022015d2e3c46600b2ebba8dcc899768874cc6851fd1ecb3fffd15db1cc3de7e10da"
+        output_htlc_success_tx_1 = "020000000001018154ecccf11a5fb56c39654c4deb4d2296f83c69268280b94d021370c94e219702000000000000000001d0070000000000002200204adb4e2f00643db396dd120d4e7dc17625f5f2c11a40d857accc862d6b7dd80e050047304402201b63ec807771baf4fdff523c644080de17f1da478989308ad13a58b51db91d360220568939d38c9ce295adba15665fa68f51d967e8ed14a007b751540a80b325f20201483045022100def389deab09cee69eaa1ec14d9428770e45bcbe9feb46468ecf481371165c2f022015d2e3c46600b2ebba8dcc899768874cc6851fd1ecb3fffd15db1cc3de7e10da012001010101010101010101010101010101010101010101010101010101010101018a76a91414011f7254d96b819c76986c277d115efce6f7b58763ac67210394854aa6eab5b2a8122cc726e9dded053a2184d88256816826d6231c068d4a5b7c8201208763a9144b6b2e5444c2639cc0fb7bcea5afba3f3cdce23988527c21030d417a46946384f88d5f3337267c5e579765875dc4daca813e21734b140639e752ae677502f501b175ac686800000000"
+
+        local_signature_htlc3 = "30440220643aacb19bbb72bd2b635bc3f7375481f5981bace78cdd8319b2988ffcc6704202203d27784ec8ad51ed3bd517a05525a5139bb0b755dd719e0054332d186ac08727"
+        output_htlc_timeout_tx_3 = "020000000001018154ecccf11a5fb56c39654c4deb4d2296f83c69268280b94d021370c94e219703000000000000000001b80b0000000000002200204adb4e2f00643db396dd120d4e7dc17625f5f2c11a40d857accc862d6b7dd80e0500483045022100daee1808f9861b6c3ecd14f7b707eca02dd6bdfc714ba2f33bc8cdba507bb182022026654bf8863af77d74f51f4e0b62d461a019561bb12acb120d3f7195d148a554014730440220643aacb19bbb72bd2b635bc3f7375481f5981bace78cdd8319b2988ffcc6704202203d27784ec8ad51ed3bd517a05525a5139bb0b755dd719e0054332d186ac0872701008576a91414011f7254d96b819c76986c277d115efce6f7b58763ac67210394854aa6eab5b2a8122cc726e9dded053a2184d88256816826d6231c068d4a5b7c820120876475527c21030d417a46946384f88d5f3337267c5e579765875dc4daca813e21734b140639e752ae67a9148a486ff2e31d6158bf39e2608864d63fefd09d5b88ac6868f7010000"
+
+        local_signature_htlc4 = "30440220549e80b4496803cbc4a1d09d46df50109f546d43fbbf86cd90b174b1484acd5402205f12a4f995cb9bded597eabfee195a285986aa6d93ae5bb72507ebc6a4e2349e"
+        output_htlc_success_tx_4 = "020000000001018154ecccf11a5fb56c39654c4deb4d2296f83c69268280b94d021370c94e219704000000000000000001a00f0000000000002200204adb4e2f00643db396dd120d4e7dc17625f5f2c11a40d857accc862d6b7dd80e050047304402207e0410e45454b0978a623f36a10626ef17b27d9ad44e2760f98cfa3efb37924f0220220bd8acd43ecaa916a80bd4f919c495a2c58982ce7c8625153f8596692a801d014730440220549e80b4496803cbc4a1d09d46df50109f546d43fbbf86cd90b174b1484acd5402205f12a4f995cb9bded597eabfee195a285986aa6d93ae5bb72507ebc6a4e2349e012004040404040404040404040404040404040404040404040404040404040404048a76a91414011f7254d96b819c76986c277d115efce6f7b58763ac67210394854aa6eab5b2a8122cc726e9dded053a2184d88256816826d6231c068d4a5b7c8201208763a91418bc1a114ccf9c052d3d23e28d3b0a9d1227434288527c21030d417a46946384f88d5f3337267c5e579765875dc4daca813e21734b140639e752ae677502f801b175ac686800000000"
+
+        def test_htlc_tx(htlc, htlc_output_index, amount_msat, ref_local_sig, htlc_payment_preimage, remote_htlc_sig, ref_tx, success, cltv_timeout):
+            our_htlc_tx_output = make_htlc_tx_output(
+                amount_msat=amount_msat,
+                local_feerate=local_feerate_per_kw,
+                revocationpubkey=local_revocation_pubkey,
+                local_delayedpubkey=local_delayedpubkey,
+                success=success,
+                to_self_delay=local_delay)
+            our_htlc_tx_inputs = make_htlc_tx_inputs(
+                htlc_output_txid=our_commit_tx.txid(),
+                htlc_output_index=htlc_output_index,
+                revocationpubkey=local_revocation_pubkey,
+                local_delayedpubkey=local_delayedpubkey,
+                amount_msat=amount_msat,
+                witness_script=bh2u(htlc))
+            our_htlc_tx = make_htlc_tx(cltv_timeout,
+                inputs=our_htlc_tx_inputs,
+                output=our_htlc_tx_output)
+
+            local_sig = our_htlc_tx.sign_txin(0, local_privkey[:-1])
+            #self.assertEqual(ref_local_sig + "01", local_sig)  # commented out as it is sufficient to compare the serialized txn
+
+            our_htlc_tx_witness = make_htlc_tx_witness(  # FIXME only correct for success=True
+                remotehtlcsig=bfh(remote_htlc_sig) + b"\x01",  # 0x01 is SIGHASH_ALL
+                localhtlcsig=bfh(local_sig),
+                payment_preimage=htlc_payment_preimage if success else b'',  # will put 00 on witness if timeout
+                witness_script=htlc)
+            our_htlc_tx._inputs[0]['witness'] = bh2u(our_htlc_tx_witness)
+            self.assertEqual(ref_tx, str(our_htlc_tx))
+
+        test_htlc_tx(htlc=htlc0, htlc_output_index=0,
+                     amount_msat=htlc0_msat,
+                     ref_local_sig=local_signature_htlc0,
+                     htlc_payment_preimage=htlc0_payment_preimage,
+                     remote_htlc_sig=signature_for_output_remote_htlc_0,
+                     ref_tx=output_htlc_success_tx_0,
+                     success=True, cltv_timeout=0)
+        test_htlc_tx(htlc=htlc1, htlc_output_index=2,
+                     amount_msat=htlc1_msat,
+                     ref_local_sig=local_signature_htlc1,
+                     htlc_payment_preimage=htlc1_payment_preimage,
+                     remote_htlc_sig=signature_for_output_remote_htlc_1,
+                     ref_tx=output_htlc_success_tx_1,
+                     success=True, cltv_timeout=0)
+        test_htlc_tx(htlc=htlc2,  htlc_output_index=1,
+                     amount_msat=htlc2_msat,
+                     ref_local_sig=local_signature_htlc2,
+                     htlc_payment_preimage=htlc2_payment_preimage,
+                     remote_htlc_sig=signature_for_output_remote_htlc_2,
+                     ref_tx=output_htlc_timeout_tx_2,
+                     success=False, cltv_timeout=htlc2_cltv_timeout)
+        test_htlc_tx(htlc=htlc3,  htlc_output_index=3,
+                     amount_msat=htlc3_msat,
+                     ref_local_sig=local_signature_htlc3,
+                     htlc_payment_preimage=htlc3_payment_preimage,
+                     remote_htlc_sig=signature_for_output_remote_htlc_3,
+                     ref_tx=output_htlc_timeout_tx_3,
+                     success=False, cltv_timeout=htlc3_cltv_timeout)
+        test_htlc_tx(htlc=htlc4,  htlc_output_index=4,
+                     amount_msat=htlc4_msat,
+                     ref_local_sig=local_signature_htlc4,
+                     htlc_payment_preimage=htlc4_payment_preimage,
+                     remote_htlc_sig=signature_for_output_remote_htlc_4,
+                     ref_tx=output_htlc_success_tx_4,
+                     success=True, cltv_timeout=0)
+
+    def test_per_commitment_secret_from_seed(self):
+        self.assertEqual(0x02a40c85b6f28da08dfdbe0926c53fab2de6d28c10301f8f7c4073d5e42e3148.to_bytes(byteorder="big", length=32),
+                         get_per_commitment_secret_from_seed(0x0000000000000000000000000000000000000000000000000000000000000000.to_bytes(byteorder="big", length=32), 281474976710655))
+        self.assertEqual(0x7cc854b54e3e0dcdb010d7a3fee464a9687be6e8db3be6854c475621e007a5dc.to_bytes(byteorder="big", length=32),
+                         get_per_commitment_secret_from_seed(0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF.to_bytes(byteorder="big", length=32), 281474976710655))
+        self.assertEqual(0x56f4008fb007ca9acf0e15b054d5c9fd12ee06cea347914ddbaed70d1c13a528.to_bytes(byteorder="big", length=32),
+                         get_per_commitment_secret_from_seed(0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF.to_bytes(byteorder="big", length=32), 0xaaaaaaaaaaa))
+        self.assertEqual(0x9015daaeb06dba4ccc05b91b2f73bd54405f2be9f217fbacd3c5ac2e62327d31.to_bytes(byteorder="big", length=32),
+                         get_per_commitment_secret_from_seed(0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF.to_bytes(byteorder="big", length=32), 0x555555555555))
+        self.assertEqual(0x915c75942a26bb3a433a8ce2cb0427c29ec6c1775cfc78328b57f6ba7bfeaa9c.to_bytes(byteorder="big", length=32),
+                         get_per_commitment_secret_from_seed(0x0101010101010101010101010101010101010101010101010101010101010101.to_bytes(byteorder="big", length=32), 1))
+
+    def test_key_derivation(self):
+        # BOLT3, Appendix E
+        base_secret = 0x000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f
+        per_commitment_secret = 0x1f1e1d1c1b1a191817161514131211100f0e0d0c0b0a09080706050403020100
+        revocation_basepoint_secret = 0x000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f
+        base_point = secret_to_pubkey(base_secret)
+        self.assertEqual(base_point, bfh('036d6caac248af96f6afa7f904f550253a0f3ef3f5aa2fe6838a95b216691468e2'))
+        per_commitment_point = secret_to_pubkey(per_commitment_secret)
+        self.assertEqual(per_commitment_point, bfh('025f7117a78150fe2ef97db7cfc83bd57b2e2c0d0dd25eaf467a4a1c2a45ce1486'))
+        localpubkey = derive_pubkey(base_point, per_commitment_point)
+        self.assertEqual(localpubkey, bfh('0235f2dbfaa89b57ec7b055afe29849ef7ddfeb1cefdb9ebdc43f5494984db29e5'))
+        localprivkey = derive_privkey(base_secret, per_commitment_point)
+        self.assertEqual(localprivkey, 0xcbced912d3b21bf196a766651e436aff192362621ce317704ea2f75d87e7be0f)
+        revocation_basepoint = secret_to_pubkey(revocation_basepoint_secret)
+        self.assertEqual(revocation_basepoint, bfh('036d6caac248af96f6afa7f904f550253a0f3ef3f5aa2fe6838a95b216691468e2'))
+        revocationpubkey = derive_blinded_pubkey(revocation_basepoint, per_commitment_point)
+        self.assertEqual(revocationpubkey, bfh('02916e326636d19c33f13e8c0c3a03dd157f332f3e99c317c141dd865eb01f8ff0'))
+
+    def test_simple_commitment_tx_with_no_HTLCs(self):
+        to_local_msat = 7000000000
+        to_remote_msat = 3000000000
+        local_feerate_per_kw = 15000
+        # base commitment transaction fee = 10860
+        # actual commitment transaction fee = 10860
+        # to_local amount 6989140 wscript 63210212a140cd0c6539d07cd08dfe09984dec3251ea808b892efeac3ede9402bf2b1967029000b2752103fd5960528dc152014952efdb702a88f71e3c1653b2314431701ec77e57fde83c68ac
+        # to_remote amount 3000000 P2WPKH(0394854aa6eab5b2a8122cc726e9dded053a2184d88256816826d6231c068d4a5b)
+        remote_signature = "3045022100f51d2e566a70ba740fc5d8c0f07b9b93d2ed741c3c0860c613173de7d39e7968022041376d520e9c0e1ad52248ddf4b22e12be8763007df977253ef45a4ca3bdb7c0"
+        # local_signature = 3044022051b75c73198c6deee1a875871c3961832909acd297c6b908d59e3319e5185a46022055c419379c5051a78d00dbbce11b5b664a0c22815fbcc6fcef6b1937c3836939
+        htlcs=[]
+        our_commit_tx = make_commitment(
+            commitment_number,
+            local_funding_pubkey, remote_funding_pubkey, remotepubkey,
+            local_payment_basepoint, remote_payment_basepoint,
+            local_revocation_pubkey, local_delayedpubkey, local_delay,
+            funding_tx_id, funding_output_index, funding_amount_satoshi,
+            to_local_msat, to_remote_msat, local_dust_limit_satoshi,
+            local_feerate_per_kw, True, we_are_initiator=True, htlcs=[])
+        self.sign_and_insert_remote_sig(our_commit_tx, remote_funding_pubkey, remote_signature, local_funding_pubkey, local_funding_privkey)
+        ref_commit_tx_str = '02000000000101bef67e4e2fb9ddeeb3461973cd4c62abb35050b1add772995b820b584a488489000000000038b02b8002c0c62d0000000000160014ccf1af2f2aabee14bb40fa3851ab2301de84311054a56a00000000002200204adb4e2f00643db396dd120d4e7dc17625f5f2c11a40d857accc862d6b7dd80e0400473044022051b75c73198c6deee1a875871c3961832909acd297c6b908d59e3319e5185a46022055c419379c5051a78d00dbbce11b5b664a0c22815fbcc6fcef6b1937c383693901483045022100f51d2e566a70ba740fc5d8c0f07b9b93d2ed741c3c0860c613173de7d39e7968022041376d520e9c0e1ad52248ddf4b22e12be8763007df977253ef45a4ca3bdb7c001475221023da092f6980e58d2c037173180e9a465476026ee50f96695963e8efe436f54eb21030e9f7b623d2ccc7c9bd44d66d5ce21ce504c0acf6385a132cec6d3c39fa711c152ae3e195220'
+        self.assertEqual(str(our_commit_tx), ref_commit_tx_str)
+
+    def sign_and_insert_remote_sig(self, tx, remote_pubkey, remote_signature, pubkey, privkey):
+        assert type(remote_pubkey) is bytes
+        assert len(remote_pubkey) == 33
+        assert type(remote_signature) is str
+        assert type(pubkey) is bytes
+        assert type(privkey) is bytes
+        assert len(pubkey) == 33
+        assert len(privkey) == 33
+        tx.sign({bh2u(pubkey): (privkey[:-1], True)})
+        pubkeys, _x_pubkeys = tx.get_sorted_pubkeys(tx.inputs()[0])
+        index_of_pubkey = pubkeys.index(bh2u(remote_pubkey))
+        tx._inputs[0]["signatures"][index_of_pubkey] = remote_signature + "01"
+        tx.raw = None

lib/transaction.py

@@ -201,7 +201,7 @@ class Enumeration:
 
     def __getattr__(self, attr):
         if attr not in self.lookup:
-            raise AttributeError
+            raise AttributeError(attr)
         return self.lookup[attr]
     def whatis(self, value):
         return self.reverseLookup[value]
@@ -237,7 +237,8 @@ opcodes = Enumeration("Opcodes", [
     "OP_HASH256", "OP_CODESEPARATOR", "OP_CHECKSIG", "OP_CHECKSIGVERIFY", "OP_CHECKMULTISIG",
     "OP_CHECKMULTISIGVERIFY",
     ("OP_NOP1", 0xB0),
-    ("OP_CHECKLOCKTIMEVERIFY", 0xB1), ("OP_CHECKSEQUENCEVERIFY", 0xB2),
+    ("OP_CLTV", 0xB1),
+    ("OP_CSV", 0xB2),
     "OP_NOP4", "OP_NOP5", "OP_NOP6", "OP_NOP7", "OP_NOP8", "OP_NOP9", "OP_NOP10",
     ("OP_INVALIDOPCODE", 0xFF),
 ])
@@ -729,11 +730,12 @@ class Transaction:
         return d
 
     @classmethod
-    def from_io(klass, inputs, outputs, locktime=0):
+    def from_io(klass, inputs, outputs, locktime=0, version=1):
         self = klass(None)
         self._inputs = inputs
         self._outputs = outputs
         self.locktime = locktime
+        self.version = version
         return self
 
     @classmethod

lib/util.py

@@ -435,6 +435,12 @@ def bh2u(x):
     return hfu(x).decode('ascii')
 
 
+def xor_bytes(a: bytes, b: bytes) -> bytes:
+    size = min(len(a), len(b))
+    return ((int.from_bytes(a[:size], "big") ^ int.from_bytes(b[:size], "big"))
+            .to_bytes(size, "big"))
+
+
 def user_dir():
     if 'ANDROID_DATA' in os.environ:
         return android_check_data_dir()

lib/wallet.py

@@ -26,7 +26,7 @@
 #   - Standard_Wallet: one keystore, P2PKH
 #   - Multisig_Wallet: several keystores, P2SH
 
-
+import asyncio
 import os
 import threading
 import random
@@ -506,16 +506,19 @@ class Abstract_Wallet(PrintError):
                 return TX_HEIGHT_LOCAL, 0, None
 
     def get_txpos(self, tx_hash):
-        "return position, even if the tx is unverified"
+        """Return (block_height, tx_pos_in_block).
+
+        If tx is unverified, tx_pos_in_block is -1.
+        """
         with self.lock:
             if tx_hash in self.verified_tx:
                 height, timestamp, pos = self.verified_tx[tx_hash]
                 return height, pos
             elif tx_hash in self.unverified_tx:
                 height = self.unverified_tx[tx_hash]
-                return (height, 0) if height > 0 else ((1e9 - height), 0)
+                return (height, -1) if height > 0 else ((1e9 - height), -1)
             else:
-                return (1e9+1, 0)
+                return (1e9+1, -1)
 
     def is_found(self):
         return self.history.values() != [[]] * len(self.history)
@@ -1304,6 +1307,9 @@ class Abstract_Wallet(PrintError):
             self.verifier = SPV(self.network, self)
             self.synchronizer = Synchronizer(self, network)
             network.add_jobs([self.verifier, self.synchronizer])
+            asyncio.set_event_loop(network.asyncio_loop)
+            from .lnworker import LNWorker
+            self.lnworker = LNWorker(self, network)
         else:
             self.verifier = None
             self.synchronizer = None

setup.py

@@ -55,6 +55,7 @@ setup(
     extras_require=extras_require,
     packages=[
         'electrum',
+        'electrum.lightning_payencode',
         'electrum_gui',
         'electrum_gui.qt',
         'electrum_plugins',