Merge upstream

contrib/build-wine/docker/Dockerfile

@@ -6,8 +6,8 @@ RUN dpkg --add-architecture i386 && \
     apt-get update -q && \
     apt-get install -qy \
         wget=1.19.4-1ubuntu2.1 \
-        gnupg2=2.2.4-1ubuntu1.1 \
+        gnupg2=2.2.4-1ubuntu1.2 \
-        dirmngr=2.2.4-1ubuntu1.1 \
+        dirmngr=2.2.4-1ubuntu1.2 \
         python3-software-properties=0.96.24.32.1 \
         software-properties-common=0.96.24.32.1 \
         && \

contrib/osx/make_osx

@@ -75,7 +75,6 @@ tar xz --directory $BUILDDIR
 cp $BUILDDIR/libusb/1.0.22/lib/libusb-1.0.dylib contrib/osx
 echo "82c368dfd4da017ceb32b12ca885576f325503428a4966cc09302cbd62702493  contrib/osx/libusb-1.0.dylib" | \
     shasum -a 256 -c || fail "libusb checksum mismatched"
-DoCodeSignMaybe "libusb" "contrib/osx/libusb-1.0.dylib" "$APP_SIGN" # If APP_SIGN is empty will be a noop
 
 info "Building libsecp256k1"
 brew install autoconf automake libtool
@@ -88,7 +87,6 @@ git clean -f -x -q
 make
 popd
 cp $BUILDDIR/secp256k1/.libs/libsecp256k1.0.dylib contrib/osx
-DoCodeSignMaybe "libsecp256k1" "contrib/osx/libsecp256k1.0.dylib" "$APP_SIGN" # If APP_SIGN is empty will be a noop
 
 info "Building CalinsQRReader..."
 d=contrib/osx/CalinsQRReader
@@ -120,7 +118,7 @@ for d in ~/Library/Python/ ~/.pyenv .; do
 done
 
 info "Building binary"
-pyinstaller --noconfirm --ascii --clean --name $VERSION contrib/osx/osx.spec || fail "Could not build binary"
+APP_SIGN="$APP_SIGN" pyinstaller --noconfirm --ascii --clean --name $VERSION contrib/osx/osx.spec || fail "Could not build binary"
 
 info "Adding bitcoin URI types to Info.plist"
 plutil -insert 'CFBundleURLTypes' \

contrib/osx/osx.spec

@@ -2,13 +2,50 @@
 
 from PyInstaller.utils.hooks import collect_data_files, collect_submodules, collect_dynamic_libs
 
-import sys
+import sys, os
-import os
 
 PACKAGE='Electrum'
 PYPKG='electrum'
 MAIN_SCRIPT='run_electrum'
 ICONS_FILE='electrum.icns'
+APP_SIGN = os.environ.get('APP_SIGN', '')
+
+def fail(*msg):
+    RED='\033[0;31m'
+    NC='\033[0m' # No Color
+    print("\r🗯 {}ERROR:{}".format(RED, NC), *msg)
+    sys.exit(1)
+
+def codesign(identity, binary):
+    d = os.path.dirname(binary)
+    saved_dir=None
+    if d:
+        # switch to directory of the binary so codesign verbose messages don't include long path
+        saved_dir = os.path.abspath(os.path.curdir)
+        os.chdir(d)
+        binary = os.path.basename(binary)
+    os.system("codesign -v -f -s '{}' '{}'".format(identity, binary))==0 or fail("Could not code sign " + binary)
+    if saved_dir:
+        os.chdir(saved_dir)
+
+def monkey_patch_pyinstaller_for_codesigning(identity):
+    # Monkey-patch PyInstaller so that we app-sign all binaries *after* they are modified by PyInstaller
+    # If we app-sign before that point, the signature will be invalid because PyInstaller modifies
+    # @loader_path in the Mach-O loader table.
+    try:
+        import PyInstaller.depend.dylib
+        _saved_func = PyInstaller.depend.dylib.mac_set_relative_dylib_deps
+    except (ImportError, NameError, AttributeError):
+        # Hmm. Likely wrong PyInstaller version.
+        fail("Could not monkey-patch PyInstaller for code signing. Please ensure that you are using PyInstaller 3.4.")
+    _signed = set()
+    def my_func(fn, distname):
+        _saved_func(fn, distname)
+        if  (fn, distname) not in _signed:
+            codesign(identity, fn)
+            _signed.add((fn,distname)) # remember we signed it so we don't sign again
+    PyInstaller.depend.dylib.mac_set_relative_dylib_deps = my_func
+
 
 for i, x in enumerate(sys.argv):
     if x == '--name':
@@ -90,6 +127,10 @@ for x in a.binaries.copy():
             a.binaries.remove(x)
             print('----> Removed x =', x)
 
+# If code signing, monkey-patch in a code signing step to pyinstaller. See: https://github.com/spesmilo/electrum/issues/4994
+if APP_SIGN:
+    monkey_patch_pyinstaller_for_codesigning(APP_SIGN)
+
 pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher)
 
 exe = EXE(pyz,

contrib/requirements/requirements-hw.txt

@@ -2,7 +2,7 @@ Cython>=0.27
 trezor[hidapi]>=0.11.0
 safet[hidapi]>=0.1.0
 keepkey
-btchip-python
+btchip-python>=0.1.26
 ckcc-protocol>=0.7.2
 websocket-client
 hidapi

electrum/address_synchronizer.py

@@ -373,11 +373,13 @@ class AddressSynchronizer(PrintError):
     @profiler
     def load_transactions(self):
         # load txi, txo, tx_fees
-        self.txi = self.storage.get('txi', {})
+        # bookkeeping data of is_mine inputs of transactions
+        self.txi = self.storage.get('txi', {})  # txid -> address -> (prev_outpoint, value)
         for txid, d in list(self.txi.items()):
             for addr, lst in d.items():
                 self.txi[txid][addr] = set([tuple(x) for x in lst])
-        self.txo = self.storage.get('txo', {})
+        # bookkeeping data of is_mine outputs of transactions
+        self.txo = self.storage.get('txo', {})  # txid -> address -> (output_index, value, is_coinbase)
         self.tx_fees = self.storage.get('tx_fees', {})
         tx_list = self.storage.get('transactions', {})
         # load transactions

electrum/base_wizard.py

@@ -233,16 +233,23 @@ class BaseWizard(object):
         title = _('Hardware Keystore')
         # check available plugins
         supported_plugins = self.plugins.get_hardware_support()
-        # scan devices
         devices = []  # type: List[Tuple[str, DeviceInfo]]
         devmgr = self.plugins.device_manager
+        debug_msg = ''
+
+        def failed_getting_device_infos(name, e):
+            nonlocal debug_msg
+            devmgr.print_error(f'error getting device infos for {name}: {e}')
+            indented_error_msg = '    '.join([''] + str(e).splitlines(keepends=True))
+            debug_msg += f'  {name}: (error getting device infos)\n{indented_error_msg}\n'
+
+        # scan devices
         try:
             scanned_devices = devmgr.scan_devices()
         except BaseException as e:
-            devmgr.print_error('error scanning devices: {}'.format(e))
+            devmgr.print_error('error scanning devices: {}'.format(repr(e)))
             debug_msg = '  {}:\n    {}'.format(_('Error scanning devices'), e)
         else:
-            debug_msg = ''
             for splugin in supported_plugins:
                 name, plugin = splugin.name, splugin.plugin
                 # plugin init errored?
@@ -256,14 +263,17 @@ class BaseWizard(object):
                 # see if plugin recognizes 'scanned_devices'
                 try:
                     # FIXME: side-effect: unpaired_device_info sets client.handler
-                    u = devmgr.unpaired_device_infos(None, plugin, devices=scanned_devices)
+                    device_infos = devmgr.unpaired_device_infos(None, plugin, devices=scanned_devices,
+                                                                include_failing_clients=True)
                 except BaseException as e:
                     traceback.print_exc()
-                    devmgr.print_error(f'error getting device infos for {name}: {e}')
+                    failed_getting_device_infos(name, e)
-                    indented_error_msg = '    '.join([''] + str(e).splitlines(keepends=True))
-                    debug_msg += f'  {name}: (error getting device infos)\n{indented_error_msg}\n'
                     continue
-                devices += list(map(lambda x: (name, x), u))
+                device_infos_failing = list(filter(lambda di: di.exception is not None, device_infos))
+                for di in device_infos_failing:
+                    failed_getting_device_infos(name, di.exception)
+                device_infos_working = list(filter(lambda di: di.exception is None, device_infos))
+                devices += list(map(lambda x: (name, x), device_infos_working))
         if not debug_msg:
             debug_msg = '  {}'.format(_('No exceptions encountered.'))
         if not devices:

electrum/blockchain.py

@@ -476,6 +476,11 @@ class Blockchain(util.PrintError):
             return None
         return deserialize_header(h, height)
 
+    def header_at_tip(self) -> Optional[dict]:
+        """Return latest header."""
+        height = self.height()
+        return self.read_header(height)
+
     def get_hash(self, height: int) -> str:
         def is_height_checkpoint():
             within_cp_range = height <= constants.net.max_checkpoint()

electrum/commands.py

@@ -38,6 +38,7 @@ from .import util, ecc
 from .util import bfh, bh2u, format_satoshis, json_decode, print_error, json_encode
 from . import bitcoin
 from .bitcoin import is_address,  hash_160, COIN, TYPE_ADDRESS
+from . import bip32
 from .i18n import _
 from .transaction import Transaction, multisig_script, TxOutput
 from .paymentrequest import PR_PAID, PR_UNPAID, PR_UNKNOWN, PR_EXPIRED
@@ -329,7 +330,8 @@ class Commands:
     def broadcast(self, tx):
         """Broadcast a transaction to the network. """
         tx = Transaction(tx)
-        return self.network.run_from_another_thread(self.network.broadcast_transaction(tx))
+        self.network.run_from_another_thread(self.network.broadcast_transaction(tx))
+        return tx.txid()
 
     @command('')
     def createmultisig(self, num, pubkeys):
@@ -428,6 +430,16 @@ class Commands:
         """Get master private key. Return your wallet\'s master private key"""
         return str(self.wallet.keystore.get_master_private_key(password))
 
+    @command('')
+    def convert_xkey(self, xkey, xtype):
+        """Convert xtype of a master key. e.g. xpub -> ypub"""
+        is_xprv = bip32.is_xprv(xkey)
+        if not bip32.is_xpub(xkey) and not is_xprv:
+            raise Exception('xkey should be a master public/private key')
+        _, depth, fingerprint, child_number, c, cK = bip32.deserialize_xkey(xkey, is_xprv)
+        serialize = bip32.serialize_xprv if is_xprv else bip32.serialize_xpub
+        return serialize(xtype, c, cK, depth, fingerprint, child_number)
+
     @command('wp')
     def getseed(self, password=None):
         """Get seed phrase. Print the generation seed of your wallet."""
@@ -909,6 +921,7 @@ def add_network_options(parser):
     parser.add_argument("-s", "--server", dest="server", default=None, help="set server host:port:protocol, where protocol is either t (tcp) or s (ssl)")
     parser.add_argument("-p", "--proxy", dest="proxy", default=None, help="set proxy [type:]host[:port], where type is socks4,socks5 or http")
     parser.add_argument("--noonion", action="store_true", dest="noonion", default=None, help="do not try to connect to onion servers")
+    parser.add_argument("--skipmerklecheck", action="store_true", dest="skipmerklecheck", default=False, help="Tolerate invalid merkle proofs from server")
 
 def add_global_options(parser):
     group = parser.add_argument_group('global options')

electrum/gui/kivy/main_window.py

@@ -16,7 +16,7 @@ from electrum.plugin import run_hook
 from electrum.util import format_satoshis, format_satoshis_plain
 from electrum.paymentrequest import PR_UNPAID, PR_PAID, PR_UNKNOWN, PR_EXPIRED
 from electrum import blockchain
-from electrum.network import Network
+from electrum.network import Network, TxBroadcastError, BestEffortRequestFailed
 from .i18n import _
 
 from kivy.app import App
@@ -917,14 +917,16 @@ class ElectrumWindow(App):
         Clock.schedule_once(lambda dt: on_success(tx))
 
     def _broadcast_thread(self, tx, on_complete):
-
+        status = False
         try:
             self.network.run_from_another_thread(self.network.broadcast_transaction(tx))
-        except Exception as e:
+        except TxBroadcastError as e:
-            ok, msg = False, repr(e)
+            msg = e.get_message_for_gui()
+        except BestEffortRequestFailed as e:
+            msg = repr(e)
         else:
-            ok, msg = True, tx.txid()
+            status, msg = True, tx.txid()
-        Clock.schedule_once(lambda dt: on_complete(ok, msg))
+        Clock.schedule_once(lambda dt: on_complete(status, msg))
 
     def broadcast(self, tx, pr=None):
         def on_complete(ok, msg):
@@ -937,11 +939,8 @@ class ElectrumWindow(App):
                     self.wallet.invoices.save()
                     self.update_tab('invoices')
             else:
-                display_msg = _('The server returned an error when broadcasting the transaction.')
+                msg = msg or ''
-                if msg:
+                self.show_error(msg)
-                    display_msg += '\n' + msg
-                display_msg = display_msg[:500]
-                self.show_error(display_msg)
 
         if self.network and self.network.is_connected():
             self.show_info(_('Sending'))

electrum/gui/qt/__init__.py

@@ -133,7 +133,7 @@ class ElectrumGui(PrintError):
                 self.app.setStyleSheet(qdarkstyle.load_stylesheet_pyqt5())
             except BaseException as e:
                 use_dark_theme = False
-                self.print_error('Error setting dark theme: {}'.format(e))
+                self.print_error('Error setting dark theme: {}'.format(repr(e)))
         # Even if we ourselves don't set the dark theme,
         # the OS/window manager/etc might set *a dark theme*.
         # Hence, try to choose colors accordingly:

electrum/gui/qt/contact_list.py

@@ -60,20 +60,20 @@ class ContactList(MyTreeView):
 
     def create_menu(self, position):
         menu = QMenu()
-        selected = self.selected_in_column(0)
-        selected_keys = []
-        for idx in selected:
-            sel_key = self.model().itemFromIndex(idx).data(Qt.UserRole)
-            selected_keys.append(sel_key)
         idx = self.indexAt(position)
+        column = idx.column() or 0
+        selected = self.selected_in_column(column)
+        selected_keys = []
+        for s_idx in selected:
+            sel_key = self.model().itemFromIndex(s_idx).data(Qt.UserRole)
+            selected_keys.append(sel_key)
         if not selected or not idx.isValid():
             menu.addAction(_("New contact"), lambda: self.parent.new_contact_dialog())
             menu.addAction(_("Import file"), lambda: self.import_contacts())
             menu.addAction(_("Export file"), lambda: self.export_contacts())
         else:
-            column = idx.column()
             column_title = self.model().horizontalHeaderItem(column).text()
-            column_data = '\n'.join(self.model().itemFromIndex(idx).text() for idx in selected)
+            column_data = '\n'.join(self.model().itemFromIndex(s_idx).text() for s_idx in selected)
             menu.addAction(_("Copy {}").format(column_title), lambda: self.parent.app.clipboard().setText(column_data))
             if column in self.editable_columns:
                 item = self.model().itemFromIndex(idx)
@@ -107,4 +107,6 @@ class ContactList(MyTreeView):
                 idx = self.model().index(row_count, 0)
                 set_current = QPersistentModelIndex(idx)
         self.set_current_idx(set_current)
+        # FIXME refresh loses sort order; so set "default" here:
+        self.sortByColumn(0, Qt.AscendingOrder)
         run_hook('update_contacts_tab', self)

electrum/gui/qt/history_list.py

@@ -29,6 +29,7 @@ from datetime import date
 from typing import TYPE_CHECKING, Tuple, Dict
 import threading
 from enum import IntEnum
+from decimal import Decimal
 
 from electrum.address_synchronizer import TX_HEIGHT_LOCAL
 from electrum.i18n import _
@@ -77,9 +78,14 @@ class HistorySortModel(QSortFilterProxyModel):
         item2 = self.sourceModel().data(source_right, Qt.UserRole)
         if item1 is None or item2 is None:
             raise Exception(f'UserRole not set for column {source_left.column()}')
-        if item1.value() is None or item2.value() is None:
+        v1 = item1.value()
+        v2 = item2.value()
+        if v1 is None or isinstance(v1, Decimal) and v1.is_nan(): v1 = -float("inf")
+        if v2 is None or isinstance(v2, Decimal) and v2.is_nan(): v2 = -float("inf")
+        try:
+            return v1 < v2
+        except:
             return False
-        return item1.value() < item2.value()
 
 class HistoryModel(QAbstractItemModel, PrintError):
 

electrum/gui/qt/main_window.py

@@ -62,7 +62,7 @@ from electrum.address_synchronizer import AddTransactionException
 from electrum.wallet import (Multisig_Wallet, CannotBumpFee, Abstract_Wallet,
                              sweep_preparations, InternalAddressCorruption)
 from electrum.version import ELECTRUM_VERSION
-from electrum.network import Network
+from electrum.network import Network, TxBroadcastError, BestEffortRequestFailed
 from electrum.exchange_rate import FxThread
 from electrum.simple_config import SimpleConfig
 
@@ -255,7 +255,7 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError):
     def toggle_tab(self, tab):
         show = not self.config.get('show_{}_tab'.format(tab.tab_name), False)
         self.config.set_key('show_{}_tab'.format(tab.tab_name), show)
-        item_text = (_("Hide") if show else _("Show")) + " " + tab.tab_description
+        item_text = (_("Hide {}") if show else _("Show {}")).format(tab.tab_description)
         tab.menu_action.setText(item_text)
         if show:
             # Find out where to place the tab
@@ -1080,7 +1080,7 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError):
         uri = util.create_URI(addr, amount, message)
         self.receive_qr.setData(uri)
         if self.qr_window and self.qr_window.isVisible():
-            self.qr_window.set_content(addr, amount, message, uri)
+            self.qr_window.qrw.setData(uri)
 
     def set_feerounding_text(self, num_satoshis_added):
         self.feerounding_text = (_('Additional {} satoshis are going to be added.')
@@ -1667,10 +1667,13 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError):
             if pr and pr.has_expired():
                 self.payment_request = None
                 return False, _("Payment request has expired")
+            status = False
             try:
                 self.network.run_from_another_thread(self.network.broadcast_transaction(tx))
-            except Exception as e:
+            except TxBroadcastError as e:
-                status, msg = False, repr(e)
+                msg = e.get_message_for_gui()
+            except BestEffortRequestFailed as e:
+                msg = repr(e)
             else:
                 status, msg = True, tx.txid()
             if pr and status is True:
@@ -1698,10 +1701,8 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError):
                     self.invoice_list.update()
                     self.do_clear()
                 else:
-                    display_msg = _('The server returned an error when broadcasting the transaction.')
+                    msg = msg or ''
-                    if msg:
+                    parent.show_error(msg)
-                        display_msg += '\n' + msg
-                    parent.show_error(display_msg)
 
         WaitingDialog(self, _('Broadcasting transaction...'),
                       broadcast_thread, broadcast_done, self.on_error)
@@ -2416,7 +2417,7 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError):
         try:
             data = bh2u(bitcoin.base_decode(data, length=None, base=43))
         except BaseException as e:
-            self.show_error((_('Could not decode QR code')+':\n{}').format(e))
+            self.show_error((_('Could not decode QR code')+':\n{}').format(repr(e)))
             return
         tx = self.tx_from_text(data)
         if not tx:

electrum/gui/qt/network_dialog.py

@@ -466,6 +466,9 @@ class NetworkChoiceLayout(object):
         self.network.run_from_another_thread(self.network.set_parameters(net_params))
 
     def suggest_proxy(self, found_proxy):
+        if found_proxy is None:
+            self.tor_cb.hide()
+            return
         self.tor_proxy = found_proxy
         self.tor_cb.setText("Use Tor proxy at port " + str(found_proxy[1]))
         if self.proxy_mode.currentIndex() == self.proxy_mode.findText('SOCKS5') \
@@ -505,10 +508,14 @@ class TorDetector(QThread):
     def run(self):
         # Probable ports for Tor to listen at
         ports = [9050, 9150]
-        for p in ports:
+        while True:
-            if TorDetector.is_tor_port(p):
+            for p in ports:
-                self.found_proxy.emit(("127.0.0.1", p))
+                if TorDetector.is_tor_port(p):
-                return
+                    self.found_proxy.emit(("127.0.0.1", p))
+                    break
+            else:
+                self.found_proxy.emit(None)
+            time.sleep(10)
 
     @staticmethod
     def is_tor_port(port):

electrum/gui/qt/qrcodewidget.py

@@ -66,16 +66,15 @@ class QRCodeWidget(QWidget):
         framesize = min(r.width(), r.height())
         boxsize = int( (framesize - 2*margin)/k )
         size = k*boxsize
-        left = (r.width() - size)/2
+        left = (framesize - size)/2
-        top = (r.height() - size)/2
+        top = (framesize - size)/2
-
+        # Draw white background with margin
-        # Make a white margin around the QR in case of dark theme use
         qp.setBrush(white)
         qp.setPen(white)
-        qp.drawRect(left-margin, top-margin, size+(margin*2), size+(margin*2))
+        qp.drawRect(0, 0, framesize, framesize)
+        # Draw qr code
         qp.setBrush(black)
         qp.setPen(black)
-
         for r in range(k):
             for c in range(k):
                 if matrix[r][c]:

electrum/gui/qt/qrwindow.py

@@ -48,43 +48,9 @@ class QR_Window(QWidget):
         QWidget.__init__(self)
         self.win = win
         self.setWindowTitle('Electrum - '+_('Payment Request'))
-        self.setMinimumSize(800, 250)
+        self.setMinimumSize(800, 800)
-        self.address = ''
-        self.label = ''
-        self.amount = 0
         self.setFocusPolicy(Qt.NoFocus)
-
         main_box = QHBoxLayout()
-
         self.qrw = QRCodeWidget()
         main_box.addWidget(self.qrw, 1)
-
-        vbox = QVBoxLayout()
-        main_box.addLayout(vbox)
-
-        self.address_label = QLabel("")
-        #self.address_label.setFont(QFont(MONOSPACE_FONT))
-        vbox.addWidget(self.address_label)
-
-        self.label_label = QLabel("")
-        vbox.addWidget(self.label_label)
-
-        self.amount_label = QLabel("")
-        vbox.addWidget(self.amount_label)
-
-        vbox.addStretch(1)
         self.setLayout(main_box)
-
-
-    def set_content(self, address, amount, message, url):
-        address_text = "<span style='font-size: 18pt'>%s</span>" % address if address else ""
-        self.address_label.setText(address_text)
-        if amount:
-            amount = self.win.format_amount(amount)
-            amount_text = "<span style='font-size: 21pt'>%s</span> <span style='font-size: 16pt'>%s</span> " % (amount, self.win.base_unit())
-        else:
-            amount_text = ''
-        self.amount_label.setText(amount_text)
-        label_text = "<span style='font-size: 21pt'>%s</span>" % message if message else ""
-        self.label_label.setText(label_text)
-        self.qrw.setData(url)

electrum/gui/stdio.py

@@ -6,6 +6,7 @@ from electrum import WalletStorage, Wallet
 from electrum.util import format_satoshis, set_verbosity
 from electrum.bitcoin import is_address, COIN, TYPE_ADDRESS
 from electrum.transaction import TxOutput
+from electrum.network import TxBroadcastError, BestEffortRequestFailed
 
 _ = lambda x:x  # i18n
 
@@ -205,10 +206,12 @@ class ElectrumGui:
         print(_("Please wait..."))
         try:
             self.network.run_from_another_thread(self.network.broadcast_transaction(tx))
-        except Exception as e:
+        except TxBroadcastError as e:
-            display_msg = _('The server returned an error when broadcasting the transaction.')
+            msg = e.get_message_for_gui()
-            display_msg += '\n' + repr(e)
+            print(msg)
-            print(display_msg)
+        except BestEffortRequestFailed as e:
+            msg = repr(e)
+            print(msg)
         else:
             print(_('Payment sent.'))
             #self.do_clear()

electrum/gui/text.py

@@ -12,7 +12,7 @@ from electrum.bitcoin import is_address, COIN, TYPE_ADDRESS
 from electrum.transaction import TxOutput
 from electrum.wallet import Wallet
 from electrum.storage import WalletStorage
-from electrum.network import NetworkParameters
+from electrum.network import NetworkParameters, TxBroadcastError, BestEffortRequestFailed
 from electrum.interface import deserialize_server
 
 _ = lambda x:x  # i18n
@@ -369,10 +369,12 @@ class ElectrumGui:
         self.show_message(_("Please wait..."), getchar=False)
         try:
             self.network.run_from_another_thread(self.network.broadcast_transaction(tx))
-        except Exception as e:
+        except TxBroadcastError as e:
-            display_msg = _('The server returned an error when broadcasting the transaction.')
+            msg = e.get_message_for_gui()
-            display_msg += '\n' + repr(e)
+            self.show_message(msg)
-            self.show_message(display_msg)
+        except BestEffortRequestFailed as e:
+            msg = repr(e)
+            self.show_message(msg)
         else:
             self.show_message(_('Payment sent.'))
             self.do_clear()

electrum/interface.py

@@ -207,7 +207,7 @@ class Interface(PrintError):
     async def _try_saving_ssl_cert_for_first_time(self, ca_ssl_context):
         try:
             ca_signed = await self.is_server_ca_signed(ca_ssl_context)
-        except (OSError, aiorpcx.socks.SOCKSFailure) as e:
+        except (OSError, aiorpcx.socks.SOCKSError) as e:
             raise ErrorGettingSSLCertFromServer(e) from e
         if ca_signed:
             with open(self.cert_path, 'w') as f:
@@ -267,7 +267,7 @@ class Interface(PrintError):
             try:
                 return await func(self, *args, **kwargs)
             except GracefulDisconnect as e:
-                self.print_error("disconnecting gracefully. {}".format(e))
+                self.print_error("disconnecting gracefully. {}".format(repr(e)))
             finally:
                 await self.network.connection_down(self)
                 self.got_disconnected.set_result(1)
@@ -284,7 +284,7 @@ class Interface(PrintError):
             return
         try:
             await self.open_session(ssl_context)
-        except (asyncio.CancelledError, OSError, aiorpcx.socks.SOCKSFailure) as e:
+        except (asyncio.CancelledError, OSError, aiorpcx.socks.SOCKSError) as e:
             self.print_error('disconnecting due to: {}'.format(repr(e)))
             return
 

electrum/network.py

@@ -37,6 +37,7 @@ import traceback
 
 import dns
 import dns.resolver
+import aiorpcx
 from aiorpcx import TaskGroup
 from aiohttp import ClientResponse
 
@@ -53,6 +54,7 @@ from .interface import (Interface, serialize_server, deserialize_server,
                         RequestTimedOut, NetworkTimeout)
 from .version import PROTOCOL_VERSION
 from .simple_config import SimpleConfig
+from .i18n import _
 
 NODES_RETRY_INTERVAL = 60
 SERVER_RETRY_INTERVAL = 10
@@ -162,6 +164,30 @@ def deserialize_proxy(s: str) -> Optional[dict]:
     return proxy
 
 
+class BestEffortRequestFailed(Exception): pass
+
+
+class TxBroadcastError(Exception):
+    def get_message_for_gui(self):
+        raise NotImplementedError()
+
+
+class TxBroadcastHashMismatch(TxBroadcastError):
+    def get_message_for_gui(self):
+        return "{}\n{}\n\n{}" \
+            .format(_("The server returned an unexpected transaction ID when broadcasting the transaction."),
+                    _("Consider trying to connect to a different server, or updating Electrum."),
+                    str(self))
+
+
+class TxBroadcastServerReturnedError(TxBroadcastError):
+    def get_message_for_gui(self):
+        return "{}\n{}\n\n{}" \
+            .format(_("The server returned an error when broadcasting the transaction."),
+                    _("Consider trying to connect to a different server, or updating Electrum."),
+                    str(self))
+
+
 INSTANCE = None
 
 
@@ -724,7 +750,7 @@ class Network(PrintError):
                             continue  # try again
                     return success_fut.result()
                 # otherwise; try again
-            raise Exception('no interface to do request on... gave up.')
+            raise BestEffortRequestFailed('no interface to do request on... gave up.')
         return make_reliable_wrapper
 
     @best_effort_reliable
@@ -732,14 +758,152 @@ class Network(PrintError):
         return await self.interface.session.send_request('blockchain.transaction.get_merkle', [tx_hash, tx_height])
 
     @best_effort_reliable
-    async def broadcast_transaction(self, tx, *, timeout=None):
+    async def broadcast_transaction(self, tx, *, timeout=None) -> None:
         if timeout is None:
             timeout = self.get_network_timeout_seconds(NetworkTimeout.Urgent)
-        out = await self.interface.session.send_request('blockchain.transaction.broadcast', [str(tx)], timeout=timeout)
+        try:
+            out = await self.interface.session.send_request('blockchain.transaction.broadcast', [str(tx)], timeout=timeout)
+            # note: both 'out' and exception messages are untrusted input from the server
+        except aiorpcx.jsonrpc.RPCError as e:
+            self.print_error(f"broadcast_transaction error: {repr(e)}")
+            raise TxBroadcastServerReturnedError(self.sanitize_tx_broadcast_response(e.message)) from e
         if out != tx.txid():
-            # note: this is untrusted input from the server
+            self.print_error(f"unexpected txid for broadcast_transaction: {out} != {tx.txid()}")
-            raise Exception(out)
+            raise TxBroadcastHashMismatch(_("Server returned unexpected transaction ID."))
-        return out  # txid
+
+    @staticmethod
+    def sanitize_tx_broadcast_response(server_msg) -> str:
+        # Unfortunately, bitcoind and hence the Electrum protocol doesn't return a useful error code.
+        # So, we use substring matching to grok the error message.
+        # server_msg is untrusted input so it should not be shown to the user. see #4968
+        server_msg = str(server_msg)
+        server_msg = server_msg.replace("\n", r"\n")
+        # https://github.com/bitcoin/bitcoin/blob/cd42553b1178a48a16017eff0b70669c84c3895c/src/policy/policy.cpp
+        # grep "reason ="
+        policy_error_messages = {
+            r"version": _("Transaction uses non-standard version."),
+            r"tx-size": _("The transaction was rejected because it is too large."),
+            r"scriptsig-size": None,
+            r"scriptsig-not-pushonly": None,
+            r"scriptpubkey": None,
+            r"bare-multisig": None,
+            r"dust": _("Transaction could not be broadcast due to dust outputs."),
+            r"multi-op-return": _("The transaction was rejected because it contains multiple OP_RETURN outputs."),
+        }
+        for substring in policy_error_messages:
+            if substring in server_msg:
+                msg = policy_error_messages[substring]
+                return msg if msg else substring
+        # https://github.com/bitcoin/bitcoin/blob/cd42553b1178a48a16017eff0b70669c84c3895c/src/script/script_error.cpp
+        script_error_messages = {
+            r"Script evaluated without error but finished with a false/empty top stack element",
+            r"Script failed an OP_VERIFY operation",
+            r"Script failed an OP_EQUALVERIFY operation",
+            r"Script failed an OP_CHECKMULTISIGVERIFY operation",
+            r"Script failed an OP_CHECKSIGVERIFY operation",
+            r"Script failed an OP_NUMEQUALVERIFY operation",
+            r"Script is too big",
+            r"Push value size limit exceeded",
+            r"Operation limit exceeded",
+            r"Stack size limit exceeded",
+            r"Signature count negative or greater than pubkey count",
+            r"Pubkey count negative or limit exceeded",
+            r"Opcode missing or not understood",
+            r"Attempted to use a disabled opcode",
+            r"Operation not valid with the current stack size",
+            r"Operation not valid with the current altstack size",
+            r"OP_RETURN was encountered",
+            r"Invalid OP_IF construction",
+            r"Negative locktime",
+            r"Locktime requirement not satisfied",
+            r"Signature hash type missing or not understood",
+            r"Non-canonical DER signature",
+            r"Data push larger than necessary",
+            r"Only non-push operators allowed in signatures",
+            r"Non-canonical signature: S value is unnecessarily high",
+            r"Dummy CHECKMULTISIG argument must be zero",
+            r"OP_IF/NOTIF argument must be minimal",
+            r"Signature must be zero for failed CHECK(MULTI)SIG operation",
+            r"NOPx reserved for soft-fork upgrades",
+            r"Witness version reserved for soft-fork upgrades",
+            r"Public key is neither compressed or uncompressed",
+            r"Extra items left on stack after execution",
+            r"Witness program has incorrect length",
+            r"Witness program was passed an empty witness",
+            r"Witness program hash mismatch",
+            r"Witness requires empty scriptSig",
+            r"Witness requires only-redeemscript scriptSig",
+            r"Witness provided for non-witness script",
+            r"Using non-compressed keys in segwit",
+            r"Using OP_CODESEPARATOR in non-witness script",
+            r"Signature is found in scriptCode",
+        }
+        for substring in script_error_messages:
+            if substring in server_msg:
+                return substring
+        # https://github.com/bitcoin/bitcoin/blob/cd42553b1178a48a16017eff0b70669c84c3895c/src/validation.cpp
+        # grep "REJECT_"
+        # should come after script_error.cpp (due to e.g. non-mandatory-script-verify-flag)
+        validation_error_messages = {
+            r"coinbase",
+            r"tx-size-small",
+            r"non-final",
+            r"txn-already-in-mempool",
+            r"txn-mempool-conflict",
+            r"txn-already-known",
+            r"non-BIP68-final",
+            r"bad-txns-nonstandard-inputs",
+            r"bad-witness-nonstandard",
+            r"bad-txns-too-many-sigops",
+            r"mempool min fee not met",
+            r"min relay fee not met",
+            r"absurdly-high-fee",
+            r"too-long-mempool-chain",
+            r"bad-txns-spends-conflicting-tx",
+            r"insufficient fee",
+            r"too many potential replacements",
+            r"replacement-adds-unconfirmed",
+            r"mempool full",
+            r"non-mandatory-script-verify-flag",
+            r"mandatory-script-verify-flag-failed",
+        }
+        for substring in validation_error_messages:
+            if substring in server_msg:
+                return substring
+        # https://github.com/bitcoin/bitcoin/blob/cd42553b1178a48a16017eff0b70669c84c3895c/src/rpc/rawtransaction.cpp
+        # grep "RPC_TRANSACTION"
+        # grep "RPC_DESERIALIZATION_ERROR"
+        rawtransaction_error_messages = {
+            r"Missing inputs",
+            r"transaction already in block chain",
+            r"TX decode failed",
+        }
+        for substring in rawtransaction_error_messages:
+            if substring in server_msg:
+                return substring
+        # https://github.com/bitcoin/bitcoin/blob/cd42553b1178a48a16017eff0b70669c84c3895c/src/consensus/tx_verify.cpp
+        # grep "REJECT_"
+        tx_verify_error_messages = {
+            r"bad-txns-vin-empty",
+            r"bad-txns-vout-empty",
+            r"bad-txns-oversize",
+            r"bad-txns-vout-negative",
+            r"bad-txns-vout-toolarge",
+            r"bad-txns-txouttotal-toolarge",
+            r"bad-txns-inputs-duplicate",
+            r"bad-cb-length",
+            r"bad-txns-prevout-null",
+            r"bad-txns-inputs-missingorspent",
+            r"bad-txns-premature-spend-of-coinbase",
+            r"bad-txns-inputvalues-outofrange",
+            r"bad-txns-in-belowout",
+            r"bad-txns-fee-outofrange",
+        }
+        for substring in tx_verify_error_messages:
+            if substring in server_msg:
+                return substring
+        # otherwise:
+        return _("Unknown error")
 
     @best_effort_reliable
     async def request_chunk(self, height, tip=None, *, can_return_early=False):

electrum/plugin.py

@@ -301,8 +301,9 @@ class Device(NamedTuple):
 
 class DeviceInfo(NamedTuple):
     device: Device
-    label: str
+    label: Optional[str] = None
-    initialized: bool
+    initialized: Optional[bool] = None
+    exception: Optional[Exception] = None
 
 
 class HardwarePluginToScan(NamedTuple):
@@ -500,7 +501,8 @@ class DeviceMgr(ThreadJob, PrintError):
               'its seed (and passphrase, if any).  Otherwise all FLO you '
               'receive will be unspendable.').format(plugin.device))
 
-    def unpaired_device_infos(self, handler, plugin: 'HW_PluginBase', devices=None):
+    def unpaired_device_infos(self, handler, plugin: 'HW_PluginBase', devices=None,
+                              include_failing_clients=False):
         '''Returns a list of DeviceInfo objects: one for each connected,
         unpaired device accepted by the plugin.'''
         if not plugin.libraries_available:
@@ -515,14 +517,16 @@ class DeviceMgr(ThreadJob, PrintError):
                 continue
             try:
                 client = self.create_client(device, handler, plugin)
-            except UserFacingException:
+            except Exception as e:
-                raise
-            except BaseException as e:
                 self.print_error(f'failed to create client for {plugin.name} at {device.path}: {repr(e)}')
+                if include_failing_clients:
+                    infos.append(DeviceInfo(device=device, exception=e))
                 continue
             if not client:
                 continue
-            infos.append(DeviceInfo(device, client.label(), client.is_initialized()))
+            infos.append(DeviceInfo(device=device,
+                                    label=client.label(),
+                                    initialized=client.is_initialized()))
 
         return infos
 

electrum/plugins/email_requests/qt.py

@@ -91,7 +91,7 @@ class Processor(threading.Thread, PrintError):
                 self.M = imaplib.IMAP4_SSL(self.imap_server)
                 self.M.login(self.username, self.password)
             except BaseException as e:
-                self.print_error('connecting failed: {}'.format(e))
+                self.print_error('connecting failed: {}'.format(repr(e)))
                 self.connect_wait *= 2
             else:
                 self.reset_connect_wait()
@@ -100,7 +100,7 @@ class Processor(threading.Thread, PrintError):
                 try:
                     self.poll()
                 except BaseException as e:
-                    self.print_error('polling failed: {}'.format(e))
+                    self.print_error('polling failed: {}'.format(repr(e)))
                     break
                 time.sleep(self.polling_interval)
             time.sleep(random.randint(0, self.connect_wait))

electrum/plugins/keepkey/qt.py

@@ -432,7 +432,7 @@ class SettingsDialog(WindowModalDialog):
 
         def slider_moved():
             mins = timeout_slider.sliderPosition()
-            timeout_minutes.setText(_("%2d minutes") % mins)
+            timeout_minutes.setText(_("{:2d} minutes").format(mins))
 
         def slider_released():
             config.set_session_timeout(timeout_slider.sliderPosition() * 60)

electrum/plugins/ledger/ledger.py

@@ -440,7 +440,7 @@ class Ledger_KeyStore(Hardware_KeyStore):
             self.get_client().enableAlternate2fa(False)
             if segwitTransaction:
                 self.get_client().startUntrustedTransaction(True, inputIndex,
-                                                            chipInputs, redeemScripts[inputIndex])
+                                                            chipInputs, redeemScripts[inputIndex], version=tx.version)
                 # we don't set meaningful outputAddress, amount and fees
                 # as we only care about the alternateEncoding==True branch
                 outputData = self.get_client().finalizeInput(b'', 0, 0, changePath, bfh(rawTx))
@@ -456,7 +456,7 @@ class Ledger_KeyStore(Hardware_KeyStore):
                 while inputIndex < len(inputs):
                     singleInput = [ chipInputs[inputIndex] ]
                     self.get_client().startUntrustedTransaction(False, 0,
-                                                            singleInput, redeemScripts[inputIndex])
+                                                            singleInput, redeemScripts[inputIndex], version=tx.version)
                     inputSignature = self.get_client().untrustedHashSign(inputsPaths[inputIndex], pin, lockTime=tx.locktime)
                     inputSignature[0] = 0x30 # force for 1.4.9+
                     signatures.append(inputSignature)
@@ -464,7 +464,7 @@ class Ledger_KeyStore(Hardware_KeyStore):
             else:
                 while inputIndex < len(inputs):
                     self.get_client().startUntrustedTransaction(firstTransaction, inputIndex,
-                                                                chipInputs, redeemScripts[inputIndex])
+                                                                chipInputs, redeemScripts[inputIndex], version=tx.version)
                     # we don't set meaningful outputAddress, amount and fees
                     # as we only care about the alternateEncoding==True branch
                     outputData = self.get_client().finalizeInput(b'', 0, 0, changePath, bfh(rawTx))

electrum/plugins/revealer/qt.py

@@ -113,7 +113,7 @@ class Plugin(RevealerPlugin):
         self.load_noise.textChanged.connect(self.on_edit)
         self.load_noise.setMaximumHeight(33)
         self.hbox.addLayout(vbox)
-        vbox.addWidget(WWLabel(_("or type a existing revealer code below and click 'next':")))
+        vbox.addWidget(WWLabel(_("or type an existing revealer code below and click 'next':")))
         vbox.addWidget(self.load_noise)
         vbox.addSpacing(3)
         self.next_button = QPushButton(_("Next"), self.d)
@@ -170,7 +170,7 @@ class Plugin(RevealerPlugin):
         code_id = self.versioned_seed.checksum
         dialog.show_message(''.join([_("{} encrypted for Revealer {}_{} saved as PNG and PDF at: ").format(self.was, version, code_id),
                                      "<b>", self.get_path_to_revealer_file(), "</b>", "<br/>",
-                                     "<br/>", "<b>", _("Always check you backups.")]),
+                                     "<br/>", "<b>", _("Always check your backups.")]),
                             rich_text=True)
         dialog.close()
 

electrum/plugins/safe_t/qt.py

@@ -111,7 +111,7 @@ class QtPlugin(QtPluginBase):
             bg = QButtonGroup()
             for i, count in enumerate([12, 18, 24]):
                 rb = QRadioButton(gb)
-                rb.setText(_("%d words") % count)
+                rb.setText(_("{:d} words").format(count))
                 bg.addButton(rb)
                 bg.setId(rb, i)
                 hbox1.addWidget(rb)
@@ -317,7 +317,7 @@ class SettingsDialog(WindowModalDialog):
 
         def slider_moved():
             mins = timeout_slider.sliderPosition()
-            timeout_minutes.setText(_("%2d minutes") % mins)
+            timeout_minutes.setText(_("{:2d} minutes").format(mins))
 
         def slider_released():
             config.set_session_timeout(timeout_slider.sliderPosition() * 60)

electrum/plugins/trezor/clientbase.py

@@ -203,6 +203,7 @@ class TrezorClientBase(PrintError):
                 self.client,
                 *args,
                 input_callback=input_callback,
+                type=recovery_type,
                 **kwargs)
 
     # ========= Unmodified trezorlib methods =========

electrum/plugins/trezor/qt.py

@@ -205,7 +205,7 @@ class QtPlugin(QtPluginBase):
         bg_numwords = QButtonGroup()
         for i, count in enumerate([12, 18, 24]):
             rb = QRadioButton(gb)
-            rb.setText(_("%d words") % count)
+            rb.setText(_("{:d} words").format(count))
             bg_numwords.addButton(rb)
             bg_numwords.setId(rb, i)
             hbox1.addWidget(rb)
@@ -407,7 +407,7 @@ class SettingsDialog(WindowModalDialog):
 
         def slider_moved():
             mins = timeout_slider.sliderPosition()
-            timeout_minutes.setText(_("%2d minutes") % mins)
+            timeout_minutes.setText(_("{:2d} minutes").format(mins))
 
         def slider_released():
             config.set_session_timeout(timeout_slider.sliderPosition() * 60)

electrum/tests/test_commands.py

@@ -3,6 +3,8 @@ from decimal import Decimal
 
 from electrum.commands import Commands, eval_bool
 
+from . import TestCaseForTestnet
+
 
 class TestCommands(unittest.TestCase):
 
@@ -39,3 +41,46 @@ class TestCommands(unittest.TestCase):
         self.assertTrue(eval_bool("True"))
         self.assertTrue(eval_bool("true"))
         self.assertTrue(eval_bool("1"))
+
+    def test_convert_xkey(self):
+        cmds = Commands(config=None, wallet=None, network=None)
+        xpubs = {
+            ("xpub6CCWFbvCbqF92kGwm9nV7t7RvVoQUKaq5USMdyVP6jvv1NgN52KAX6NNYCeE8Ca7JQC4K5tZcnQrubQcjJ6iixfPs4pwAQJAQgTt6hBjg11", "standard"),
+            ("ypub6X2mZGb7kWnct3U4bWa7KyCw6TwrQwaKzaxaRNPGUkJo4UVbKgUj9A2WZQbp87E2i3Js4ZV85SmQnt2BSzWjXCLzjQXMkK7egQXXVHT4eKn", "p2wpkh-p2sh"),
+            ("zpub6qs2rwG2uCL6jLfBRsMjY4JSGS6JMZZpuhUoCmH9rkgg7aJpaLeHmDgeacZQ81sx7gRfp35gY77xgAdkAgvkKS2bbkDnLDw8x8bAsuKBrvP", "p2wpkh"),
+        }
+        for xkey1, xtype1 in xpubs:
+            for xkey2, xtype2 in xpubs:
+                self.assertEqual(xkey2, cmds.convert_xkey(xkey1, xtype2))
+
+        xprvs = {
+            ("xprv9yD9r6PJmTgqpGCUf8FUkkAhNTxv4rryiFWkqb5mYQPw8aMDXUzuyJ3tgv5vUqYkdK1E6Q5jKxPss4HkMBYV4q8AfG8t7rxgyS4xQX4ndAm", "standard"),
+            ("yprvAJ3R9m4Dv9EKfZPbVV36xqGCYS7N1UrUdN2ycyyevQmpBgASn9AUbMi2i83WUkCg2x82qsgHnckRkLuK4sxVs4omXbqJhmnBFA8bo8ssinK", "p2wpkh-p2sh"),
+            ("zprvAcsgTRj94pmoWraiKqpjAvMhiQFox6qyYUZCQNsYJR9hEmyg2oL3DRNAjL16UerbSbEqbMGrFH6yddWsnaNWfJVNPwXjHgbfWtCFBgDxFkX", "p2wpkh"),
+        }
+        for xkey1, xtype1 in xprvs:
+            for xkey2, xtype2 in xprvs:
+                self.assertEqual(xkey2, cmds.convert_xkey(xkey1, xtype2))
+
+
+class TestCommandsTestnet(TestCaseForTestnet):
+
+    def test_convert_xkey(self):
+        cmds = Commands(config=None, wallet=None, network=None)
+        xpubs = {
+            ("tpubD8p5qNfjczgTGbh9qgNxsbFgyhv8GgfVkmp3L88qtRm5ibUYiDVCrn6WYfnGey5XVVw6Bc5QNQUZW5B4jFQsHjmaenvkFUgWtKtgj5AdPm9", "standard"),
+            ("upub59wfQ8qJTg6ZSuvwtR313Qdp8gP8TSBwTof5dPQ3QVsYp1N9t29Rr9TGF1pj8kAXUg3mKbmrTKasA2qmBJKb1bGUzB6ApDZpVC7LoHhyvBo", "p2wpkh-p2sh"),
+            ("vpub5UmvhoWDcMe3JD84impdFVjKJeXaQ4BSNvBJQnHvnWFRs7BP8gJzUD7QGDnK8epStKAa55NQuywR3KTKtzjbopx5rWnbQ8PJkvAzBtgaGBc", "p2wpkh"),
+        }
+        for xkey1, xtype1 in xpubs:
+            for xkey2, xtype2 in xpubs:
+                self.assertEqual(xkey2, cmds.convert_xkey(xkey1, xtype2))
+
+        xprvs = {
+            ("tprv8c83gxdVUcznP8fMx2iNUBbaQgQC7MUbBUDG3c6YU9xgt7Dn5pfcgHUeNZTAvuYmNgVHjyTzYzGWwJr7GvKCm2FkPaaJipyipbfJeB3tdPW", "standard"),
+            ("uprv8vxJzdJQdJYGERrUnPVzgGh5aeYe3yU66ajUpzzRrALZwD31LUqBJM8nPmQkvpCgnKc6VT4Z1ed4pbTfzcjDZFwMFvGjJjoD6Kix2pCwVe7", "p2wpkh-p2sh"),
+            ("vprv9FnaJHyKmz5k5j3bckHctMnakch5zbTb1hFhcPtKEAiSzJrEb8zjvQnvQyNLvircBxiuEvf7UJycht5EiK9EMVcx8Fy9techN3nbRQRFhEv", "p2wpkh"),
+        }
+        for xkey1, xtype1 in xprvs:
+            for xkey2, xtype2 in xprvs:
+                self.assertEqual(xkey2, cmds.convert_xkey(xkey1, xtype2))

electrum/tests/test_wallet_vertical.py

@@ -1033,7 +1033,6 @@ class TestWalletSending(TestCaseForTestnet):
 
         class NetworkMock:
             relay_fee = 1000
-            def get_local_height(self): return 1325785
             def run_from_another_thread(self, coro):
                 loop = asyncio.get_event_loop()
                 return loop.run_until_complete(coro)
@@ -1046,7 +1045,7 @@ class TestWalletSending(TestCaseForTestnet):
         privkeys = ['93NQ7CFbwTPyKDJLXe97jczw33fiLijam2SCZL3Uinz1NSbHrTu', ]
         network = NetworkMock()
         dest_addr = 'tb1q3ws2p0qjk5vrravv065xqlnkckvzcpclk79eu2'
-        tx = sweep(privkeys, network, config=None, recipient=dest_addr, fee=5000)
+        tx = sweep(privkeys, network, config=None, recipient=dest_addr, fee=5000, locktime=1325785)
 
         tx_copy = Transaction(tx.serialize())
         self.assertEqual('010000000129349e5641d79915e9d0282fdbaee8c3df0b6731bab9d70bf626e8588bde24ac010000004847304402206bf0d0a93abae0d5873a62ebf277a5dd2f33837821e8b93e74d04e19d71b578002201a6d729bc159941ef5c4c9e5fe13ece9fc544351ba531b00f68ba549c8b38a9a01fdffffff01b82e0f00000000001600148ba0a0bc12b51831f58c7ea8607e76c5982c071fd93a1400',

electrum/verifier.py

@@ -116,8 +116,11 @@ class SPV(NetworkJobOnDefaultServer):
         try:
             verify_tx_is_in_block(tx_hash, merkle_branch, pos, header, tx_height)
         except MerkleVerificationFailure as e:
-            self.print_error(str(e))
+            if self.network.config.get("skipmerklecheck"):
-            raise GracefulDisconnect(e)
+                self.print_error("skipping merkle proof check %s" % tx_hash)
+            else:
+                self.print_error(str(e))
+                raise GracefulDisconnect(e)
         # we passed all the tests
         self.merkle_roots[tx_hash] = header.get('merkle_root')
         try: self.requested_merkle.remove(tx_hash)

electrum/wallet.py

@@ -125,7 +125,8 @@ def sweep_preparations(privkeys, network: 'Network', imax=100):
     return inputs, keypairs
 
 
-def sweep(privkeys, network: 'Network', config: 'SimpleConfig', recipient, fee=None, imax=100):
+def sweep(privkeys, network: 'Network', config: 'SimpleConfig', recipient, fee=None, imax=100,
+          *, locktime=None):
     inputs, keypairs = sweep_preparations(privkeys, network, imax)
     total = sum(i.get('value') for i in inputs)
     if fee is None:
@@ -138,7 +139,8 @@ def sweep(privkeys, network: 'Network', config: 'SimpleConfig', recipient, fee=N
         raise Exception(_('Not enough funds on address.') + '\nTotal: %d satoshis\nFee: %d\nDust Threshold: %d'%(total, fee, dust_threshold(network)))
 
     outputs = [TxOutput(TYPE_ADDRESS, recipient, total - fee)]
-    locktime = network.get_local_height()
+    if locktime is None:
+        locktime = get_locktime_for_new_transaction(network)
 
     tx = Transaction.from_io(inputs, outputs, locktime=locktime)
     tx.set_rbf(True)
@@ -146,6 +148,26 @@ def sweep(privkeys, network: 'Network', config: 'SimpleConfig', recipient, fee=N
     return tx
 
 
+def get_locktime_for_new_transaction(network: 'Network') -> int:
+    # if no network or not up to date, just set locktime to zero
+    if not network:
+        return 0
+    chain = network.blockchain()
+    header = chain.header_at_tip()
+    if not header:
+        return 0
+    STALE_DELAY = 8 * 60 * 60  # in seconds
+    if header['timestamp'] + STALE_DELAY < time.time():
+        return 0
+    # discourage "fee sniping"
+    locktime = chain.height()
+    # sometimes pick locktime a bit further back, to help privacy
+    # of setups that need more time (offline/multisig/coinjoin/...)
+    if random.randint(0, 9) == 0:
+        locktime = max(0, locktime - random.randint(0, 99))
+    return locktime
+
+
 
 class CannotBumpFee(Exception): pass
 
@@ -694,7 +716,7 @@ class Abstract_Wallet(AddressSynchronizer):
             tx = Transaction.from_io(coins, outputs[:])
 
         # Timelock tx to current height.
-        tx.locktime = self.get_local_height()
+        tx.locktime = get_locktime_for_new_transaction(self.network)
         # Transactions with transaction comments/floData are version 2
         if flodata != "":
             tx.version = 2
@@ -800,7 +822,7 @@ class Abstract_Wallet(AddressSynchronizer):
                     continue
         if delta > 0:
             raise CannotBumpFee(_('Cannot bump fee') + ': ' + _('could not find suitable outputs'))
-        locktime = self.get_local_height()
+        locktime = get_locktime_for_new_transaction(self.network)
         tx_new = Transaction.from_io(inputs, outputs, locktime=locktime)
         return tx_new
 
@@ -820,7 +842,7 @@ class Abstract_Wallet(AddressSynchronizer):
         inputs = [item]
         out_address = self.get_unused_address() or address
         outputs = [TxOutput(TYPE_ADDRESS, out_address, value - fee)]
-        locktime = self.get_local_height()
+        locktime = get_locktime_for_new_transaction(self.network)
         return Transaction.from_io(inputs, outputs, locktime=locktime)
 
     def add_input_sig_info(self, txin, address):

setup.py

@@ -72,7 +72,7 @@ class CustomInstallCommand(install):
                 if not os.path.exists(path):
                     subprocess.call(["pyrcc5", "icons.qrc", "-o", path])
             except Exception as e:
-                print('Warning: building icons file failed with {}'.format(e))
+                print('Warning: building icons file failed with {}'.format(repr(e)))
 
 
 setup(