Merge pull request #2 from spesmilo/master

Merging upstream commits

.travis.yml

@@ -33,7 +33,7 @@ jobs:
       install:
         - sudo docker build  --no-cache -t electrum-wine-builder-img ./contrib/build-wine/docker/
       script:
-        - sudo docker run --name electrum-wine-builder-cont -v $PWD:/opt/electrum --rm --workdir /opt/electrum/contrib/build-wine electrum-wine-builder-img ./build.sh $TRAVIS_COMMIT
+        - sudo docker run --name electrum-wine-builder-cont -v $PWD:/opt/wine64/drive_c/electrum --rm --workdir /opt/wine64/drive_c/electrum/contrib/build-wine electrum-wine-builder-img ./build.sh
       after_success: true
     - os: osx
       language: c
@@ -53,4 +53,4 @@ jobs:
       script:
         - ./contrib/deterministic-build/check_submodules.sh
       after_success: true
-      if: tag IS present
+      if: tag IS present

MANIFEST.in

@@ -3,15 +3,16 @@ include README.rst
 include electrum.conf.sample
 include electrum.desktop
 include *.py
-include electrum
+include run_electrum
 include contrib/requirements/requirements.txt
 include contrib/requirements/requirements-hw.txt
-recursive-include lib *.py
-recursive-include gui *.py
-recursive-include plugins *.py
 recursive-include packages *.py
 recursive-include packages cacert.pem
 include icons.qrc
-recursive-include icons *
-recursive-include scripts *
+graft icons
 
+graft electrum
+prune electrum/tests
+
+global-exclude __pycache__
+global-exclude *.py[co]

README.rst

@@ -15,9 +15,9 @@ Electrum - Lightweight Bitcoin client
 .. image:: https://coveralls.io/repos/github/spesmilo/electrum/badge.svg?branch=master
     :target: https://coveralls.io/github/spesmilo/electrum?branch=master
     :alt: Test coverage statistics
-.. image:: https://img.shields.io/badge/help-translating-blue.svg
+.. image:: https://d322cqt584bo4o.cloudfront.net/electrum/localized.svg
     :target: https://crowdin.com/project/electrum
-    :alt: Help translating Electrum online
+    :alt: Help translate Electrum online
 
 
 

contrib/build-osx/osx.spec

@@ -23,6 +23,7 @@ block_cipher = None
 # see https://github.com/pyinstaller/pyinstaller/issues/2005
 hiddenimports = []
 hiddenimports += collect_submodules('trezorlib')
+hiddenimports += collect_submodules('safetlib')
 hiddenimports += collect_submodules('btchip')
 hiddenimports += collect_submodules('keepkeylib')
 hiddenimports += collect_submodules('websocket')
@@ -33,10 +34,11 @@ datas = [
     (electrum+'electrum/locale', PYPKG + '/locale')
 ]
 datas += collect_data_files('trezorlib')
+datas += collect_data_files('safetlib')
 datas += collect_data_files('btchip')
 datas += collect_data_files('keepkeylib')
 
-# Add libusb so Trezor will work
+# Add libusb so Trezor and Safe-T mini will work
 binaries = [(electrum + "contrib/build-osx/libusb-1.0.dylib", ".")]
 binaries += [(electrum + "contrib/build-osx/libsecp256k1.0.dylib", ".")]
 
@@ -57,6 +59,8 @@ a = Analysis([electrum+ MAIN_SCRIPT,
               electrum+'electrum/plugins/email_requests/qt.py',
               electrum+'electrum/plugins/trezor/client.py',
               electrum+'electrum/plugins/trezor/qt.py',
+              electrum+'electrum/plugins/safe_t/client.py',
+              electrum+'electrum/plugins/safe_t/qt.py',
               electrum+'electrum/plugins/keepkey/qt.py',
               electrum+'electrum/plugins/ledger/qt.py',
               ],

contrib/build-wine/build-electrum-git.sh

@@ -19,29 +19,13 @@ set -e
 mkdir -p tmp
 cd tmp
 
-if [ -d ./electrum ]; then
-  rm ./electrum -rf
-fi
-
-git clone https://github.com/spesmilo/electrum -b master
-
-pushd electrum
-if [ ! -z "$1" ]; then
-    # a commit/tag/branch was specified
-    if ! git cat-file -e "$1" 2> /dev/null
-    then  # can't find target
-        # try pull requests
-        git config --local --add remote.origin.fetch '+refs/pull/*/merge:refs/remotes/origin/pr/*'
-        git fetch --all
-    fi
-    git checkout $1
-fi
+pushd $WINEPREFIX/drive_c/electrum
 
 # Load electrum-icons and electrum-locale for this release
 git submodule init
 git submodule update
 
-VERSION=`git describe --tags --dirty`
+VERSION=`git describe --tags --dirty || printf 'custom'`
 echo "Last commit: $VERSION"
 
 pushd ./contrib/deterministic-build/electrum-locale
@@ -59,11 +43,9 @@ popd
 find -exec touch -d '2000-11-11T11:11:11+00:00' {} +
 popd
 
-rm -rf $WINEPREFIX/drive_c/electrum
-cp -r electrum $WINEPREFIX/drive_c/electrum
-cp electrum/LICENCE .
-cp -r ./electrum/contrib/deterministic-build/electrum-locale/locale $WINEPREFIX/drive_c/electrum/electrum/
-cp ./electrum/contrib/deterministic-build/electrum-icons/icons_rc.py $WINEPREFIX/drive_c/electrum/electrum/gui/qt/
+cp $WINEPREFIX/drive_c/electrum/LICENCE .
+cp -r $WINEPREFIX/drive_c/electrum/contrib/deterministic-build/electrum-locale/locale $WINEPREFIX/drive_c/electrum/electrum/
+cp $WINEPREFIX/drive_c/electrum/contrib/deterministic-build/electrum-icons/icons_rc.py $WINEPREFIX/drive_c/electrum/electrum/gui/qt/
 
 # Install frozen dependencies
 $PYTHON -m pip install -r ../../deterministic-build/requirements.txt

contrib/build-wine/build.sh

@@ -2,10 +2,6 @@
 # Lucky number
 export PYTHONHASHSEED=22
 
-if [ ! -z "$1" ]; then
-    to_build="$1"
-fi
-
 here=$(dirname "$0")
 test -n "$here" -a -d "$here" || exit
 
@@ -28,5 +24,5 @@ find -exec touch -d '2000-11-11T11:11:11+00:00' {} +
 popd
 ls -l /opt/wine64/drive_c/python*
 
-$here/build-electrum-git.sh $to_build && \
+$here/build-electrum-git.sh && \
 echo "Done."

contrib/build-wine/deterministic.spec

@@ -18,6 +18,7 @@ home = 'C:\\electrum\\'
 # see https://github.com/pyinstaller/pyinstaller/issues/2005
 hiddenimports = []
 hiddenimports += collect_submodules('trezorlib')
+hiddenimports += collect_submodules('safetlib')
 hiddenimports += collect_submodules('btchip')
 hiddenimports += collect_submodules('keepkeylib')
 hiddenimports += collect_submodules('websocket')
@@ -37,6 +38,7 @@ datas = [
     ('C:\\Program Files (x86)\\ZBar\\bin\\', '.')
 ]
 datas += collect_data_files('trezorlib')
+datas += collect_data_files('safetlib')
 datas += collect_data_files('btchip')
 datas += collect_data_files('keepkeylib')
 
@@ -54,6 +56,8 @@ a = Analysis([home+'run_electrum',
               home+'electrum/plugins/email_requests/qt.py',
               home+'electrum/plugins/trezor/client.py',
               home+'electrum/plugins/trezor/qt.py',
+              home+'electrum/plugins/safe_t/client.py',
+              home+'electrum/plugins/safe_t/qt.py',
               home+'electrum/plugins/keepkey/qt.py',
               home+'electrum/plugins/ledger/qt.py',
               #home+'packages/requests/utils.py'

contrib/build-wine/docker/Dockerfile

@@ -8,7 +8,7 @@ RUN dpkg --add-architecture i386 && \
         wget=1.19.4-1ubuntu2.1 \
         gnupg2=2.2.4-1ubuntu1.1 \
         dirmngr=2.2.4-1ubuntu1.1 \
-        software-properties-common=0.96.24.32.3 \
+        software-properties-common=0.96.24.32.4 \
         && \
     wget -nc https://dl.winehq.org/wine-builds/Release.key && \
     apt-key add Release.key && \

contrib/build-wine/docker/README.md

@@ -25,14 +25,14 @@ folder.
 3. Build Windows binaries
 
     ```
-    $ TARGET=master
+    $ git checkout $REV
     $ sudo docker run \
         --name electrum-wine-builder-cont \
-        -v .:/opt/electrum \
+        -v $PWD:/opt/wine64/drive_c/electrum \
         --rm \
-        --workdir /opt/electrum/contrib/build-wine \
+        --workdir /opt/wine64/drive_c/electrum/contrib/build-wine \
         electrum-wine-builder-img \
-        ./build.sh $TARGET
+        ./build.sh
     ```
 4. The generated binaries are in `./contrib/build-wine/dist`.
 

contrib/build-wine/electrum.nsi

@@ -72,7 +72,7 @@
   !define MUI_ABORTWARNING
   !define MUI_ABORTWARNING_TEXT "Are you sure you wish to abort the installation of ${PRODUCT_NAME}?"
   
-  !define MUI_ICON "tmp\electrum\icons\electrum.ico"
+  !define MUI_ICON "c:\electrum\icons\electrum.ico"
   
 ;--------------------------------
 ;Pages
@@ -111,7 +111,7 @@ Section
   
   ;Files to pack into the installer
   File /r "dist\electrum\*.*"
-  File "..\..\icons\electrum.ico"
+  File "c:\electrum\icons\electrum.ico"
 
   ;Store installation folder
   WriteRegStr HKCU "Software\${PRODUCT_NAME}" "" $INSTDIR

contrib/build-wine/prepare-wine.sh

@@ -79,11 +79,6 @@ retry() {
 here=$(dirname $(readlink -e $0))
 set -e
 
-# Clean up Wine environment
-echo "Cleaning $WINEPREFIX"
-rm -rf $WINEPREFIX
-echo "done"
-
 wine 'wineboot'
 
 # HACK to work around https://bugs.winehq.org/show_bug.cgi?id=42474#c22
@@ -101,8 +96,7 @@ KEYRING_PYTHON_DEV="keyring-electrum-build-python-dev.gpg"
 for server in $(shuf -e ha.pool.sks-keyservers.net \
                         hkp://p80.pool.sks-keyservers.net:80 \
                         keyserver.ubuntu.com \
-                        hkp://keyserver.ubuntu.com:80 \
-                        pgp.mit.edu) ; do
+                        hkp://keyserver.ubuntu.com:80) ; do
     retry gpg --no-default-keyring --keyring $KEYRING_PYTHON_DEV --keyserver "$server" --recv-keys $KEYLIST_PYTHON_DEV \
     && break || : ;
 done

contrib/deterministic-build/requirements-hw.txt

@@ -94,6 +94,9 @@ pyblake2==1.1.2 \
 requests==2.19.1 \
     --hash=sha256:63b52e3c866428a224f97cab011de738c36aec0185aa91cfacd418b5d58911d1 \
     --hash=sha256:ec22d826a36ed72a7358ff3fe56cbd4ba69dd7a6718ffd450ff0e9df7a47ce6a
+safet==0.1.3 \
+    --hash=sha256:ba80fe9f6ba317ab9514a8726cd3792e68eb46dd419f380d48ae4a0ccae646dc \
+    --hash=sha256:e5d8e6a87c8bdf1cefd07004181b93fd7631557fdab09d143ba8d1b29291d6dc
 setuptools==40.0.0 \
     --hash=sha256:012adb8e25fbfd64c652e99e7bab58799a3aaf05d39ab38561f69190a909015f \
     --hash=sha256:d68abee4eed409fbe8c302ac4d8429a1ffef912cd047a903b5701c024048dd49

contrib/deterministic-build/requirements.txt

@@ -16,8 +16,6 @@ idna==2.7 \
 jsonrpclib-pelix==0.3.1 \
     --hash=sha256:5417b1508d5a50ec64f6e5b88907f111155d52607b218ff3ba9a777afb2e49e3 \
     --hash=sha256:bd89a6093bc4d47dc8a096197aacb827359944a4533be5193f3845f57b9f91b4
-pbkdf2==1.3 \
-    --hash=sha256:ac6397369f128212c43064a2b4878038dab78dab41875364554aaf2a684e6979
 pip==10.0.1 \
     --hash=sha256:717cdffb2833be8409433a93746744b59505f42146e8d37de6c62b430e25d6d7 \
     --hash=sha256:f2bd08e0cd1b06e10218feaf6fef299f473ba706582eb3bd9d52203fdbd7ee68
@@ -56,6 +54,10 @@ setuptools==40.0.0 \
 six==1.11.0 \
     --hash=sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9 \
     --hash=sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb
+typing==3.6.4 \
+    --hash=sha256:3a887b021a77b292e151afb75323dea88a7bc1b3dfa92176cff8e44c8b68bddf \
+    --hash=sha256:b2c689d54e1144bbcfd191b0832980a21c2dbcf7b5ff7a66248a60c90e951eb8 \
+    --hash=sha256:d400a9344254803a2368533e4533a4200d21eb7b6b729c173bc38201a74db3f2
 urllib3==1.23 \
     --hash=sha256:a68ac5e15e76e7e5dd2b8f94007233e01effe3e50e8daddf69acfd81cb686baf \
     --hash=sha256:b5725a0bd4ba422ab0e66e89e030c806576753ea3ee08554382c14e685d117b5

contrib/make_locale

@@ -54,7 +54,7 @@ if crowdin_api_key:
 
 # Download & unzip
 print('Download translations')
-s = requests.request('GET', 'https://crowdin.com/download/project/' + crowdin_identifier + '.zip').content
+s = requests.request('GET', 'https://crowdin.com/backend/download/project/' + crowdin_identifier + '.zip').content
 zfobj = zipfile.ZipFile(io.BytesIO(s))
 
 print('Unzip translations')

contrib/requirements/requirements-hw.txt

@@ -1,5 +1,6 @@
 Cython>=0.27
 trezor[hidapi]>=0.9.0
+safet[hidapi]>=0.1.0
 keepkey
 btchip-python
 websocket-client

contrib/requirements/requirements.txt

@@ -1,6 +1,5 @@
 pyaes>=0.1a1
 ecdsa>=0.9
-pbkdf2
 requests
 qrcode
 protobuf
@@ -8,3 +7,4 @@ dnspython
 jsonrpclib-pelix
 PySocks>=1.6.6
 qdarkstyle<3.0
+typing>=3.0.0

electrum/__init__.py

@@ -1,6 +1,6 @@
 from .version import ELECTRUM_VERSION
 from .util import format_satoshis, print_msg, print_error, set_verbosity
-from .wallet import Synchronizer, Wallet
+from .wallet import Wallet
 from .storage import WalletStorage
 from .coinchooser import COIN_CHOOSERS
 from .network import Network, pick_random_server

electrum/address_synchronizer.py

@@ -0,0 +1,789 @@
+# Electrum - lightweight Bitcoin client
+# Copyright (C) 2018 The Electrum Developers
+#
+# Permission is hereby granted, free of charge, to any person
+# obtaining a copy of this software and associated documentation files
+# (the "Software"), to deal in the Software without restriction,
+# including without limitation the rights to use, copy, modify, merge,
+# publish, distribute, sublicense, and/or sell copies of the Software,
+# and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import threading
+import itertools
+from collections import defaultdict
+
+from . import bitcoin
+from .bitcoin import COINBASE_MATURITY, TYPE_ADDRESS, TYPE_PUBKEY
+from .util import PrintError, profiler, bfh, VerifiedTxInfo, TxMinedStatus
+from .transaction import Transaction, TxOutput
+from .synchronizer import Synchronizer
+from .verifier import SPV
+from .blockchain import hash_header
+from .i18n import _
+
+TX_HEIGHT_LOCAL = -2
+TX_HEIGHT_UNCONF_PARENT = -1
+TX_HEIGHT_UNCONFIRMED = 0
+
+class AddTransactionException(Exception):
+    pass
+
+
+class UnrelatedTransactionException(AddTransactionException):
+    def __str__(self):
+        return _("Transaction is unrelated to this wallet.")
+
+
+class AddressSynchronizer(PrintError):
+    """
+    inherited by wallet
+    """
+
+    def __init__(self, storage):
+        self.storage = storage
+        self.network = None
+        # verifier (SPV) and synchronizer are started in start_threads
+        self.synchronizer = None
+        self.verifier = None
+        # locks: if you need to take multiple ones, acquire them in the order they are defined here!
+        self.lock = threading.RLock()
+        self.transaction_lock = threading.RLock()
+        # address -> list(txid, height)
+        self.history = storage.get('addr_history',{})
+        # Verified transactions.  txid -> VerifiedTxInfo.  Access with self.lock.
+        verified_tx = storage.get('verified_tx3', {})
+        self.verified_tx = {}
+        for txid, (height, timestamp, txpos, header_hash) in verified_tx.items():
+            self.verified_tx[txid] = VerifiedTxInfo(height, timestamp, txpos, header_hash)
+        # Transactions pending verification.  txid -> tx_height. Access with self.lock.
+        self.unverified_tx = defaultdict(int)
+        # true when synchronized
+        self.up_to_date = False
+        # thread local storage for caching stuff
+        self.threadlocal_cache = threading.local()
+
+        self.load_and_cleanup()
+
+    def load_and_cleanup(self):
+        self.load_transactions()
+        self.load_local_history()
+        self.check_history()
+        self.load_unverified_transactions()
+        self.remove_local_transactions_we_dont_have()
+
+    def is_mine(self, address):
+        return address in self.history
+
+    def get_addresses(self):
+        return sorted(self.history.keys())
+
+    def get_address_history(self, addr):
+        h = []
+        # we need self.transaction_lock but get_tx_height will take self.lock
+        # so we need to take that too here, to enforce order of locks
+        with self.lock, self.transaction_lock:
+            related_txns = self._history_local.get(addr, set())
+            for tx_hash in related_txns:
+                tx_height = self.get_tx_height(tx_hash).height
+                h.append((tx_hash, tx_height))
+        return h
+
+    def get_address_history_len(self, addr: str) -> int:
+        return len(self._history_local.get(addr, ()))
+
+    def get_txin_address(self, txi):
+        addr = txi.get('address')
+        if addr and addr != "(pubkey)":
+            return addr
+        prevout_hash = txi.get('prevout_hash')
+        prevout_n = txi.get('prevout_n')
+        dd = self.txo.get(prevout_hash, {})
+        for addr, l in dd.items():
+            for n, v, is_cb in l:
+                if n == prevout_n:
+                    return addr
+        return None
+
+    def get_txout_address(self, txo: TxOutput):
+        if txo.type == TYPE_ADDRESS:
+            addr = txo.address
+        elif txo.type == TYPE_PUBKEY:
+            addr = bitcoin.public_key_to_p2pkh(bfh(txo.address))
+        else:
+            addr = None
+        return addr
+
+    def load_unverified_transactions(self):
+        # review transactions that are in the history
+        for addr, hist in self.history.items():
+            for tx_hash, tx_height in hist:
+                # add it in case it was previously unconfirmed
+                self.add_unverified_tx(tx_hash, tx_height)
+
+    def start_threads(self, network):
+        self.network = network
+        if self.network is not None:
+            self.verifier = SPV(self.network, self)
+            self.synchronizer = Synchronizer(self, network)
+            network.add_jobs([self.verifier, self.synchronizer])
+        else:
+            self.verifier = None
+            self.synchronizer = None
+
+    def stop_threads(self):
+        if self.network:
+            self.network.remove_jobs([self.synchronizer, self.verifier])
+            self.synchronizer.release()
+            self.synchronizer = None
+            self.verifier = None
+            # Now no references to the synchronizer or verifier
+            # remain so they will be GC-ed
+            self.storage.put('stored_height', self.get_local_height())
+        self.save_transactions()
+        self.save_verified_tx()
+        self.storage.write()
+
+    def add_address(self, address):
+        if address not in self.history:
+            self.history[address] = []
+            self.set_up_to_date(False)
+        if self.synchronizer:
+            self.synchronizer.add(address)
+
+    def get_conflicting_transactions(self, tx):
+        """Returns a set of transaction hashes from the wallet history that are
+        directly conflicting with tx, i.e. they have common outpoints being
+        spent with tx. If the tx is already in wallet history, that will not be
+        reported as a conflict.
+        """
+        conflicting_txns = set()
+        with self.transaction_lock:
+            for txin in tx.inputs():
+                if txin['type'] == 'coinbase':
+                    continue
+                prevout_hash = txin['prevout_hash']
+                prevout_n = txin['prevout_n']
+                spending_tx_hash = self.spent_outpoints[prevout_hash].get(prevout_n)
+                if spending_tx_hash is None:
+                    continue
+                # this outpoint has already been spent, by spending_tx
+                assert spending_tx_hash in self.transactions
+                conflicting_txns |= {spending_tx_hash}
+            txid = tx.txid()
+            if txid in conflicting_txns:
+                # this tx is already in history, so it conflicts with itself
+                if len(conflicting_txns) > 1:
+                    raise Exception('Found conflicting transactions already in wallet history.')
+                conflicting_txns -= {txid}
+            return conflicting_txns
+
+    def add_transaction(self, tx_hash, tx, allow_unrelated=False):
+        assert tx_hash, tx_hash
+        assert tx, tx
+        assert tx.is_complete()
+        # we need self.transaction_lock but get_tx_height will take self.lock
+        # so we need to take that too here, to enforce order of locks
+        with self.lock, self.transaction_lock:
+            # NOTE: returning if tx in self.transactions might seem like a good idea
+            # BUT we track is_mine inputs in a txn, and during subsequent calls
+            # of add_transaction tx, we might learn of more-and-more inputs of
+            # being is_mine, as we roll the gap_limit forward
+            is_coinbase = tx.inputs()[0]['type'] == 'coinbase'
+            tx_height = self.get_tx_height(tx_hash).height
+            if not allow_unrelated:
+                # note that during sync, if the transactions are not properly sorted,
+                # it could happen that we think tx is unrelated but actually one of the inputs is is_mine.
+                # this is the main motivation for allow_unrelated
+                is_mine = any([self.is_mine(self.get_txin_address(txin)) for txin in tx.inputs()])
+                is_for_me = any([self.is_mine(self.get_txout_address(txo)) for txo in tx.outputs()])
+                if not is_mine and not is_for_me:
+                    raise UnrelatedTransactionException()
+            # Find all conflicting transactions.
+            # In case of a conflict,
+            #     1. confirmed > mempool > local
+            #     2. this new txn has priority over existing ones
+            # When this method exits, there must NOT be any conflict, so
+            # either keep this txn and remove all conflicting (along with dependencies)
+            #     or drop this txn
+            conflicting_txns = self.get_conflicting_transactions(tx)
+            if conflicting_txns:
+                existing_mempool_txn = any(
+                    self.get_tx_height(tx_hash2).height in (TX_HEIGHT_UNCONFIRMED, TX_HEIGHT_UNCONF_PARENT)
+                    for tx_hash2 in conflicting_txns)
+                existing_confirmed_txn = any(
+                    self.get_tx_height(tx_hash2).height > 0
+                    for tx_hash2 in conflicting_txns)
+                if existing_confirmed_txn and tx_height <= 0:
+                    # this is a non-confirmed tx that conflicts with confirmed txns; drop.
+                    return False
+                if existing_mempool_txn and tx_height == TX_HEIGHT_LOCAL:
+                    # this is a local tx that conflicts with non-local txns; drop.
+                    return False
+                # keep this txn and remove all conflicting
+                to_remove = set()
+                to_remove |= conflicting_txns
+                for conflicting_tx_hash in conflicting_txns:
+                    to_remove |= self.get_depending_transactions(conflicting_tx_hash)
+                for tx_hash2 in to_remove:
+                    self.remove_transaction(tx_hash2)
+            # add inputs
+            def add_value_from_prev_output():
+                dd = self.txo.get(prevout_hash, {})
+                # note: this nested loop takes linear time in num is_mine outputs of prev_tx
+                for addr, outputs in dd.items():
+                    # note: instead of [(n, v, is_cb), ...]; we could store: {n -> (v, is_cb)}
+                    for n, v, is_cb in outputs:
+                        if n == prevout_n:
+                            if addr and self.is_mine(addr):
+                                if d.get(addr) is None:
+                                    d[addr] = set()
+                                d[addr].add((ser, v))
+                            return
+            self.txi[tx_hash] = d = {}
+            for txi in tx.inputs():
+                if txi['type'] == 'coinbase':
+                    continue
+                prevout_hash = txi['prevout_hash']
+                prevout_n = txi['prevout_n']
+                ser = prevout_hash + ':%d' % prevout_n
+                self.spent_outpoints[prevout_hash][prevout_n] = tx_hash
+                add_value_from_prev_output()
+            # add outputs
+            self.txo[tx_hash] = d = {}
+            for n, txo in enumerate(tx.outputs()):
+                v = txo[2]
+                ser = tx_hash + ':%d'%n
+                addr = self.get_txout_address(txo)
+                if addr and self.is_mine(addr):
+                    if d.get(addr) is None:
+                        d[addr] = []
+                    d[addr].append((n, v, is_coinbase))
+                    # give v to txi that spends me
+                    next_tx = self.spent_outpoints[tx_hash].get(n)
+                    if next_tx is not None:
+                        dd = self.txi.get(next_tx, {})
+                        if dd.get(addr) is None:
+                            dd[addr] = set()
+                        if (ser, v) not in dd[addr]:
+                            dd[addr].add((ser, v))
+                        self._add_tx_to_local_history(next_tx)
+            # add to local history
+            self._add_tx_to_local_history(tx_hash)
+            # save
+            self.transactions[tx_hash] = tx
+            return True
+
+    def remove_transaction(self, tx_hash):
+        def remove_from_spent_outpoints():
+            # undo spends in spent_outpoints
+            if tx is not None:  # if we have the tx, this branch is faster
+                for txin in tx.inputs():
+                    if txin['type'] == 'coinbase':
+                        continue
+                    prevout_hash = txin['prevout_hash']
+                    prevout_n = txin['prevout_n']
+                    self.spent_outpoints[prevout_hash].pop(prevout_n, None)
+                    if not self.spent_outpoints[prevout_hash]:
+                        self.spent_outpoints.pop(prevout_hash)
+            else:  # expensive but always works
+                for prevout_hash, d in list(self.spent_outpoints.items()):
+                    for prevout_n, spending_txid in d.items():
+                        if spending_txid == tx_hash:
+                            self.spent_outpoints[prevout_hash].pop(prevout_n, None)
+                            if not self.spent_outpoints[prevout_hash]:
+                                self.spent_outpoints.pop(prevout_hash)
+            # Remove this tx itself; if nothing spends from it.
+            # It is not so clear what to do if other txns spend from it, but it will be
+            # removed when those other txns are removed.
+            if not self.spent_outpoints[tx_hash]:
+                self.spent_outpoints.pop(tx_hash)
+
+        with self.transaction_lock:
+            self.print_error("removing tx from history", tx_hash)
+            tx = self.transactions.pop(tx_hash, None)
+            remove_from_spent_outpoints()
+            self._remove_tx_from_local_history(tx_hash)
+            self.txi.pop(tx_hash, None)
+            self.txo.pop(tx_hash, None)
+
+    def receive_tx_callback(self, tx_hash, tx, tx_height):
+        self.add_unverified_tx(tx_hash, tx_height)
+        self.add_transaction(tx_hash, tx, allow_unrelated=True)
+
+    def receive_history_callback(self, addr, hist, tx_fees):
+        with self.lock:
+            old_hist = self.get_address_history(addr)
+            for tx_hash, height in old_hist:
+                if (tx_hash, height) not in hist:
+                    # make tx local
+                    self.unverified_tx.pop(tx_hash, None)
+                    self.verified_tx.pop(tx_hash, None)
+                    if self.verifier:
+                        self.verifier.remove_spv_proof_for_tx(tx_hash)
+            self.history[addr] = hist
+
+        for tx_hash, tx_height in hist:
+            # add it in case it was previously unconfirmed
+            self.add_unverified_tx(tx_hash, tx_height)
+            # if addr is new, we have to recompute txi and txo
+            tx = self.transactions.get(tx_hash)
+            if tx is None:
+                continue
+            self.add_transaction(tx_hash, tx, allow_unrelated=True)
+
+        # Store fees
+        self.tx_fees.update(tx_fees)
+
+    @profiler
+    def load_transactions(self):
+        # load txi, txo, tx_fees
+        self.txi = self.storage.get('txi', {})
+        for txid, d in list(self.txi.items()):
+            for addr, lst in d.items():
+                self.txi[txid][addr] = set([tuple(x) for x in lst])
+        self.txo = self.storage.get('txo', {})
+        self.tx_fees = self.storage.get('tx_fees', {})
+        tx_list = self.storage.get('transactions', {})
+        # load transactions
+        self.transactions = {}
+        for tx_hash, raw in tx_list.items():
+            tx = Transaction(raw)
+            self.transactions[tx_hash] = tx
+            if self.txi.get(tx_hash) is None and self.txo.get(tx_hash) is None:
+                self.print_error("removing unreferenced tx", tx_hash)
+                self.transactions.pop(tx_hash)
+        # load spent_outpoints
+        _spent_outpoints = self.storage.get('spent_outpoints', {})
+        self.spent_outpoints = defaultdict(dict)
+        for prevout_hash, d in _spent_outpoints.items():
+            for prevout_n_str, spending_txid in d.items():
+                prevout_n = int(prevout_n_str)
+                self.spent_outpoints[prevout_hash][prevout_n] = spending_txid
+
+    @profiler
+    def load_local_history(self):
+        self._history_local = {}  # address -> set(txid)
+        for txid in itertools.chain(self.txi, self.txo):
+            self._add_tx_to_local_history(txid)
+
+    @profiler
+    def check_history(self):
+        save = False
+        hist_addrs_mine = list(filter(lambda k: self.is_mine(k), self.history.keys()))
+        hist_addrs_not_mine = list(filter(lambda k: not self.is_mine(k), self.history.keys()))
+        for addr in hist_addrs_not_mine:
+            self.history.pop(addr)
+            save = True
+        for addr in hist_addrs_mine:
+            hist = self.history[addr]
+            for tx_hash, tx_height in hist:
+                if self.txi.get(tx_hash) or self.txo.get(tx_hash):
+                    continue
+                tx = self.transactions.get(tx_hash)
+                if tx is not None:
+                    self.add_transaction(tx_hash, tx, allow_unrelated=True)
+                    save = True
+        if save:
+            self.save_transactions()
+
+    def remove_local_transactions_we_dont_have(self):
+        txid_set = set(self.txi) | set(self.txo)
+        for txid in txid_set:
+            tx_height = self.get_tx_height(txid).height
+            if tx_height == TX_HEIGHT_LOCAL and txid not in self.transactions:
+                self.remove_transaction(txid)
+
+    @profiler
+    def save_transactions(self, write=False):
+        with self.transaction_lock:
+            tx = {}
+            for k,v in self.transactions.items():
+                tx[k] = str(v)
+            self.storage.put('transactions', tx)
+            self.storage.put('txi', self.txi)
+            self.storage.put('txo', self.txo)
+            self.storage.put('tx_fees', self.tx_fees)
+            self.storage.put('addr_history', self.history)
+            self.storage.put('spent_outpoints', self.spent_outpoints)
+            if write:
+                self.storage.write()
+
+    def save_verified_tx(self, write=False):
+        with self.lock:
+            self.storage.put('verified_tx3', self.verified_tx)
+            if write:
+                self.storage.write()
+
+    def clear_history(self):
+        with self.lock:
+            with self.transaction_lock:
+                self.txi = {}
+                self.txo = {}
+                self.tx_fees = {}
+                self.spent_outpoints = defaultdict(dict)
+                self.history = {}
+                self.verified_tx = {}
+                self.transactions = {}
+                self.save_transactions()
+
+    def get_txpos(self, tx_hash):
+        """Returns (height, txpos) tuple, even if the tx is unverified."""
+        with self.lock:
+            if tx_hash in self.verified_tx:
+                info = self.verified_tx[tx_hash]
+                return info.height, info.txpos
+            elif tx_hash in self.unverified_tx:
+                height = self.unverified_tx[tx_hash]
+                return (height, 0) if height > 0 else ((1e9 - height), 0)
+            else:
+                return (1e9+1, 0)
+
+    def with_local_height_cached(func):
+        # get local height only once, as it's relatively expensive.
+        # take care that nested calls work as expected
+        def f(self, *args, **kwargs):
+            orig_val = getattr(self.threadlocal_cache, 'local_height', None)
+            self.threadlocal_cache.local_height = orig_val or self.get_local_height()
+            try:
+                return func(self, *args, **kwargs)
+            finally:
+                self.threadlocal_cache.local_height = orig_val
+        return f
+
+    @with_local_height_cached
+    def get_history(self, domain=None):
+        # get domain
+        if domain is None:
+            domain = self.history.keys()
+        domain = set(domain)
+        # 1. Get the history of each address in the domain, maintain the
+        #    delta of a tx as the sum of its deltas on domain addresses
+        tx_deltas = defaultdict(int)
+        for addr in domain:
+            h = self.get_address_history(addr)
+            for tx_hash, height in h:
+                delta = self.get_tx_delta(tx_hash, addr)
+                if delta is None or tx_deltas[tx_hash] is None:
+                    tx_deltas[tx_hash] = None
+                else:
+                    tx_deltas[tx_hash] += delta
+        # 2. create sorted history
+        history = []
+        for tx_hash in tx_deltas:
+            delta = tx_deltas[tx_hash]
+            tx_mined_status = self.get_tx_height(tx_hash)
+            history.append((tx_hash, tx_mined_status, delta))
+        history.sort(key = lambda x: self.get_txpos(x[0]))
+        history.reverse()
+        # 3. add balance
+        c, u, x = self.get_balance(domain)
+        balance = c + u + x
+        h2 = []
+        for tx_hash, tx_mined_status, delta in history:
+            h2.append((tx_hash, tx_mined_status, delta, balance))
+            if balance is None or delta is None:
+                balance = None
+            else:
+                balance -= delta
+        h2.reverse()
+        # fixme: this may happen if history is incomplete
+        if balance not in [None, 0]:
+            self.print_error("Error: history not synchronized")
+            return []
+
+        return h2
+
+    def _add_tx_to_local_history(self, txid):
+        with self.transaction_lock:
+            for addr in itertools.chain(self.txi.get(txid, []), self.txo.get(txid, [])):
+                cur_hist = self._history_local.get(addr, set())
+                cur_hist.add(txid)
+                self._history_local[addr] = cur_hist
+
+    def _remove_tx_from_local_history(self, txid):
+        with self.transaction_lock:
+            for addr in itertools.chain(self.txi.get(txid, []), self.txo.get(txid, [])):
+                cur_hist = self._history_local.get(addr, set())
+                try:
+                    cur_hist.remove(txid)
+                except KeyError:
+                    pass
+                else:
+                    self._history_local[addr] = cur_hist
+
+    def add_unverified_tx(self, tx_hash, tx_height):
+        if tx_hash in self.verified_tx:
+            if tx_height in (TX_HEIGHT_UNCONFIRMED, TX_HEIGHT_UNCONF_PARENT):
+                with self.lock:
+                    self.verified_tx.pop(tx_hash)
+                if self.verifier:
+                    self.verifier.remove_spv_proof_for_tx(tx_hash)
+        else:
+            with self.lock:
+                # tx will be verified only if height > 0
+                self.unverified_tx[tx_hash] = tx_height
+            # to remove pending proof requests:
+            if self.verifier:
+                self.verifier.remove_spv_proof_for_tx(tx_hash)
+
+    def add_verified_tx(self, tx_hash: str, info: VerifiedTxInfo):
+        # Remove from the unverified map and add to the verified map
+        with self.lock:
+            self.unverified_tx.pop(tx_hash, None)
+            self.verified_tx[tx_hash] = info
+        tx_mined_status = self.get_tx_height(tx_hash)
+        self.network.trigger_callback('verified', tx_hash, tx_mined_status)
+
+    def get_unverified_txs(self):
+        '''Returns a map from tx hash to transaction height'''
+        with self.lock:
+            return dict(self.unverified_tx)  # copy
+
+    def undo_verifications(self, blockchain, height):
+        '''Used by the verifier when a reorg has happened'''
+        txs = set()
+        with self.lock:
+            for tx_hash, info in list(self.verified_tx.items()):
+                tx_height = info.height
+                if tx_height >= height:
+                    header = blockchain.read_header(tx_height)
+                    if not header or hash_header(header) != info.header_hash:
+                        self.verified_tx.pop(tx_hash, None)
+                        # NOTE: we should add these txns to self.unverified_tx,
+                        # but with what height?
+                        # If on the new fork after the reorg, the txn is at the
+                        # same height, we will not get a status update for the
+                        # address. If the txn is not mined or at a diff height,
+                        # we should get a status update. Unless we put tx into
+                        # unverified_tx, it will turn into local. So we put it
+                        # into unverified_tx with the old height, and if we get
+                        # a status update, that will overwrite it.
+                        self.unverified_tx[tx_hash] = tx_height
+                        txs.add(tx_hash)
+        return txs
+
+    def get_local_height(self):
+        """ return last known height if we are offline """
+        cached_local_height = getattr(self.threadlocal_cache, 'local_height', None)
+        if cached_local_height is not None:
+            return cached_local_height
+        return self.network.get_local_height() if self.network else self.storage.get('stored_height', 0)
+
+    def get_tx_height(self, tx_hash: str) -> TxMinedStatus:
+        """ Given a transaction, returns (height, conf, timestamp, header_hash) """
+        with self.lock:
+            if tx_hash in self.verified_tx:
+                info = self.verified_tx[tx_hash]
+                conf = max(self.get_local_height() - info.height + 1, 0)
+                return TxMinedStatus(info.height, conf, info.timestamp, info.header_hash)
+            elif tx_hash in self.unverified_tx:
+                height = self.unverified_tx[tx_hash]
+                return TxMinedStatus(height, 0, None, None)
+            else:
+                # local transaction
+                return TxMinedStatus(TX_HEIGHT_LOCAL, 0, None, None)
+
+    def set_up_to_date(self, up_to_date):
+        with self.lock:
+            self.up_to_date = up_to_date
+        if up_to_date:
+            self.save_transactions(write=True)
+            # if the verifier is also up to date, persist that too;
+            # otherwise it will persist its results when it finishes
+            if self.verifier and self.verifier.is_up_to_date():
+                self.save_verified_tx(write=True)
+
+    def is_up_to_date(self):
+        with self.lock: return self.up_to_date
+
+    def get_num_tx(self, address):
+        """ return number of transactions where address is involved """
+        return len(self.history.get(address, []))
+
+    def get_tx_delta(self, tx_hash, address):
+        "effect of tx on address"
+        delta = 0
+        # substract the value of coins sent from address
+        d = self.txi.get(tx_hash, {}).get(address, [])
+        for n, v in d:
+            delta -= v
+        # add the value of the coins received at address
+        d = self.txo.get(tx_hash, {}).get(address, [])
+        for n, v, cb in d:
+            delta += v
+        return delta
+
+    def get_tx_value(self, txid):
+        " effect of tx on the entire domain"
+        delta = 0
+        for addr, d in self.txi.get(txid, {}).items():
+            for n, v in d:
+                delta -= v
+        for addr, d in self.txo.get(txid, {}).items():
+            for n, v, cb in d:
+                delta += v
+        return delta
+
+    def get_wallet_delta(self, tx):
+        """ effect of tx on wallet """
+        is_relevant = False  # "related to wallet?"
+        is_mine = False
+        is_pruned = False
+        is_partial = False
+        v_in = v_out = v_out_mine = 0
+        for txin in tx.inputs():
+            addr = self.get_txin_address(txin)
+            if self.is_mine(addr):
+                is_mine = True
+                is_relevant = True
+                d = self.txo.get(txin['prevout_hash'], {}).get(addr, [])
+                for n, v, cb in d:
+                    if n == txin['prevout_n']:
+                        value = v
+                        break
+                else:
+                    value = None
+                if value is None:
+                    is_pruned = True
+                else:
+                    v_in += value
+            else:
+                is_partial = True
+        if not is_mine:
+            is_partial = False
+        for addr, value in tx.get_outputs():
+            v_out += value
+            if self.is_mine(addr):
+                v_out_mine += value
+                is_relevant = True
+        if is_pruned:
+            # some inputs are mine:
+            fee = None
+            if is_mine:
+                v = v_out_mine - v_out
+            else:
+                # no input is mine
+                v = v_out_mine
+        else:
+            v = v_out_mine - v_in
+            if is_partial:
+                # some inputs are mine, but not all
+                fee = None
+            else:
+                # all inputs are mine
+                fee = v_in - v_out
+        if not is_mine:
+            fee = None
+        return is_relevant, is_mine, v, fee
+
+    def get_addr_io(self, address):
+        h = self.get_address_history(address)
+        received = {}
+        sent = {}
+        for tx_hash, height in h:
+            l = self.txo.get(tx_hash, {}).get(address, [])
+            for n, v, is_cb in l:
+                received[tx_hash + ':%d'%n] = (height, v, is_cb)
+        for tx_hash, height in h:
+            l = self.txi.get(tx_hash, {}).get(address, [])
+            for txi, v in l:
+                sent[txi] = height
+        return received, sent
+
+    def get_addr_utxo(self, address):
+        coins, spent = self.get_addr_io(address)
+        for txi in spent:
+            coins.pop(txi)
+        out = {}
+        for txo, v in coins.items():
+            tx_height, value, is_cb = v
+            prevout_hash, prevout_n = txo.split(':')
+            x = {
+                'address':address,
+                'value':value,
+                'prevout_n':int(prevout_n),
+                'prevout_hash':prevout_hash,
+                'height':tx_height,
+                'coinbase':is_cb
+            }
+            out[txo] = x
+        return out
+
+    # return the total amount ever received by an address
+    def get_addr_received(self, address):
+        received, sent = self.get_addr_io(address)
+        return sum([v for height, v, is_cb in received.values()])
+
+    @with_local_height_cached
+    def get_addr_balance(self, address):
+        """Return the balance of a bitcoin address:
+        confirmed and matured, unconfirmed, unmatured
+        """
+        received, sent = self.get_addr_io(address)
+        c = u = x = 0
+        local_height = self.get_local_height()
+        for txo, (tx_height, v, is_cb) in received.items():
+            if is_cb and tx_height + COINBASE_MATURITY > local_height:
+                x += v
+            elif tx_height > 0:
+                c += v
+            else:
+                u += v
+            if txo in sent:
+                if sent[txo] > 0:
+                    c -= v
+                else:
+                    u -= v
+        return c, u, x
+
+    @with_local_height_cached
+    def get_utxos(self, domain=None, excluded=None, mature=False, confirmed_only=False):
+        coins = []
+        if domain is None:
+            domain = self.get_addresses()
+        domain = set(domain)
+        if excluded:
+            domain = set(domain) - excluded
+        for addr in domain:
+            utxos = self.get_addr_utxo(addr)
+            for x in utxos.values():
+                if confirmed_only and x['height'] <= 0:
+                    continue
+                if mature and x['coinbase'] and x['height'] + COINBASE_MATURITY > self.get_local_height():
+                    continue
+                coins.append(x)
+                continue
+        return coins
+
+    def get_balance(self, domain=None):
+        if domain is None:
+            domain = self.get_addresses()
+        domain = set(domain)
+        cc = uu = xx = 0
+        for addr in domain:
+            c, u, x = self.get_addr_balance(addr)
+            cc += c
+            uu += u
+            xx += x
+        return cc, uu, xx
+
+    def is_used(self, address):
+        h = self.history.get(address,[])
+        return len(h) != 0
+
+    def is_empty(self, address):
+        c, u, x = self.get_addr_balance(address)
+        return c+u+x == 0

electrum/base_wizard.py

@@ -34,7 +34,7 @@ from .keystore import bip44_derivation, purpose48_derivation
 from .wallet import Imported_Wallet, Standard_Wallet, Multisig_Wallet, wallet_types, Wallet
 from .storage import STO_EV_USER_PW, STO_EV_XPUB_PW, get_derivation_used_for_hw_device_encryption
 from .i18n import _
-from .util import UserCancelled, InvalidPassword
+from .util import UserCancelled, InvalidPassword, WalletFileException
 
 # hardware device setup purpose
 HWD_SETUP_NEW_WALLET, HWD_SETUP_DECRYPT_WALLET = range(0, 2)
@@ -106,10 +106,20 @@ class BaseWizard(object):
         self.choice_dialog(title=title, message=message, choices=choices, run_next=self.on_wallet_type)
 
     def upgrade_storage(self):
+        exc = None
         def on_finished():
-            self.wallet = Wallet(self.storage)
-            self.terminate()
-        self.waiting_dialog(partial(self.storage.upgrade), _('Upgrading wallet format...'), on_finished=on_finished)
+            if exc is None:
+                self.wallet = Wallet(self.storage)
+                self.terminate()
+            else:
+                raise exc
+        def do_upgrade():
+            nonlocal exc
+            try:
+                self.storage.upgrade()
+            except Exception as e:
+                exc = e
+        self.waiting_dialog(do_upgrade, _('Upgrading wallet format...'), on_finished=on_finished)
 
     def load_2fa(self):
         self.storage.put('wallet_type', '2fa')
@@ -348,7 +358,7 @@ class BaseWizard(object):
         k = hardware_keystore(d)
         self.on_keystore(k)
 
-    def passphrase_dialog(self, run_next):
+    def passphrase_dialog(self, run_next, is_restoring=False):
         title = _('Seed extension')
         message = '\n'.join([
             _('You may extend your seed with custom words.'),
@@ -358,7 +368,10 @@ class BaseWizard(object):
             _('Note that this is NOT your encryption password.'),
             _('If you do not know what this is, leave this field empty.'),
         ])
-        self.line_dialog(title=title, message=message, warning=warning, default='', test=lambda x:True, run_next=run_next)
+        warn_issue4566 = is_restoring and self.seed_type == 'bip39'
+        self.line_dialog(title=title, message=message, warning=warning,
+                         default='', test=lambda x:True, run_next=run_next,
+                         warn_issue4566=warn_issue4566)
 
     def restore_from_seed(self):
         self.opt_bip39 = True
@@ -371,10 +384,10 @@ class BaseWizard(object):
         self.seed_type = 'bip39' if is_bip39 else bitcoin.seed_type(seed)
         if self.seed_type == 'bip39':
             f = lambda passphrase: self.on_restore_bip39(seed, passphrase)
-            self.passphrase_dialog(run_next=f) if is_ext else f('')
+            self.passphrase_dialog(run_next=f, is_restoring=True) if is_ext else f('')
         elif self.seed_type in ['standard', 'segwit']:
             f = lambda passphrase: self.run('create_keystore', seed, passphrase)
-            self.passphrase_dialog(run_next=f) if is_ext else f('')
+            self.passphrase_dialog(run_next=f, is_restoring=True) if is_ext else f('')
         elif self.seed_type == 'old':
             self.run('create_keystore', seed, '')
         elif self.seed_type == '2fa':

electrum/blockchain.py

@@ -79,12 +79,12 @@ def read_blockchains(config):
     l = filter(lambda x: x.startswith('fork_'), os.listdir(fdir))
     l = sorted(l, key = lambda x: int(x.split('_')[1]))
     for filename in l:
-        checkpoint = int(filename.split('_')[2])
+        forkpoint = int(filename.split('_')[2])
         parent_id = int(filename.split('_')[1])
-        b = Blockchain(config, checkpoint, parent_id)
-        h = b.read_header(b.checkpoint)
+        b = Blockchain(config, forkpoint, parent_id)
+        h = b.read_header(b.forkpoint)
         if b.parent().can_connect(h, check_height=False):
-            blockchains[b.checkpoint] = b
+            blockchains[b.forkpoint] = b
         else:
             util.print_error("cannot connect", filename)
     return blockchains
@@ -109,32 +109,39 @@ class Blockchain(util.PrintError):
     Manages blockchain headers and their verification
     """
 
-    def __init__(self, config, checkpoint, parent_id):
+    def __init__(self, config, forkpoint, parent_id):
         self.config = config
-        self.catch_up = None # interface catching up
-        self.checkpoint = checkpoint
+        self.catch_up = None  # interface catching up
+        self.forkpoint = forkpoint
         self.checkpoints = constants.net.CHECKPOINTS
         self.parent_id = parent_id
-        self.lock = threading.Lock()
+        assert parent_id != forkpoint
+        self.lock = threading.RLock()
         with self.lock:
             self.update_size()
 
+    def with_lock(func):
+        def func_wrapper(self, *args, **kwargs):
+            with self.lock:
+                return func(self, *args, **kwargs)
+        return func_wrapper
+
     def parent(self):
         return blockchains[self.parent_id]
 
     def get_max_child(self):
-        children = list(filter(lambda y: y.parent_id==self.checkpoint, blockchains.values()))
-        return max([x.checkpoint for x in children]) if children else None
+        children = list(filter(lambda y: y.parent_id==self.forkpoint, blockchains.values()))
+        return max([x.forkpoint for x in children]) if children else None
 
-    def get_checkpoint(self):
+    def get_forkpoint(self):
         mc = self.get_max_child()
-        return mc if mc is not None else self.checkpoint
+        return mc if mc is not None else self.forkpoint
 
     def get_branch_size(self):
-        return self.height() - self.get_checkpoint() + 1
+        return self.height() - self.get_forkpoint() + 1
 
     def get_name(self):
-        return self.get_hash(self.get_checkpoint()).lstrip('00')[0:10]
+        return self.get_hash(self.get_forkpoint()).lstrip('00')[0:10]
 
     def check_header(self, header):
         header_hash = hash_header(header)
@@ -142,14 +149,14 @@ class Blockchain(util.PrintError):
         return header_hash == self.get_hash(height)
 
     def fork(parent, header):
-        checkpoint = header.get('block_height')
-        self = Blockchain(parent.config, checkpoint, parent.checkpoint)
+        forkpoint = header.get('block_height')
+        self = Blockchain(parent.config, forkpoint, parent.forkpoint)
         open(self.path(), 'w+').close()
         self.save_header(header)
         return self
 
     def height(self):
-        return self.checkpoint + self.size() - 1
+        return self.forkpoint + self.size() - 1
 
     def size(self):
         with self.lock:
@@ -183,44 +190,55 @@ class Blockchain(util.PrintError):
 
     def path(self):
         d = util.get_headers_dir(self.config)
-        filename = 'blockchain_headers' if self.parent_id is None else os.path.join('forks', 'fork_%d_%d'%(self.parent_id, self.checkpoint))
+        filename = 'blockchain_headers' if self.parent_id is None else os.path.join('forks', 'fork_%d_%d'%(self.parent_id, self.forkpoint))
         return os.path.join(d, filename)
 
+    @with_lock
     def save_chunk(self, index, chunk):
-        filename = self.path()
-        d = (index * 2016 - self.checkpoint) * 80
-        if d < 0:
-            chunk = chunk[-d:]
-            d = 0
-        truncate = index >= len(self.checkpoints)
-        self.write(chunk, d, truncate)
+        chunk_within_checkpoint_region = index < len(self.checkpoints)
+        # chunks in checkpoint region are the responsibility of the 'main chain'
+        if chunk_within_checkpoint_region and self.parent_id is not None:
+            main_chain = blockchains[0]
+            main_chain.save_chunk(index, chunk)
+            return
+
+        delta_height = (index * 2016 - self.forkpoint)
+        delta_bytes = delta_height * 80
+        # if this chunk contains our forkpoint, only save the part after forkpoint
+        # (the part before is the responsibility of the parent)
+        if delta_bytes < 0:
+            chunk = chunk[-delta_bytes:]
+            delta_bytes = 0
+        truncate = not chunk_within_checkpoint_region
+        self.write(chunk, delta_bytes, truncate)
         self.swap_with_parent()
 
+    @with_lock
     def swap_with_parent(self):
         if self.parent_id is None:
             return
-        parent_branch_size = self.parent().height() - self.checkpoint + 1
+        parent_branch_size = self.parent().height() - self.forkpoint + 1
         if parent_branch_size >= self.size():
             return
-        self.print_error("swap", self.checkpoint, self.parent_id)
+        self.print_error("swap", self.forkpoint, self.parent_id)
         parent_id = self.parent_id
-        checkpoint = self.checkpoint
+        forkpoint = self.forkpoint
         parent = self.parent()
         self.assert_headers_file_available(self.path())
         with open(self.path(), 'rb') as f:
             my_data = f.read()
         self.assert_headers_file_available(parent.path())
         with open(parent.path(), 'rb') as f:
-            f.seek((checkpoint - parent.checkpoint)*80)
+            f.seek((forkpoint - parent.forkpoint)*80)
             parent_data = f.read(parent_branch_size*80)
         self.write(parent_data, 0)
-        parent.write(my_data, (checkpoint - parent.checkpoint)*80)
+        parent.write(my_data, (forkpoint - parent.forkpoint)*80)
         # store file path
         for b in blockchains.values():
             b.old_path = b.path()
         # swap parameters
         self.parent_id = parent.parent_id; parent.parent_id = parent_id
-        self.checkpoint = parent.checkpoint; parent.checkpoint = checkpoint
+        self.forkpoint = parent.forkpoint; parent.forkpoint = forkpoint
         self._size = parent._size; parent._size = parent_branch_size
         # move files
         for b in blockchains.values():
@@ -229,8 +247,8 @@ class Blockchain(util.PrintError):
                 self.print_error("renaming", b.old_path, b.path())
                 os.rename(b.old_path, b.path())
         # update pointers
-        blockchains[self.checkpoint] = self
-        blockchains[parent.checkpoint] = parent
+        blockchains[self.forkpoint] = self
+        blockchains[parent.forkpoint] = parent
 
     def assert_headers_file_available(self, path):
         if os.path.exists(path):
@@ -254,23 +272,25 @@ class Blockchain(util.PrintError):
                 os.fsync(f.fileno())
             self.update_size()
 
+    @with_lock
     def save_header(self, header):
-        delta = header.get('block_height') - self.checkpoint
+        delta = header.get('block_height') - self.forkpoint
         data = bfh(serialize_header(header))
+        # headers are only _appended_ to the end:
         assert delta == self.size()
         assert len(data) == 80
         self.write(data, delta*80)
         self.swap_with_parent()
 
     def read_header(self, height):
-        assert self.parent_id != self.checkpoint
+        assert self.parent_id != self.forkpoint
         if height < 0:
             return
-        if height < self.checkpoint:
+        if height < self.forkpoint:
             return self.parent().read_header(height)
         if height > self.height():
             return
-        delta = height - self.checkpoint
+        delta = height - self.forkpoint
         name = self.path()
         self.assert_headers_file_available(name)
         with open(name, 'rb') as f:

electrum/coinchooser.py

@@ -26,7 +26,7 @@ from collections import defaultdict, namedtuple
 from math import floor, log10
 
 from .bitcoin import sha256, COIN, TYPE_ADDRESS, is_address
-from .transaction import Transaction
+from .transaction import Transaction, TxOutput
 from .util import NotEnoughFunds, PrintError
 
 
@@ -178,7 +178,7 @@ class CoinChooserBase(PrintError):
         # size of the change output, add it to the transaction.
         dust = sum(amount for amount in amounts if amount < dust_threshold)
         amounts = [amount for amount in amounts if amount >= dust_threshold]
-        change = [(TYPE_ADDRESS, addr, amount)
+        change = [TxOutput(TYPE_ADDRESS, addr, amount)
                   for addr, amount in zip(change_addrs, amounts)]
         self.print_error('change:', change)
         if dust:

electrum/commands.py

@@ -38,7 +38,7 @@ from .util import bfh, bh2u, format_satoshis, json_decode, print_error, json_enc
 from . import bitcoin
 from .bitcoin import is_address,  hash_160, COIN, TYPE_ADDRESS
 from .i18n import _
-from .transaction import Transaction, multisig_script
+from .transaction import Transaction, multisig_script, TxOutput
 from .paymentrequest import PR_PAID, PR_UNPAID, PR_UNKNOWN, PR_EXPIRED
 from .plugin import run_hook
 
@@ -226,7 +226,7 @@ class Commands:
                 txin['signatures'] = [None]
                 txin['num_sig'] = 1
 
-        outputs = [(TYPE_ADDRESS, x['address'], int(x['value'])) for x in outputs]
+        outputs = [TxOutput(TYPE_ADDRESS, x['address'], int(x['value'])) for x in outputs]
         tx = Transaction.from_io(inputs, outputs, locktime=locktime)
         tx.sign(keypairs)
         return tx.as_dict()
@@ -415,7 +415,7 @@ class Commands:
         for address, amount in outputs:
             address = self._resolver(address)
             amount = satoshis(amount)
-            final_outputs.append((TYPE_ADDRESS, address, amount))
+            final_outputs.append(TxOutput(TYPE_ADDRESS, address, amount))
 
         coins = self.wallet.get_spendable_coins(domain, self.config)
         tx = self.wallet.make_unsigned_transaction(coins, final_outputs, self.config, fee, change_addr)
@@ -826,10 +826,13 @@ def add_network_options(parser):
     parser.add_argument("-1", "--oneserver", action="store_true", dest="oneserver", default=None, help="connect to one server only")
     parser.add_argument("-s", "--server", dest="server", default=None, help="set server host:port:protocol, where protocol is either t (tcp) or s (ssl)")
     parser.add_argument("-p", "--proxy", dest="proxy", default=None, help="set proxy [type:]host[:port], where type is socks4,socks5 or http")
+    parser.add_argument("--noonion", action="store_true", dest="noonion", default=None, help="do not try to connect to onion servers")
 
 def add_global_options(parser):
     group = parser.add_argument_group('global options')
-    group.add_argument("-v", "--verbose", action="store_true", dest="verbose", default=False, help="Show debugging information")
+    # const is for when no argument is given to verbosity
+    # default is for when the flag is missing
+    group.add_argument("-v", "--verbosity", dest="verbosity", help="Set verbosity filter", default='', const='*', nargs='?')
     group.add_argument("-D", "--dir", dest="electrum_path", help="electrum directory")
     group.add_argument("-P", "--portable", action="store_true", dest="portable", default=False, help="Use local 'electrum_data' directory")
     group.add_argument("-w", "--wallet", dest="wallet_path", help="wallet path")

electrum/gui/kivy/main_window.py

@@ -10,12 +10,12 @@ import threading
 from electrum.bitcoin import TYPE_ADDRESS
 from electrum.storage import WalletStorage
 from electrum.wallet import Wallet
-from electrum.i18n import _
 from electrum.paymentrequest import InvoiceStore
 from electrum.util import profiler, InvalidPassword
 from electrum.plugin import run_hook
 from electrum.util import format_satoshis, format_satoshis_plain
 from electrum.paymentrequest import PR_UNPAID, PR_PAID, PR_UNKNOWN, PR_EXPIRED
+from .i18n import _
 
 from kivy.app import App
 from kivy.core.window import Window
@@ -88,7 +88,7 @@ class ElectrumWindow(App):
     balance = StringProperty('')
     fiat_balance = StringProperty('')
     is_fiat = BooleanProperty(False)
-    blockchain_checkpoint = NumericProperty(0)
+    blockchain_forkpoint = NumericProperty(0)
 
     auto_connect = BooleanProperty(False)
     def on_auto_connect(self, instance, x):
@@ -159,6 +159,9 @@ class ElectrumWindow(App):
         Logger.info("on_history")
         self._trigger_update_history()
 
+    def on_fee_histogram(self, *args):
+        self._trigger_update_history()
+
     def _get_bu(self):
         decimal_point = self.electrum_config.get('decimal_point', 5)
         return decimal_point_to_base_unit_name(decimal_point)
@@ -391,9 +394,16 @@ class ElectrumWindow(App):
         popup.export = self.export_private_keys
         popup.open()
 
-    def qr_dialog(self, title, data, show_text=False):
+    def qr_dialog(self, title, data, show_text=False, text_for_clipboard=None):
         from .uix.dialogs.qr_dialog import QRDialog
-        popup = QRDialog(title, data, show_text)
+        def on_qr_failure():
+            popup.dismiss()
+            msg = _('Failed to display QR code.')
+            if text_for_clipboard:
+                msg += '\n' + _('Text copied to clipboard.')
+                self._clipboard.copy(text_for_clipboard)
+            Clock.schedule_once(lambda dt: self.show_info(msg))
+        popup = QRDialog(title, data, show_text, on_qr_failure)
         popup.open()
 
     def scan_qr(self, on_complete):
@@ -476,6 +486,7 @@ class ElectrumWindow(App):
             interests = ['updated', 'status', 'new_transaction', 'verified', 'interfaces']
             self.network.register_callback(self.on_network_event, interests)
             self.network.register_callback(self.on_fee, ['fee'])
+            self.network.register_callback(self.on_fee_histogram, ['fee_histogram'])
             self.network.register_callback(self.on_quotes, ['on_quotes'])
             self.network.register_callback(self.on_history, ['on_history'])
         # load wallet
@@ -643,7 +654,7 @@ class ElectrumWindow(App):
         self.num_nodes = len(self.network.get_interfaces())
         self.num_chains = len(self.network.get_blockchains())
         chain = self.network.blockchain()
-        self.blockchain_checkpoint = chain.get_checkpoint()
+        self.blockchain_forkpoint = chain.get_forkpoint()
         self.blockchain_name = chain.get_name()
         interface = self.network.interface
         if interface:
@@ -702,13 +713,14 @@ class ElectrumWindow(App):
         self.fiat_balance = self.fx.format_amount(c+u+x) + ' [size=22dp]%s[/size]'% self.fx.ccy
 
     def get_max_amount(self):
+        from electrum.transaction import TxOutput
         if run_hook('abort_send', self):
             return ''
         inputs = self.wallet.get_spendable_coins(None, self.electrum_config)
         if not inputs:
             return ''
         addr = str(self.send_screen.screen.address) or self.wallet.dummy_address()
-        outputs = [(TYPE_ADDRESS, addr, '!')]
+        outputs = [TxOutput(TYPE_ADDRESS, addr, '!')]
         try:
             tx = self.wallet.make_unsigned_transaction(inputs, outputs, self.electrum_config)
         except NoDynamicFeeEstimates as e:
@@ -875,6 +887,7 @@ class ElectrumWindow(App):
                     self.wallet.invoices.save()
                     self.update_tab('invoices')
             else:
+                msg = msg[:500] if msg else _('There was an error broadcasting the transaction.')
                 self.show_error(msg)
 
         if self.network and self.network.is_connected():

electrum/gui/kivy/uix/dialogs/__init__.py

@@ -206,9 +206,9 @@ class OutputList(RecycleView):
 
     def update(self, outputs):
         res = []
-        for (type, address, amount) in outputs:
-            value = self.app.format_amount_and_units(amount)
-            res.append({'address': address, 'value': value})
+        for o in outputs:
+            value = self.app.format_amount_and_units(o.value)
+            res.append({'address': o.address, 'value': value})
         self.data = res
 
 

electrum/gui/kivy/uix/dialogs/addresses.py

@@ -136,14 +136,14 @@ class AddressesDialog(Factory.Popup):
         for address in _list:
             label = wallet.labels.get(address, '')
             balance = sum(wallet.get_addr_balance(address))
-            is_used = wallet.is_used(address)
-            if self.show_used == 1 and (balance or is_used):
+            is_used_and_empty = wallet.is_used(address) and balance == 0
+            if self.show_used == 1 and (balance or is_used_and_empty):
                 continue
             if self.show_used == 2 and balance == 0:
                 continue
-            if self.show_used == 3 and not is_used:
+            if self.show_used == 3 and not is_used_and_empty:
                 continue
-            card = self.get_card(address, balance, is_used, label)
+            card = self.get_card(address, balance, is_used_and_empty, label)
             if search and not self.ext_search(card, search):
                 continue
             cards.append(card)

electrum/gui/kivy/uix/dialogs/crash_reporter.py

@@ -9,9 +9,9 @@ from kivy.lang import Builder
 from kivy.uix.label import Label
 from kivy.utils import platform
 
+from electrum.gui.kivy.i18n import _
 
 from electrum.base_crash_reporter import BaseCrashReporter
-from electrum.i18n import _
 
 
 Builder.load_string('''

electrum/gui/kivy/uix/dialogs/installwizard.py

@@ -957,7 +957,12 @@ class InstallWizard(BaseWizard, Widget):
             # on  completion hide message
             Clock.schedule_once(lambda dt: app.info_bubble.hide(now=True), -1)
             if on_finished:
-                Clock.schedule_once(lambda dt: on_finished(), -1)
+                def protected_on_finished():
+                    try:
+                        on_finished()
+                    except Exception as e:
+                        self.show_error(str(e))
+                Clock.schedule_once(lambda dt: protected_on_finished(), -1)
 
         app = App.get_running_app()
         app.show_info_bubble(

electrum/gui/kivy/uix/dialogs/qr_dialog.py

@@ -36,11 +36,12 @@ Builder.load_string('''
 ''')
 
 class QRDialog(Factory.Popup):
-    def __init__(self, title, data, show_text):
+    def __init__(self, title, data, show_text, failure_cb=None):
         Factory.Popup.__init__(self)
         self.title = title
         self.data = data
         self.show_text = show_text
+        self.failure_cb = failure_cb
 
     def on_open(self):
-        self.ids.qr.set_data(self.data)
+        self.ids.qr.set_data(self.data, self.failure_cb)

electrum/gui/kivy/uix/dialogs/tx_dialog.py

@@ -179,6 +179,7 @@ class TxDialog(Factory.Popup):
 
     def show_qr(self):
         from electrum.bitcoin import base_encode, bfh
-        text = bfh(str(self.tx))
+        raw_tx = str(self.tx)
+        text = bfh(raw_tx)
         text = base_encode(text, base=43)
-        self.app.qr_dialog(_("Raw Transaction"), text)
+        self.app.qr_dialog(_("Raw Transaction"), text, text_for_clipboard=raw_tx)

electrum/gui/kivy/uix/qrcodewidget.py

@@ -5,6 +5,7 @@ from threading import Thread
 from functools import partial
 
 import qrcode
+from qrcode import exceptions
 
 from kivy.uix.floatlayout import FloatLayout
 from kivy.graphics.texture import Texture
@@ -50,15 +51,23 @@ class QRCodeWidget(FloatLayout):
         self.data = None
         self.qr = None
         self._qrtexture = None
+        self.failure_cb = None
 
     def on_data(self, instance, value):
         if not (self.canvas or value):
             return
-        self.update_qr()
+        try:
+            self.update_qr()
+        except qrcode.exceptions.DataOverflowError:
+            if self.failure_cb:
+                self.failure_cb()
+            else:
+                raise
 
-    def set_data(self, data):
+    def set_data(self, data, failure_cb=None):
         if self.data == data:
             return
+        self.failure_cb = failure_cb
         MinSize = 210 if len(data) < 128 else 500
         self.setMinimumSize((MinSize, MinSize))
         self.data = data

electrum/gui/kivy/uix/screens.py

@@ -20,6 +20,7 @@ from kivy.utils import platform
 
 from electrum.util import profiler, parse_URI, format_time, InvalidPassword, NotEnoughFunds, Fiat
 from electrum import bitcoin
+from electrum.transaction import TxOutput
 from electrum.util import timestamp_to_datetime
 from electrum.paymentrequest import PR_UNPAID, PR_PAID, PR_UNKNOWN, PR_EXPIRED
 from electrum.plugin import run_hook
@@ -131,8 +132,8 @@ class HistoryScreen(CScreen):
         d = LabelDialog(_('Enter Transaction Label'), text, callback)
         d.open()
 
-    def get_card(self, tx_hash, height, conf, timestamp, value, balance):
-        status, status_str = self.app.wallet.get_tx_status(tx_hash, height, conf, timestamp)
+    def get_card(self, tx_hash, tx_mined_status, value, balance):
+        status, status_str = self.app.wallet.get_tx_status(tx_hash, tx_mined_status)
         icon = "atlas://electrum/gui/kivy/theming/light/" + TX_ICONS[status]
         label = self.app.wallet.get_label(tx_hash) if tx_hash else _('Pruned transaction outputs')
         ri = {}
@@ -141,7 +142,7 @@ class HistoryScreen(CScreen):
         ri['icon'] = icon
         ri['date'] = status_str
         ri['message'] = label
-        ri['confirmations'] = conf
+        ri['confirmations'] = tx_mined_status.conf
         if value is not None:
             ri['is_mine'] = value < 0
             if value < 0: value = - value
@@ -158,7 +159,6 @@ class HistoryScreen(CScreen):
             return
         history = reversed(self.app.wallet.get_history())
         history_card = self.screen.ids.history_container
-        count = 0
         history_card.data = [self.get_card(*item) for item in history]
 
 
@@ -257,7 +257,7 @@ class SendScreen(CScreen):
             except:
                 self.app.show_error(_('Invalid amount') + ':\n' + self.screen.amount)
                 return
-            outputs = [(bitcoin.TYPE_ADDRESS, address, amount)]
+            outputs = [TxOutput(bitcoin.TYPE_ADDRESS, address, amount)]
         message = self.screen.message
         amount = sum(map(lambda x:x[2], outputs))
         if self.app.electrum_config.get('use_rbf'):

electrum/gui/kivy/uix/ui_screens/network.kv

@@ -46,7 +46,7 @@ Popup:
 
                 CardSeparator
                 SettingsItem:
-                    title: _('Fork detected at block {}').format(app.blockchain_checkpoint) if app.num_chains>1 else _('No fork detected')
+                    title: _('Fork detected at block {}').format(app.blockchain_forkpoint) if app.num_chains>1 else _('No fork detected')
                     fork_description: (_('You are following branch') if app.auto_connect else _("Your server is on branch")) + ' ' + app.blockchain_name
                     description: self.fork_description if app.num_chains>1 else _('Connected nodes are on the same chain')
                     action: app.choose_blockchain_dialog

electrum/gui/qt/address_list.py

@@ -93,20 +93,20 @@ class AddressList(MyTreeWidget):
         else:
             addr_list = self.wallet.get_addresses()
         self.clear()
+        fx = self.parent.fx
         for address in addr_list:
-            num = len(self.wallet.get_address_history(address))
-            is_used = self.wallet.is_used(address)
+            num = self.wallet.get_address_history_len(address)
             label = self.wallet.labels.get(address, '')
             c, u, x = self.wallet.get_addr_balance(address)
             balance = c + u + x
-            if self.show_used == 1 and (balance or is_used):
+            is_used_and_empty = self.wallet.is_used(address) and balance == 0
+            if self.show_used == 1 and (balance or is_used_and_empty):
                 continue
             if self.show_used == 2 and balance == 0:
                 continue
-            if self.show_used == 3 and not is_used:
+            if self.show_used == 3 and not is_used_and_empty:
                 continue
             balance_text = self.parent.format_amount(balance, whitespaces=True)
-            fx = self.parent.fx
             # create item
             if fx and fx.get_fiat_address_config():
                 rate = fx.exchange_rate()

electrum/gui/qt/history_list.py

@@ -26,10 +26,10 @@
 import webbrowser
 import datetime
 
-from electrum.wallet import AddTransactionException, TX_HEIGHT_LOCAL
+from electrum.address_synchronizer import TX_HEIGHT_LOCAL
 from .util import *
 from electrum.i18n import _
-from electrum.util import block_explorer_URL, profiler, print_error
+from electrum.util import block_explorer_URL, profiler, print_error, TxMinedStatus
 
 try:
     from electrum.plot import plot_history, NothingToPlotException
@@ -229,6 +229,9 @@ class HistoryList(MyTreeWidget, AcceptFileDragDrop):
         current_tx = item.data(0, Qt.UserRole) if item else None
         self.clear()
         if fx: fx.history_used_spot = False
+        blue_brush = QBrush(QColor("#1E1EFF"))
+        red_brush = QBrush(QColor("#BC1E1E"))
+        monospace_font = QFont(MONOSPACE_FONT)
         for tx_item in self.transactions:
             tx_hash = tx_item['txid']
             height = tx_item['height']
@@ -237,7 +240,8 @@ class HistoryList(MyTreeWidget, AcceptFileDragDrop):
             value = tx_item['value'].value
             balance = tx_item['balance'].value
             label = tx_item['label']
-            status, status_str = self.wallet.get_tx_status(tx_hash, height, conf, timestamp)
+            tx_mined_status = TxMinedStatus(height, conf, timestamp, None)
+            status, status_str = self.wallet.get_tx_status(tx_hash, tx_mined_status)
             has_invoice = self.wallet.invoices.paid.get(tx_hash)
             icon = self.icon_cache.get(":icons/" + TX_ICONS[status])
             v_str = self.parent.format_amount(value, is_diff=True, whitespaces=True)
@@ -262,12 +266,12 @@ class HistoryList(MyTreeWidget, AcceptFileDragDrop):
                 if i>3:
                     item.setTextAlignment(i, Qt.AlignRight | Qt.AlignVCenter)
                 if i!=2:
-                    item.setFont(i, QFont(MONOSPACE_FONT))
+                    item.setFont(i, monospace_font)
             if value and value < 0:
-                item.setForeground(3, QBrush(QColor("#BC1E1E")))
-                item.setForeground(4, QBrush(QColor("#BC1E1E")))
+                item.setForeground(3, red_brush)
+                item.setForeground(4, red_brush)
             if fiat_value and not tx_item['fiat_default']:
-                item.setForeground(6, QBrush(QColor("#1E1EFF")))
+                item.setForeground(6, blue_brush)
             if tx_hash:
                 item.setData(0, Qt.UserRole, tx_hash)
             self.insertTopLevelItem(0, item)
@@ -304,10 +308,11 @@ class HistoryList(MyTreeWidget, AcceptFileDragDrop):
             label = self.wallet.get_label(txid)
             item.setText(3, label)
 
-    def update_item(self, tx_hash, height, conf, timestamp):
+    def update_item(self, tx_hash, tx_mined_status):
         if self.wallet is None:
             return
-        status, status_str = self.wallet.get_tx_status(tx_hash, height, conf, timestamp)
+        conf = tx_mined_status.conf
+        status, status_str = self.wallet.get_tx_status(tx_hash, tx_mined_status)
         icon = self.icon_cache.get(":icons/" +  TX_ICONS[status])
         items = self.findItems(tx_hash, Qt.UserRole|Qt.MatchContains|Qt.MatchRecursive, column=1)
         if items:
@@ -332,7 +337,7 @@ class HistoryList(MyTreeWidget, AcceptFileDragDrop):
             column_title = self.headerItem().text(column)
             column_data = item.text(column)
         tx_URL = block_explorer_URL(self.config, 'tx', tx_hash)
-        height, conf, timestamp = self.wallet.get_tx_height(tx_hash)
+        height = self.wallet.get_tx_height(tx_hash).height
         tx = self.wallet.transactions.get(tx_hash)
         is_relevant, is_mine, v, fee = self.wallet.get_wallet_delta(tx)
         is_unconfirmed = height <= 0

electrum/gui/qt/installwizard.py

@@ -32,6 +32,12 @@ WIF_HELP_TEXT = (_('WIF keys are typed in Electrum, based on script type.') + '\
                  'p2wpkh-p2sh:KxZcY47uGp9a... \t-> 3NhNeZQXF...\n' +
                  'p2wpkh:KxZcY47uGp9a...      \t-> bc1q3fjfk...')
 # note: full key is KxZcY47uGp9aVQAb6VVvuBs8SwHKgkSR2DbZUzjDzXf2N2GPhG9n
+MSG_PASSPHRASE_WARN_ISSUE4566 = _("Warning") + ": "\
+                              + _("You have multiple consecutive whitespaces or leading/trailing "
+                                  "whitespaces in your passphrase.") + " " \
+                              + _("This is discouraged.") + " " \
+                              + _("Due to a bug, old versions of Electrum will NOT be creating the "
+                                  "same wallet as newer versions or other software.")
 
 
 class CosignWidget(QWidget):
@@ -550,17 +556,24 @@ class InstallWizard(QDialog, MessageBoxMixin, BaseWizard):
 
     @wizard_dialog
     def line_dialog(self, run_next, title, message, default, test, warning='',
-                    presets=()):
+                    presets=(), warn_issue4566=False):
         vbox = QVBoxLayout()
         vbox.addWidget(WWLabel(message))
         line = QLineEdit()
         line.setText(default)
         def f(text):
             self.next_button.setEnabled(test(text))
+            if warn_issue4566:
+                text_whitespace_normalised = ' '.join(text.split())
+                warn_issue4566_label.setVisible(text != text_whitespace_normalised)
         line.textEdited.connect(f)
         vbox.addWidget(line)
         vbox.addWidget(WWLabel(warning))
 
+        warn_issue4566_label = WWLabel(MSG_PASSPHRASE_WARN_ISSUE4566)
+        warn_issue4566_label.setVisible(False)
+        vbox.addWidget(warn_issue4566_label)
+
         for preset in presets:
             button = QPushButton(preset[0])
             button.clicked.connect(lambda __, text=preset[1]: line.setText(text))
@@ -570,7 +583,7 @@ class InstallWizard(QDialog, MessageBoxMixin, BaseWizard):
             vbox.addLayout(hbox)
 
         self.exec_layout(vbox, title, next_enabled=test(default))
-        return ' '.join(line.text().split())
+        return line.text()
 
     @wizard_dialog
     def show_xpub_dialog(self, xpub, run_next):

electrum/gui/qt/main_window.py

@@ -50,8 +50,9 @@ from electrum.util import (format_time, format_satoshis, format_fee_satoshis,
                            export_meta, import_meta, bh2u, bfh, InvalidPassword,
                            base_units, base_units_list, base_unit_name_to_decimal_point,
                            decimal_point_to_base_unit_name, quantize_feerate)
-from electrum.transaction import Transaction
-from electrum.wallet import Multisig_Wallet, AddTransactionException, CannotBumpFee
+from electrum.transaction import Transaction, TxOutput
+from electrum.address_synchronizer import AddTransactionException
+from electrum.wallet import Multisig_Wallet, CannotBumpFee
 
 from .amountedit import AmountEdit, BTCAmountEdit, MyLineEdit, FeerateEdit
 from .qrcodewidget import QRCodeWidget, QRDialog
@@ -1305,7 +1306,7 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError):
             outputs = self.payto_e.get_outputs(self.is_max)
             if not outputs:
                 _type, addr = self.get_payto_or_dummy()
-                outputs = [(_type, addr, amount)]
+                outputs = [TxOutput(_type, addr, amount)]
             is_sweep = bool(self.tx_external_keypairs)
             make_tx = lambda fee_est: \
                 self.wallet.make_unsigned_transaction(
@@ -1484,14 +1485,14 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError):
             self.show_error(_('No outputs'))
             return
 
-        for _type, addr, amount in outputs:
-            if addr is None:
+        for o in outputs:
+            if o.address is None:
                 self.show_error(_('Bitcoin Address is None'))
                 return
-            if _type == TYPE_ADDRESS and not bitcoin.is_address(addr):
+            if o.type == TYPE_ADDRESS and not bitcoin.is_address(o.address):
                 self.show_error(_('Invalid Bitcoin Address'))
                 return
-            if amount is None:
+            if o.value is None:
                 self.show_error(_('Invalid Amount'))
                 return
 

electrum/gui/qt/network_dialog.py

@@ -106,9 +106,9 @@ class NodesListWidget(QTreeWidget):
             b = network.blockchains[k]
             name = b.get_name()
             if n_chains >1:
-                x = QTreeWidgetItem([name + '@%d'%b.get_checkpoint(), '%d'%b.height()])
+                x = QTreeWidgetItem([name + '@%d'%b.get_forkpoint(), '%d'%b.height()])
                 x.setData(0, Qt.UserRole, 1)
-                x.setData(1, Qt.UserRole, b.checkpoint)
+                x.setData(1, Qt.UserRole, b.forkpoint)
             else:
                 x = self
             for i in items:
@@ -357,9 +357,9 @@ class NetworkChoiceLayout(object):
         chains = self.network.get_blockchains()
         if len(chains)>1:
             chain = self.network.blockchain()
-            checkpoint = chain.get_checkpoint()
+            forkpoint = chain.get_forkpoint()
             name = chain.get_name()
-            msg = _('Chain split detected at block {0}').format(checkpoint) + '\n'
+            msg = _('Chain split detected at block {0}').format(forkpoint) + '\n'
             msg += (_('You are following branch') if auto_connect else _('Your server is on branch'))+ ' ' + name
             msg += ' (%d %s)' % (chain.get_branch_size(), _('blocks'))
         else:

electrum/gui/qt/paytoedit.py

@@ -29,13 +29,13 @@ from decimal import Decimal
 
 from electrum import bitcoin
 from electrum.util import bfh
+from electrum.transaction import TxOutput
 
 from .qrtextedit import ScanQRTextEdit
 from .completion_text_edit import CompletionTextEdit
 from . import util
 
-RE_ADDRESS = '[1-9A-HJ-NP-Za-km-z]{26,}'
-RE_ALIAS = '(.*?)\s*\<([1-9A-HJ-NP-Za-km-z]{26,})\>'
+RE_ALIAS = '(.*?)\s*\<([0-9A-Za-z]{1,})\>'
 
 frozen_style = "QWidget { background-color:none; border:none;}"
 normal_style = "QPlainTextEdit { }"
@@ -78,7 +78,7 @@ class PayToEdit(CompletionTextEdit, ScanQRTextEdit):
         x, y = line.split(',')
         out_type, out = self.parse_output(x)
         amount = self.parse_amount(y)
-        return out_type, out, amount
+        return TxOutput(out_type, out, amount)
 
     def parse_output(self, x):
         try:
@@ -140,16 +140,16 @@ class PayToEdit(CompletionTextEdit, ScanQRTextEdit):
         is_max = False
         for i, line in enumerate(lines):
             try:
-                _type, to_address, amount = self.parse_address_and_amount(line)
+                output = self.parse_address_and_amount(line)
             except:
                 self.errors.append((i, line.strip()))
                 continue
 
-            outputs.append((_type, to_address, amount))
-            if amount == '!':
+            outputs.append(output)
+            if output.value == '!':
                 is_max = True
             else:
-                total += amount
+                total += output.value
 
         self.win.is_max = is_max
         self.outputs = outputs
@@ -175,7 +175,7 @@ class PayToEdit(CompletionTextEdit, ScanQRTextEdit):
                 amount = self.amount_edit.get_amount()
 
             _type, addr = self.payto_address
-            self.outputs = [(_type, addr, amount)]
+            self.outputs = [TxOutput(_type, addr, amount)]
 
         return self.outputs[:]
 

electrum/gui/qt/seed_dialog.py

@@ -207,5 +207,6 @@ class SeedDialog(WindowModalDialog):
         title =  _("Your wallet generation seed is:")
         slayout = SeedLayout(title=title, seed=seed, msg=True, passphrase=passphrase)
         vbox.addLayout(slayout)
-        run_hook('set_seed', seed, slayout.seed_e)
+        has_extension = True if passphrase else False
+        run_hook('set_seed', seed, has_extension, slayout.seed_e)
         vbox.addLayout(Buttons(CloseButton(self)))

electrum/gui/qt/transaction_dialog.py

@@ -31,13 +31,15 @@ from PyQt5.QtCore import *
 from PyQt5.QtGui import *
 from PyQt5.QtWidgets import *
 
+import qrcode
+from qrcode import exceptions
+
 from electrum.bitcoin import base_encode
 from electrum.i18n import _
 from electrum.plugin import run_hook
 from electrum import simple_config
 
 from electrum.util import bfh
-from electrum.wallet import AddTransactionException
 from electrum.transaction import SerializationError
 
 from .util import *
@@ -184,8 +186,11 @@ class TxDialog(QDialog, MessageBoxMixin):
         text = base_encode(text, base=43)
         try:
             self.main_window.show_qrcode(text, 'Transaction', parent=self)
+        except qrcode.exceptions.DataOverflowError:
+            self.show_error(_('Failed to display QR code.') + '\n' +
+                            _('Transaction is too large in size.'))
         except Exception as e:
-            self.show_message(str(e))
+            self.show_error(_('Failed to display QR code.') + '\n' + str(e))
 
     def sign(self):
         def sign_done(success):

electrum/gui/stdio.py

@@ -4,6 +4,7 @@ _ = lambda x:x
 from electrum import WalletStorage, Wallet
 from electrum.util import format_satoshis, set_verbosity
 from electrum.bitcoin import is_address, COIN, TYPE_ADDRESS
+from electrum.transaction import TxOutput
 import getpass, datetime
 
 # minimal fdisk like gui for console usage
@@ -87,9 +88,9 @@ class ElectrumGui:
         + "%d"%(width[2]+delta)+"s"+"%"+"%d"%(width[3]+delta)+"s"
         messages = []
 
-        for item in self.wallet.get_history():
-            tx_hash, height, conf, timestamp, delta, balance = item
-            if conf:
+        for tx_hash, tx_mined_status, delta, balance in self.wallet.get_history():
+            if tx_mined_status.conf:
+                timestamp = tx_mined_status.timestamp
                 try:
                     time_str = datetime.datetime.fromtimestamp(timestamp).isoformat(' ')[:-3]
                 except Exception:
@@ -189,7 +190,8 @@ class ElectrumGui:
             if c == "n": return
 
         try:
-            tx = self.wallet.mktx([(TYPE_ADDRESS, self.str_recipient, amount)], password, self.config, fee)
+            tx = self.wallet.mktx([TxOutput(TYPE_ADDRESS, self.str_recipient, amount)],
+                                  password, self.config, fee)
         except Exception as e:
             print(str(e))
             return

electrum/gui/text.py

@@ -6,6 +6,7 @@ import getpass
 import electrum
 from electrum.util import format_satoshis, set_verbosity
 from electrum.bitcoin import is_address, COIN, TYPE_ADDRESS
+from electrum.transaction import TxOutput
 from .. import Wallet, WalletStorage
 
 _ = lambda x:x
@@ -109,9 +110,9 @@ class ElectrumGui:
 
         b = 0
         self.history = []
-        for item in self.wallet.get_history():
-            tx_hash, height, conf, timestamp, value, balance = item
-            if conf:
+        for tx_hash, tx_mined_status, value, balance in self.wallet.get_history():
+            if tx_mined_status.conf:
+                timestamp = tx_mined_status.timestamp
                 try:
                     time_str = datetime.datetime.fromtimestamp(timestamp).isoformat(' ')[:-3]
                 except Exception:
@@ -340,7 +341,8 @@ class ElectrumGui:
         else:
             password = None
         try:
-            tx = self.wallet.mktx([(TYPE_ADDRESS, self.str_recipient, amount)], password, self.config, fee)
+            tx = self.wallet.mktx([TxOutput(TYPE_ADDRESS, self.str_recipient, amount)],
+                                  password, self.config, fee)
         except Exception as e:
             self.show_message(str(e))
             return

electrum/interface.py

@@ -59,6 +59,7 @@ def Connection(server, queue, config_path):
 
 
 class TcpConnection(threading.Thread, util.PrintError):
+    verbosity_filter = 'i'
 
     def __init__(self, server, queue, config_path):
         threading.Thread.__init__(self)

electrum/keystore.py

@@ -552,13 +552,12 @@ def bip39_normalize_passphrase(passphrase):
     return normalize('NFKD', passphrase or '')
 
 def bip39_to_seed(mnemonic, passphrase):
-    import pbkdf2, hashlib, hmac
+    import hashlib, hmac
     PBKDF2_ROUNDS = 2048
     mnemonic = normalize('NFKD', ' '.join(mnemonic.split()))
     passphrase = bip39_normalize_passphrase(passphrase)
-    return pbkdf2.PBKDF2(mnemonic, 'mnemonic' + passphrase,
-                         iterations = PBKDF2_ROUNDS, macmodule = hmac,
-                         digestmodule = hashlib.sha512).read(64)
+    return hashlib.pbkdf2_hmac('sha512', mnemonic.encode('utf-8'),
+        b'mnemonic' + passphrase.encode('utf-8'), iterations = PBKDF2_ROUNDS)
 
 # returns tuple (is_checksum_valid, is_wordlist_valid)
 def bip39_is_checksum_valid(mnemonic):

electrum/mnemonic.py

@@ -30,7 +30,6 @@ import unicodedata
 import string
 
 import ecdsa
-import pbkdf2
 
 from .util import print_error
 from .bitcoin import is_old_seed, is_new_seed
@@ -131,7 +130,7 @@ class Mnemonic(object):
         PBKDF2_ROUNDS = 2048
         mnemonic = normalize_text(mnemonic)
         passphrase = normalize_text(passphrase)
-        return pbkdf2.PBKDF2(mnemonic, 'electrum' + passphrase, iterations = PBKDF2_ROUNDS, macmodule = hmac, digestmodule = hashlib.sha512).read(64)
+        return hashlib.pbkdf2_hmac('sha512', mnemonic.encode('utf-8'), b'electrum' + passphrase.encode('utf-8'), iterations = PBKDF2_ROUNDS)
 
     def mnemonic_encode(self, i):
         n = len(self.wordlist)

electrum/network.py

@@ -89,6 +89,10 @@ def filter_version(servers):
     return {k: v for k, v in servers.items() if is_recent(v.get('version'))}
 
 
+def filter_noonion(servers):
+    return {k: v for k, v in servers.items() if not k.endswith('.onion')}
+
+
 def filter_protocol(hostmap, protocol='s'):
     '''Filters the hostmap for those implementing protocol.
     The result is a list in serialized form.'''
@@ -170,6 +174,7 @@ class Network(util.DaemonThread):
           get_parameters(), get_server_height(), get_status_value(),
           is_connected(), set_parameters(), stop()
     """
+    verbosity_filter = 'n'
 
     def __init__(self, config=None):
         if config is None:
@@ -408,6 +413,8 @@ class Network(util.DaemonThread):
                     continue
                 if host not in out:
                     out[host] = {protocol: port}
+        if self.config.get('noonion'):
+            out = filter_noonion(out)
         return out
 
     @with_interface_lock
@@ -875,13 +882,17 @@ class Network(util.DaemonThread):
         if not connect:
             self.connection_down(interface.server)
             return
-        # If not finished, get the next chunk
-        if index >= len(blockchain.checkpoints) and blockchain.height() < interface.tip:
-            self.request_chunk(interface, index+1)
+        if index >= len(blockchain.checkpoints):
+            # If not finished, get the next chunk
+            if blockchain.height() < interface.tip:
+                self.request_chunk(interface, index+1)
+            else:
+                interface.mode = 'default'
+                interface.print_error('catch up done', blockchain.height())
+                blockchain.catch_up = None
         else:
-            interface.mode = 'default'
-            interface.print_error('catch up done', blockchain.height())
-            blockchain.catch_up = None
+            # the verifier must have asked for this chunk
+            pass
         self.notify('updated')
 
     def on_get_header(self, interface, response):
@@ -892,6 +903,7 @@ class Network(util.DaemonThread):
             self.connection_down(interface.server)
             return
         height = header.get('block_height')
+        #interface.print_error('got header', height, blockchain.hash_header(header))
         if interface.request != height:
             interface.print_error("unsolicited header",interface.request, height)
             self.connection_down(interface.server)
@@ -906,6 +918,9 @@ class Network(util.DaemonThread):
                 next_height = height + 1
                 interface.blockchain.catch_up = interface.server
             elif chain:
+                # FIXME should await "initial chunk download".
+                # binary search will NOT do the correct thing if we don't yet
+                # have all headers up to the fork height
                 interface.print_error("binary search")
                 interface.mode = 'binary'
                 interface.blockchain = chain
@@ -947,9 +962,9 @@ class Network(util.DaemonThread):
                     elif branch.parent().check_header(header):
                         interface.print_error('reorg', interface.bad, interface.tip)
                         interface.blockchain = branch.parent()
-                        next_height = None
+                        next_height = interface.bad
                     else:
-                        interface.print_error('checkpoint conflicts with existing fork', branch.path())
+                        interface.print_error('forkpoint conflicts with existing fork', branch.path())
                         branch.write(b'', 0)
                         branch.save_header(interface.bad_header)
                         interface.mode = 'catch_up'
@@ -965,10 +980,12 @@ class Network(util.DaemonThread):
                             with self.blockchains_lock:
                                 self.blockchains[interface.bad] = b
                             interface.blockchain = b
-                            interface.print_error("new chain", b.checkpoint)
+                            interface.print_error("new chain", b.forkpoint)
                             interface.mode = 'catch_up'
-                            next_height = interface.bad + 1
-                            interface.blockchain.catch_up = interface.server
+                            maybe_next_height = interface.bad + 1
+                            if maybe_next_height <= interface.tip:
+                                next_height = maybe_next_height
+                                interface.blockchain.catch_up = interface.server
                     else:
                         assert bh == interface.good
                         if interface.blockchain.catch_up is None and bh < interface.tip:
@@ -1081,6 +1098,7 @@ class Network(util.DaemonThread):
         except InvalidHeader:
             self.connection_down(interface.server)
             return
+        #interface.print_error('notified of header', height, blockchain.hash_header(header))
         if height < self.max_checkpoint():
             self.connection_down(interface.server)
             return
@@ -1125,7 +1143,7 @@ class Network(util.DaemonThread):
     @with_interface_lock
     def blockchain(self):
         if self.interface and self.interface.blockchain is not None:
-            self.blockchain_index = self.interface.blockchain.checkpoint
+            self.blockchain_index = self.interface.blockchain.forkpoint
         return self.blockchains[self.blockchain_index]
 
     @with_interface_lock

electrum/paymentrequest.py

@@ -42,6 +42,7 @@ from .util import print_error, bh2u, bfh
 from .util import export_meta, import_meta
 
 from .bitcoin import TYPE_ADDRESS
+from .transaction import TxOutput
 
 REQUEST_HEADERS = {'Accept': 'application/bitcoin-paymentrequest', 'User-Agent': 'Electrum'}
 ACK_HEADERS = {'Content-Type':'application/bitcoin-payment','Accept':'application/bitcoin-paymentack','User-Agent':'Electrum'}
@@ -123,7 +124,7 @@ class PaymentRequest:
         self.outputs = []
         for o in self.details.outputs:
             addr = transaction.get_address_from_output_script(o.script)[1]
-            self.outputs.append((TYPE_ADDRESS, addr, o.amount))
+            self.outputs.append(TxOutput(TYPE_ADDRESS, addr, o.amount))
         self.memo = self.details.memo
         self.payment_url = self.details.payment_url
 
@@ -225,8 +226,8 @@ class PaymentRequest:
 
     def get_address(self):
         o = self.outputs[0]
-        assert o[0] == TYPE_ADDRESS
-        return o[1]
+        assert o.type == TYPE_ADDRESS
+        return o.address
 
     def get_requestor(self):
         return self.requestor if self.requestor else self.get_address()

electrum/plugin.py

@@ -26,7 +26,6 @@ from collections import namedtuple
 import traceback
 import sys
 import os
-import imp
 import pkgutil
 import time
 import threading
@@ -43,6 +42,7 @@ hooks = {}
 
 
 class Plugins(DaemonThread):
+    verbosity_filter = 'p'
 
     @profiler
     def __init__(self, config, is_local, gui_name):

electrum/plugins/cosigner_pool/qt.py

@@ -38,6 +38,7 @@ from electrum.wallet import Multisig_Wallet
 from electrum.util import bh2u, bfh
 
 from electrum.gui.qt.transaction_dialog import show_transaction
+from electrum.gui.qt.util import WaitingDialog
 
 import sys
 import traceback
@@ -170,20 +171,26 @@ class Plugin(BasePlugin):
         return cosigner_xpub in xpub_set
 
     def do_send(self, tx):
+        def on_success(result):
+            window.show_message(_("Your transaction was sent to the cosigning pool.") + '\n' +
+                                _("Open your cosigner wallet to retrieve it."))
+        def on_failure(exc_info):
+            e = exc_info[1]
+            try: traceback.print_exception(*exc_info)
+            except OSError: pass
+            window.show_error(_("Failed to send transaction to cosigning pool") + ':\n' + str(e))
+
         for window, xpub, K, _hash in self.cosigner_list:
             if not self.cosigner_can_sign(tx, xpub):
                 continue
+            # construct message
             raw_tx_bytes = bfh(str(tx))
             public_key = ecc.ECPubkey(K)
             message = public_key.encrypt_message(raw_tx_bytes).decode('ascii')
-            try:
-                server.put(_hash, message)
-            except Exception as e:
-                traceback.print_exc(file=sys.stdout)
-                window.show_error(_("Failed to send transaction to cosigning pool") + ':\n' + str(e))
-                return
-            window.show_message(_("Your transaction was sent to the cosigning pool.") + '\n' +
-                                _("Open your cosigner wallet to retrieve it."))
+            # send message
+            task = lambda: server.put(_hash, message)
+            msg = _('Sending transaction to cosigning pool...')
+            WaitingDialog(window, msg, task, on_success, on_failure)
 
     def on_receive(self, keyhash, message):
         self.print_error("signal arrived for", keyhash)

electrum/plugins/digitalbitbox/digitalbitbox.py

@@ -120,8 +120,8 @@ class DigitalBitbox_Client():
 
 
     def stretch_key(self, key):
-        import pbkdf2, hmac
-        return to_hexstr(pbkdf2.PBKDF2(key, b'Digital Bitbox', iterations = 20480, macmodule = hmac, digestmodule = hashlib.sha512).read(64))
+        import hmac
+        return to_hexstr(hashlib.pbkdf2_hmac('sha512', key.encode('utf-8'), b'Digital Bitbox', iterations = 20480))
 
 
     def backup_password_dialog(self):
@@ -534,9 +534,9 @@ class DigitalBitbox_KeyStore(Hardware_KeyStore):
                     self.give_error("No matching x_key for sign_transaction") # should never happen
 
             # Build pubkeyarray from outputs
-            for _type, address, amount in tx.outputs():
-                assert _type == TYPE_ADDRESS
-                info = tx.output_info.get(address)
+            for o in tx.outputs():
+                assert o.type == TYPE_ADDRESS
+                info = tx.output_info.get(o.address)
                 if info is not None:
                     index, xpubs, m = info
                     changePath = self.get_derivation() + "/%d/%d" % index

electrum/plugins/hw_wallet/plugin.py

@@ -26,7 +26,9 @@
 
 from electrum.plugin import BasePlugin, hook
 from electrum.i18n import _
-from electrum.bitcoin import is_address
+from electrum.bitcoin import is_address, TYPE_SCRIPT
+from electrum.util import bfh
+from electrum.transaction import opcodes, TxOutput
 
 
 class HW_PluginBase(BasePlugin):
@@ -87,3 +89,15 @@ def is_any_tx_output_on_change_branch(tx):
             if index[0] == 1:
                 return True
     return False
+
+
+def trezor_validate_op_return_output_and_get_data(output: TxOutput) -> bytes:
+    if output.type != TYPE_SCRIPT:
+        raise Exception("Unexpected output type: {}".format(output.type))
+    script = bfh(output.address)
+    if not (script[0] == opcodes.OP_RETURN and
+            script[1] == len(script) - 2 and script[1] <= 75):
+        raise Exception(_("Only OP_RETURN scripts, with one constant push, are supported."))
+    if output.value != 0:
+        raise Exception(_("Amount for OP_RETURN output must be zero."))
+    return script[2:]

electrum/plugins/keepkey/keepkey.py

@@ -15,7 +15,7 @@ from electrum.wallet import Standard_Wallet
 from electrum.base_wizard import ScriptTypeNotSupported
 
 from ..hw_wallet import HW_PluginBase
-from ..hw_wallet.plugin import is_any_tx_output_on_change_branch
+from ..hw_wallet.plugin import is_any_tx_output_on_change_branch, trezor_validate_op_return_output_and_get_data
 
 
 # TREZOR initialization methods
@@ -141,7 +141,10 @@ class KeepKeyPlugin(HW_PluginBase):
                      'download the updated firmware from {}')
                    .format(self.device, client.label(), self.firmware_URL))
             self.print_error(msg)
-            handler.show_error(msg)
+            if handler:
+                handler.show_error(msg)
+            else:
+                raise Exception(msg)
             return None
 
         return client
@@ -288,7 +291,7 @@ class KeepKeyPlugin(HW_PluginBase):
         for txin in tx.inputs():
             txinputtype = self.types.TxInputType()
             if txin['type'] == 'coinbase':
-                prev_hash = "\0"*32
+                prev_hash = b"\x00"*32
                 prev_index = 0xffffffff  # signed int -1
             else:
                 if for_sig:
@@ -379,7 +382,7 @@ class KeepKeyPlugin(HW_PluginBase):
             txoutputtype.amount = amount
             if _type == TYPE_SCRIPT:
                 txoutputtype.script_type = self.types.PAYTOOPRETURN
-                txoutputtype.op_return_data = address[2:]
+                txoutputtype.op_return_data = trezor_validate_op_return_output_and_get_data(o)
             elif _type == TYPE_ADDRESS:
                 if is_segwit_address(address):
                     txoutputtype.script_type = self.types.PAYTOWITNESS
@@ -398,7 +401,8 @@ class KeepKeyPlugin(HW_PluginBase):
         has_change = False
         any_output_on_change_branch = is_any_tx_output_on_change_branch(tx)
 
-        for _type, address, amount in tx.outputs():
+        for o in tx.outputs():
+            _type, address, amount = o.type, o.address, o.value
             use_create_by_derivation = False
 
             info = tx.output_info.get(address)

electrum/plugins/ledger/ledger.py

@@ -12,7 +12,7 @@ from electrum.transaction import Transaction
 from electrum.wallet import Standard_Wallet
 from ..hw_wallet import HW_PluginBase
 from ..hw_wallet.plugin import is_any_tx_output_on_change_branch
-from electrum.util import print_error, is_verbose, bfh, bh2u, versiontuple
+from electrum.util import print_error, bfh, bh2u, versiontuple
 from electrum.base_wizard import ScriptTypeNotSupported
 
 try:
@@ -24,7 +24,7 @@ try:
     from btchip.btchipFirmwareWizard import checkFirmware, updateFirmware
     from btchip.btchipException import BTChipException
     BTCHIP = True
-    BTCHIP_DEBUG = is_verbose
+    BTCHIP_DEBUG = False
 except ImportError:
     BTCHIP = False
 
@@ -394,9 +394,9 @@ class Ledger_KeyStore(Hardware_KeyStore):
                     self.give_error("Transaction with more than 2 outputs not supported")
             has_change = False
             any_output_on_change_branch = is_any_tx_output_on_change_branch(tx)
-            for _type, address, amount in tx.outputs():
-                assert _type == TYPE_ADDRESS
-                info = tx.output_info.get(address)
+            for o in tx.outputs():
+                assert o.type == TYPE_ADDRESS
+                info = tx.output_info.get(o.address)
                 if (info is not None) and len(tx.outputs()) > 1 \
                         and not has_change:
                     index, xpubs, m = info
@@ -407,9 +407,9 @@ class Ledger_KeyStore(Hardware_KeyStore):
                         changePath = self.get_derivation()[2:] + "/%d/%d"%index
                         has_change = True
                     else:
-                        output = address
+                        output = o.address
                 else:
-                    output = address
+                    output = o.address
 
         self.handler.show_message(_("Confirm Transaction on your Ledger device..."))
         try:

electrum/plugins/revealer/qt.py

@@ -39,9 +39,9 @@ class Plugin(BasePlugin):
         BasePlugin.__init__(self, parent, config, name)
         self.base_dir = config.electrum_path()+'/revealer/'
 
-        if self.config.get('calibration_h') == None:
+        if self.config.get('calibration_h') is None:
             self.config.set_key('calibration_h', 0)
-        if self.config.get('calibration_v') == None:
+        if self.config.get('calibration_v') is None:
             self.config.set_key('calibration_v', 0)
 
         self.calibration_h = self.config.get('calibration_h')
@@ -57,8 +57,9 @@ class Plugin(BasePlugin):
         make_dir(self.base_dir)
 
     @hook
-    def set_seed(self, seed, parent):
+    def set_seed(self, seed, has_extension, parent):
         self.cseed = seed.upper()
+        self.has_extension = has_extension
         parent.addButton(':icons/revealer.png', partial(self.setup_dialog, parent), "Revealer"+_(" secret backup utility"))
 
     def requires_settings(self):
@@ -168,6 +169,10 @@ class Plugin(BasePlugin):
                                      "<br/>","<b>", self.base_dir+ self.filename+self.version+"_"+self.code_id,"</b>"]))
         dialog.close()
 
+    def ext_warning(self, dialog):
+        dialog.show_message(''.join(["<b>",_("Warning: "), "</b>", _("your seed extension will not be included in the encrypted backup.")]))
+        dialog.close()
+
     def bdone(self, dialog):
         dialog.show_message(''.join([_("Digital Revealer ({}_{}) saved as PNG and PDF at:").format(self.version, self.code_id),
                                      "<br/>","<b>", self.base_dir + 'revealer_' +self.version + '_'+ self.code_id, '</b>']))
@@ -263,7 +268,7 @@ class Plugin(BasePlugin):
             max_letters = 17
             max_lines = 6
             max_words = 3
-        if len(txt) > 102:
+        else:
             fontsize = 9
             linespace = 10
             max_letters = 24
@@ -360,6 +365,9 @@ class Plugin(BasePlugin):
             self.filename = self.wallet_name+'_'+ _('seed')+'_'
             self.was = self.wallet_name +' ' + _('seed')
 
+        if self.has_extension:
+            self.ext_warning(self.c_dialog)
+
         if not calibration:
             self.toPdf(QImage(cypherseed))
             QDesktopServices.openUrl (QUrl.fromLocalFile(os.path.abspath(self.base_dir+self.filename+self.version+'_'+self.code_id+'.pdf')))
@@ -588,8 +596,8 @@ class Plugin(BasePlugin):
                 qr_qt = self.paintQR(self.hex_noise.upper() +self.code_id)
                 target = QRectF(base_img.width()-65-qr_size,
                                 base_img.height()-65-qr_size,
-                                qr_size, qr_size );
-                painter.drawImage(target, qr_qt);
+                                qr_size, qr_size )
+                painter.drawImage(target, qr_qt)
                 painter.setPen(QPen(Qt.black, 4))
                 painter.drawLine(base_img.width()-65-qr_size,
                                 base_img.height()-65-qr_size,

electrum/plugins/safe_t/__init__.py

@@ -0,0 +1,8 @@
+from electrum.i18n import _
+
+fullname = 'Safe-T mini Wallet'
+description = _('Provides support for Safe-T mini hardware wallet')
+requires = [('safetlib','github.com/archos-safe-t/python-safet')]
+registers_keystore = ('hardware', 'safe_t', _("Safe-T mini wallet"))
+available_for = ['qt', 'cmdline']
+

electrum/plugins/safe_t/client.py

@@ -0,0 +1,11 @@
+from safetlib.client import proto, BaseClient, ProtocolMixin
+from .clientbase import SafeTClientBase
+
+class SafeTClient(SafeTClientBase, ProtocolMixin, BaseClient):
+    def __init__(self, transport, handler, plugin):
+        BaseClient.__init__(self, transport=transport)
+        ProtocolMixin.__init__(self, transport=transport)
+        SafeTClientBase.__init__(self, handler, plugin, proto)
+
+
+SafeTClientBase.wrap_methods(SafeTClient)

electrum/plugins/safe_t/clientbase.py

@@ -0,0 +1,252 @@
+import time
+from struct import pack
+
+from electrum.i18n import _
+from electrum.util import PrintError, UserCancelled
+from electrum.keystore import bip39_normalize_passphrase
+from electrum.bitcoin import serialize_xpub
+
+
+class GuiMixin(object):
+    # Requires: self.proto, self.device
+
+    # ref: https://github.com/trezor/trezor-common/blob/44dfb07cfaafffada4b2ce0d15ba1d90d17cf35e/protob/types.proto#L89
+    messages = {
+        3: _("Confirm the transaction output on your {} device"),
+        4: _("Confirm internal entropy on your {} device to begin"),
+        5: _("Write down the seed word shown on your {}"),
+        6: _("Confirm on your {} that you want to wipe it clean"),
+        7: _("Confirm on your {} device the message to sign"),
+        8: _("Confirm the total amount spent and the transaction fee on your "
+             "{} device"),
+        10: _("Confirm wallet address on your {} device"),
+        14: _("Choose on your {} device where to enter your passphrase"),
+        'default': _("Check your {} device to continue"),
+    }
+
+    def callback_Failure(self, msg):
+        # BaseClient's unfortunate call() implementation forces us to
+        # raise exceptions on failure in order to unwind the stack.
+        # However, making the user acknowledge they cancelled
+        # gets old very quickly, so we suppress those.  The NotInitialized
+        # one is misnamed and indicates a passphrase request was cancelled.
+        if msg.code in (self.types.FailureType.PinCancelled,
+                        self.types.FailureType.ActionCancelled,
+                        self.types.FailureType.NotInitialized):
+            raise UserCancelled()
+        raise RuntimeError(msg.message)
+
+    def callback_ButtonRequest(self, msg):
+        message = self.msg
+        if not message:
+            message = self.messages.get(msg.code, self.messages['default'])
+        self.handler.show_message(message.format(self.device), self.cancel)
+        return self.proto.ButtonAck()
+
+    def callback_PinMatrixRequest(self, msg):
+        if msg.type == 2:
+            msg = _("Enter a new PIN for your {}:")
+        elif msg.type == 3:
+            msg = (_("Re-enter the new PIN for your {}.\n\n"
+                     "NOTE: the positions of the numbers have changed!"))
+        else:
+            msg = _("Enter your current {} PIN:")
+        pin = self.handler.get_pin(msg.format(self.device))
+        if len(pin) > 9:
+            self.handler.show_error(_('The PIN cannot be longer than 9 characters.'))
+            pin = ''  # to cancel below
+        if not pin:
+            return self.proto.Cancel()
+        return self.proto.PinMatrixAck(pin=pin)
+
+    def callback_PassphraseRequest(self, req):
+        if req and hasattr(req, 'on_device') and req.on_device is True:
+            return self.proto.PassphraseAck()
+
+        if self.creating_wallet:
+            msg = _("Enter a passphrase to generate this wallet.  Each time "
+                    "you use this wallet your {} will prompt you for the "
+                    "passphrase.  If you forget the passphrase you cannot "
+                    "access the bitcoins in the wallet.").format(self.device)
+        else:
+            msg = _("Enter the passphrase to unlock this wallet:")
+        passphrase = self.handler.get_passphrase(msg, self.creating_wallet)
+        if passphrase is None:
+            return self.proto.Cancel()
+        passphrase = bip39_normalize_passphrase(passphrase)
+
+        ack = self.proto.PassphraseAck(passphrase=passphrase)
+        length = len(ack.passphrase)
+        if length > 50:
+            self.handler.show_error(_("Too long passphrase ({} > 50 chars).").format(length))
+            return self.proto.Cancel()
+        return ack
+
+    def callback_PassphraseStateRequest(self, msg):
+        return self.proto.PassphraseStateAck()
+
+    def callback_WordRequest(self, msg):
+        self.step += 1
+        msg = _("Step {}/24.  Enter seed word as explained on "
+                "your {}:").format(self.step, self.device)
+        word = self.handler.get_word(msg)
+        # Unfortunately the device can't handle self.proto.Cancel()
+        return self.proto.WordAck(word=word)
+
+
+class SafeTClientBase(GuiMixin, PrintError):
+
+    def __init__(self, handler, plugin, proto):
+        assert hasattr(self, 'tx_api')  # ProtocolMixin already constructed?
+        self.proto = proto
+        self.device = plugin.device
+        self.handler = handler
+        self.tx_api = plugin
+        self.types = plugin.types
+        self.msg = None
+        self.creating_wallet = False
+        self.used()
+
+    def __str__(self):
+        return "%s/%s" % (self.label(), self.features.device_id)
+
+    def label(self):
+        '''The name given by the user to the device.'''
+        return self.features.label
+
+    def is_initialized(self):
+        '''True if initialized, False if wiped.'''
+        return self.features.initialized
+
+    def is_pairable(self):
+        return not self.features.bootloader_mode
+
+    def has_usable_connection_with_device(self):
+        try:
+            res = self.ping("electrum pinging device")
+            assert res == "electrum pinging device"
+        except BaseException:
+            return False
+        return True
+
+    def used(self):
+        self.last_operation = time.time()
+
+    def prevent_timeouts(self):
+        self.last_operation = float('inf')
+
+    def timeout(self, cutoff):
+        '''Time out the client if the last operation was before cutoff.'''
+        if self.last_operation < cutoff:
+            self.print_error("timed out")
+            self.clear_session()
+
+    @staticmethod
+    def expand_path(n):
+        '''Convert bip32 path to list of uint32 integers with prime flags
+        0/-1/1' -> [0, 0x80000001, 0x80000001]'''
+        # This code is similar to code in safetlib where it unfortunately
+        # is not declared as a staticmethod.  Our n has an extra element.
+        PRIME_DERIVATION_FLAG = 0x80000000
+        path = []
+        for x in n.split('/')[1:]:
+            prime = 0
+            if x.endswith("'"):
+                x = x.replace('\'', '')
+                prime = PRIME_DERIVATION_FLAG
+            if x.startswith('-'):
+                prime = PRIME_DERIVATION_FLAG
+            path.append(abs(int(x)) | prime)
+        return path
+
+    def cancel(self):
+        '''Provided here as in keepkeylib but not safetlib.'''
+        self.transport.write(self.proto.Cancel())
+
+    def i4b(self, x):
+        return pack('>I', x)
+
+    def get_xpub(self, bip32_path, xtype):
+        address_n = self.expand_path(bip32_path)
+        creating = False
+        node = self.get_public_node(address_n, creating).node
+        return serialize_xpub(xtype, node.chain_code, node.public_key, node.depth, self.i4b(node.fingerprint), self.i4b(node.child_num))
+
+    def toggle_passphrase(self):
+        if self.features.passphrase_protection:
+            self.msg = _("Confirm on your {} device to disable passphrases")
+        else:
+            self.msg = _("Confirm on your {} device to enable passphrases")
+        enabled = not self.features.passphrase_protection
+        self.apply_settings(use_passphrase=enabled)
+
+    def change_label(self, label):
+        self.msg = _("Confirm the new label on your {} device")
+        self.apply_settings(label=label)
+
+    def change_homescreen(self, homescreen):
+        self.msg = _("Confirm on your {} device to change your home screen")
+        self.apply_settings(homescreen=homescreen)
+
+    def set_pin(self, remove):
+        if remove:
+            self.msg = _("Confirm on your {} device to disable PIN protection")
+        elif self.features.pin_protection:
+            self.msg = _("Confirm on your {} device to change your PIN")
+        else:
+            self.msg = _("Confirm on your {} device to set a PIN")
+        self.change_pin(remove)
+
+    def clear_session(self):
+        '''Clear the session to force pin (and passphrase if enabled)
+        re-entry.  Does not leak exceptions.'''
+        self.print_error("clear session:", self)
+        self.prevent_timeouts()
+        try:
+            super(SafeTClientBase, self).clear_session()
+        except BaseException as e:
+            # If the device was removed it has the same effect...
+            self.print_error("clear_session: ignoring error", str(e))
+
+    def get_public_node(self, address_n, creating):
+        self.creating_wallet = creating
+        return super(SafeTClientBase, self).get_public_node(address_n)
+
+    def close(self):
+        '''Called when Our wallet was closed or the device removed.'''
+        self.print_error("closing client")
+        self.clear_session()
+        # Release the device
+        self.transport.close()
+
+    def firmware_version(self):
+        f = self.features
+        return (f.major_version, f.minor_version, f.patch_version)
+
+    def atleast_version(self, major, minor=0, patch=0):
+        return self.firmware_version() >= (major, minor, patch)
+
+    @staticmethod
+    def wrapper(func):
+        '''Wrap methods to clear any message box they opened.'''
+
+        def wrapped(self, *args, **kwargs):
+            try:
+                self.prevent_timeouts()
+                return func(self, *args, **kwargs)
+            finally:
+                self.used()
+                self.handler.finished()
+                self.creating_wallet = False
+                self.msg = None
+
+        return wrapped
+
+    @staticmethod
+    def wrap_methods(cls):
+        for method in ['apply_settings', 'change_pin',
+                       'get_address', 'get_public_node',
+                       'load_device_by_mnemonic', 'load_device_by_xprv',
+                       'recovery_device', 'reset_device', 'sign_message',
+                       'sign_tx', 'wipe_device']:
+            setattr(cls, method, cls.wrapper(getattr(cls, method)))

electrum/plugins/safe_t/cmdline.py

@@ -0,0 +1,14 @@
+from electrum.plugin import hook
+from .safe_t import SafeTPlugin
+from ..hw_wallet import CmdLineHandler
+
+class Plugin(SafeTPlugin):
+    handler = CmdLineHandler()
+    @hook
+    def init_keystore(self, keystore):
+        if not isinstance(keystore, self.keystore_class):
+            return
+        keystore.handler = self.handler
+
+    def create_handler(self, window):
+        return self.handler

electrum/plugins/safe_t/qt.py

@@ -0,0 +1,492 @@
+from functools import partial
+import threading
+
+from PyQt5.Qt import Qt
+from PyQt5.Qt import QGridLayout, QInputDialog, QPushButton
+from PyQt5.Qt import QVBoxLayout, QLabel
+
+from electrum.gui.qt.util import *
+from electrum.i18n import _
+from electrum.plugin import hook, DeviceMgr
+from electrum.util import PrintError, UserCancelled, bh2u
+from electrum.wallet import Wallet, Standard_Wallet
+
+from ..hw_wallet.qt import QtHandlerBase, QtPluginBase
+from .safe_t import SafeTPlugin, TIM_NEW, TIM_RECOVER, TIM_MNEMONIC
+
+
+PASSPHRASE_HELP_SHORT =_(
+    "Passphrases allow you to access new wallets, each "
+    "hidden behind a particular case-sensitive passphrase.")
+PASSPHRASE_HELP = PASSPHRASE_HELP_SHORT + "  " + _(
+    "You need to create a separate Electrum wallet for each passphrase "
+    "you use as they each generate different addresses.  Changing "
+    "your passphrase does not lose other wallets, each is still "
+    "accessible behind its own passphrase.")
+RECOMMEND_PIN = _(
+    "You should enable PIN protection.  Your PIN is the only protection "
+    "for your bitcoins if your device is lost or stolen.")
+PASSPHRASE_NOT_PIN = _(
+    "If you forget a passphrase you will be unable to access any "
+    "bitcoins in the wallet behind it.  A passphrase is not a PIN. "
+    "Only change this if you are sure you understand it.")
+
+
+class QtHandler(QtHandlerBase):
+
+    pin_signal = pyqtSignal(object)
+
+    def __init__(self, win, pin_matrix_widget_class, device):
+        super(QtHandler, self).__init__(win, device)
+        self.pin_signal.connect(self.pin_dialog)
+        self.pin_matrix_widget_class = pin_matrix_widget_class
+
+    def get_pin(self, msg):
+        self.done.clear()
+        self.pin_signal.emit(msg)
+        self.done.wait()
+        return self.response
+
+    def pin_dialog(self, msg):
+        # Needed e.g. when resetting a device
+        self.clear_dialog()
+        dialog = WindowModalDialog(self.top_level_window(), _("Enter PIN"))
+        matrix = self.pin_matrix_widget_class()
+        vbox = QVBoxLayout()
+        vbox.addWidget(QLabel(msg))
+        vbox.addWidget(matrix)
+        vbox.addLayout(Buttons(CancelButton(dialog), OkButton(dialog)))
+        dialog.setLayout(vbox)
+        dialog.exec_()
+        self.response = str(matrix.get_value())
+        self.done.set()
+
+
+class QtPlugin(QtPluginBase):
+    # Derived classes must provide the following class-static variables:
+    #   icon_file
+    #   pin_matrix_widget_class
+
+    def create_handler(self, window):
+        return QtHandler(window, self.pin_matrix_widget_class(), self.device)
+
+    @hook
+    def receive_menu(self, menu, addrs, wallet):
+        if len(addrs) != 1:
+            return
+        for keystore in wallet.get_keystores():
+            if type(keystore) == self.keystore_class:
+                def show_address():
+                    keystore.thread.add(partial(self.show_address, wallet, addrs[0], keystore))
+                menu.addAction(_("Show on {}").format(self.device), show_address)
+                break
+
+    def show_settings_dialog(self, window, keystore):
+        device_id = self.choose_device(window, keystore)
+        if device_id:
+            SettingsDialog(window, self, keystore, device_id).exec_()
+
+    def request_safe_t_init_settings(self, wizard, method, device):
+        vbox = QVBoxLayout()
+        next_enabled = True
+        label = QLabel(_("Enter a label to name your device:"))
+        name = QLineEdit()
+        hl = QHBoxLayout()
+        hl.addWidget(label)
+        hl.addWidget(name)
+        hl.addStretch(1)
+        vbox.addLayout(hl)
+
+        def clean_text(widget):
+            text = widget.toPlainText().strip()
+            return ' '.join(text.split())
+
+        if method in [TIM_NEW, TIM_RECOVER]:
+            gb = QGroupBox()
+            hbox1 = QHBoxLayout()
+            gb.setLayout(hbox1)
+            vbox.addWidget(gb)
+            gb.setTitle(_("Select your seed length:"))
+            bg = QButtonGroup()
+            for i, count in enumerate([12, 18, 24]):
+                rb = QRadioButton(gb)
+                rb.setText(_("%d words") % count)
+                bg.addButton(rb)
+                bg.setId(rb, i)
+                hbox1.addWidget(rb)
+                rb.setChecked(True)
+            cb_pin = QCheckBox(_('Enable PIN protection'))
+            cb_pin.setChecked(True)
+        else:
+            text = QTextEdit()
+            text.setMaximumHeight(60)
+            if method == TIM_MNEMONIC:
+                msg = _("Enter your BIP39 mnemonic:")
+            else:
+                msg = _("Enter the master private key beginning with xprv:")
+                def set_enabled():
+                    from electrum.keystore import is_xprv
+                    wizard.next_button.setEnabled(is_xprv(clean_text(text)))
+                text.textChanged.connect(set_enabled)
+                next_enabled = False
+
+            vbox.addWidget(QLabel(msg))
+            vbox.addWidget(text)
+            pin = QLineEdit()
+            pin.setValidator(QRegExpValidator(QRegExp('[1-9]{0,9}')))
+            pin.setMaximumWidth(100)
+            hbox_pin = QHBoxLayout()
+            hbox_pin.addWidget(QLabel(_("Enter your PIN (digits 1-9):")))
+            hbox_pin.addWidget(pin)
+            hbox_pin.addStretch(1)
+
+        if method in [TIM_NEW, TIM_RECOVER]:
+            vbox.addWidget(WWLabel(RECOMMEND_PIN))
+            vbox.addWidget(cb_pin)
+        else:
+            vbox.addLayout(hbox_pin)
+
+        passphrase_msg = WWLabel(PASSPHRASE_HELP_SHORT)
+        passphrase_warning = WWLabel(PASSPHRASE_NOT_PIN)
+        passphrase_warning.setStyleSheet("color: red")
+        cb_phrase = QCheckBox(_('Enable passphrases'))
+        cb_phrase.setChecked(False)
+        vbox.addWidget(passphrase_msg)
+        vbox.addWidget(passphrase_warning)
+        vbox.addWidget(cb_phrase)
+
+        wizard.exec_layout(vbox, next_enabled=next_enabled)
+
+        if method in [TIM_NEW, TIM_RECOVER]:
+            item = bg.checkedId()
+            pin = cb_pin.isChecked()
+        else:
+            item = ' '.join(str(clean_text(text)).split())
+            pin = str(pin.text())
+
+        return (item, name.text(), pin, cb_phrase.isChecked())
+
+
+class Plugin(SafeTPlugin, QtPlugin):
+    icon_unpaired = ":icons/safe-t_unpaired.png"
+    icon_paired = ":icons/safe-t.png"
+
+    @classmethod
+    def pin_matrix_widget_class(self):
+        from safetlib.qt.pinmatrix import PinMatrixWidget
+        return PinMatrixWidget
+
+
+class SettingsDialog(WindowModalDialog):
+    '''This dialog doesn't require a device be paired with a wallet.
+    We want users to be able to wipe a device even if they've forgotten
+    their PIN.'''
+
+    def __init__(self, window, plugin, keystore, device_id):
+        title = _("{} Settings").format(plugin.device)
+        super(SettingsDialog, self).__init__(window, title)
+        self.setMaximumWidth(540)
+
+        devmgr = plugin.device_manager()
+        config = devmgr.config
+        handler = keystore.handler
+        thread = keystore.thread
+        hs_rows, hs_cols = (64, 128)
+
+        def invoke_client(method, *args, **kw_args):
+            unpair_after = kw_args.pop('unpair_after', False)
+
+            def task():
+                client = devmgr.client_by_id(device_id)
+                if not client:
+                    raise RuntimeError("Device not connected")
+                if method:
+                    getattr(client, method)(*args, **kw_args)
+                if unpair_after:
+                    devmgr.unpair_id(device_id)
+                return client.features
+
+            thread.add(task, on_success=update)
+
+        def update(features):
+            self.features = features
+            set_label_enabled()
+            if features.bootloader_hash:
+                bl_hash = bh2u(features.bootloader_hash)
+                bl_hash = "\n".join([bl_hash[:32], bl_hash[32:]])
+            else:
+                bl_hash = "N/A"
+            noyes = [_("No"), _("Yes")]
+            endis = [_("Enable Passphrases"), _("Disable Passphrases")]
+            disen = [_("Disabled"), _("Enabled")]
+            setchange = [_("Set a PIN"), _("Change PIN")]
+
+            version = "%d.%d.%d" % (features.major_version,
+                                    features.minor_version,
+                                    features.patch_version)
+
+            device_label.setText(features.label)
+            pin_set_label.setText(noyes[features.pin_protection])
+            passphrases_label.setText(disen[features.passphrase_protection])
+            bl_hash_label.setText(bl_hash)
+            label_edit.setText(features.label)
+            device_id_label.setText(features.device_id)
+            initialized_label.setText(noyes[features.initialized])
+            version_label.setText(version)
+            clear_pin_button.setVisible(features.pin_protection)
+            clear_pin_warning.setVisible(features.pin_protection)
+            pin_button.setText(setchange[features.pin_protection])
+            pin_msg.setVisible(not features.pin_protection)
+            passphrase_button.setText(endis[features.passphrase_protection])
+            language_label.setText(features.language)
+
+        def set_label_enabled():
+            label_apply.setEnabled(label_edit.text() != self.features.label)
+
+        def rename():
+            invoke_client('change_label', label_edit.text())
+
+        def toggle_passphrase():
+            title = _("Confirm Toggle Passphrase Protection")
+            currently_enabled = self.features.passphrase_protection
+            if currently_enabled:
+                msg = _("After disabling passphrases, you can only pair this "
+                        "Electrum wallet if it had an empty passphrase.  "
+                        "If its passphrase was not empty, you will need to "
+                        "create a new wallet with the install wizard.  You "
+                        "can use this wallet again at any time by re-enabling "
+                        "passphrases and entering its passphrase.")
+            else:
+                msg = _("Your current Electrum wallet can only be used with "
+                        "an empty passphrase.  You must create a separate "
+                        "wallet with the install wizard for other passphrases "
+                        "as each one generates a new set of addresses.")
+            msg += "\n\n" + _("Are you sure you want to proceed?")
+            if not self.question(msg, title=title):
+                return
+            invoke_client('toggle_passphrase', unpair_after=currently_enabled)
+
+        def change_homescreen():
+            dialog = QFileDialog(self, _("Choose Homescreen"))
+            filename, __ = dialog.getOpenFileName()
+            if not filename:
+                return  # user cancelled
+
+            if filename.endswith('.toif'):
+                img = open(filename, 'rb').read()
+                if img[:8] != b'TOIf\x90\x00\x90\x00':
+                    handler.show_error('File is not a TOIF file with size of 144x144')
+                    return
+            else:
+                from PIL import Image # FIXME
+                im = Image.open(filename)
+                if im.size != (128, 64):
+                    handler.show_error('Image must be 128 x 64 pixels')
+                    return
+                im = im.convert('1')
+                pix = im.load()
+                img = bytearray(1024)
+                for j in range(64):
+                    for i in range(128):
+                        if pix[i, j]:
+                            o = (i + j * 128)
+                            img[o // 8] |= (1 << (7 - o % 8))
+                img = bytes(img)
+            invoke_client('change_homescreen', img)
+
+        def clear_homescreen():
+            invoke_client('change_homescreen', b'\x00')
+
+        def set_pin():
+            invoke_client('set_pin', remove=False)
+
+        def clear_pin():
+            invoke_client('set_pin', remove=True)
+
+        def wipe_device():
+            wallet = window.wallet
+            if wallet and sum(wallet.get_balance()):
+                title = _("Confirm Device Wipe")
+                msg = _("Are you SURE you want to wipe the device?\n"
+                        "Your wallet still has bitcoins in it!")
+                if not self.question(msg, title=title,
+                                     icon=QMessageBox.Critical):
+                    return
+            invoke_client('wipe_device', unpair_after=True)
+
+        def slider_moved():
+            mins = timeout_slider.sliderPosition()
+            timeout_minutes.setText(_("%2d minutes") % mins)
+
+        def slider_released():
+            config.set_session_timeout(timeout_slider.sliderPosition() * 60)
+
+        # Information tab
+        info_tab = QWidget()
+        info_layout = QVBoxLayout(info_tab)
+        info_glayout = QGridLayout()
+        info_glayout.setColumnStretch(2, 1)
+        device_label = QLabel()
+        pin_set_label = QLabel()
+        passphrases_label = QLabel()
+        version_label = QLabel()
+        device_id_label = QLabel()
+        bl_hash_label = QLabel()
+        bl_hash_label.setWordWrap(True)
+        language_label = QLabel()
+        initialized_label = QLabel()
+        rows = [
+            (_("Device Label"), device_label),
+            (_("PIN set"), pin_set_label),
+            (_("Passphrases"), passphrases_label),
+            (_("Firmware Version"), version_label),
+            (_("Device ID"), device_id_label),
+            (_("Bootloader Hash"), bl_hash_label),
+            (_("Language"), language_label),
+            (_("Initialized"), initialized_label),
+        ]
+        for row_num, (label, widget) in enumerate(rows):
+            info_glayout.addWidget(QLabel(label), row_num, 0)
+            info_glayout.addWidget(widget, row_num, 1)
+        info_layout.addLayout(info_glayout)
+
+        # Settings tab
+        settings_tab = QWidget()
+        settings_layout = QVBoxLayout(settings_tab)
+        settings_glayout = QGridLayout()
+
+        # Settings tab - Label
+        label_msg = QLabel(_("Name this {}.  If you have multiple devices "
+                             "their labels help distinguish them.")
+                           .format(plugin.device))
+        label_msg.setWordWrap(True)
+        label_label = QLabel(_("Device Label"))
+        label_edit = QLineEdit()
+        label_edit.setMinimumWidth(150)
+        label_edit.setMaxLength(plugin.MAX_LABEL_LEN)
+        label_apply = QPushButton(_("Apply"))
+        label_apply.clicked.connect(rename)
+        label_edit.textChanged.connect(set_label_enabled)
+        settings_glayout.addWidget(label_label, 0, 0)
+        settings_glayout.addWidget(label_edit, 0, 1, 1, 2)
+        settings_glayout.addWidget(label_apply, 0, 3)
+        settings_glayout.addWidget(label_msg, 1, 1, 1, -1)
+
+        # Settings tab - PIN
+        pin_label = QLabel(_("PIN Protection"))
+        pin_button = QPushButton()
+        pin_button.clicked.connect(set_pin)
+        settings_glayout.addWidget(pin_label, 2, 0)
+        settings_glayout.addWidget(pin_button, 2, 1)
+        pin_msg = QLabel(_("PIN protection is strongly recommended.  "
+                           "A PIN is your only protection against someone "
+                           "stealing your bitcoins if they obtain physical "
+                           "access to your {}.").format(plugin.device))
+        pin_msg.setWordWrap(True)
+        pin_msg.setStyleSheet("color: red")
+        settings_glayout.addWidget(pin_msg, 3, 1, 1, -1)
+
+        # Settings tab - Homescreen
+        homescreen_label = QLabel(_("Homescreen"))
+        homescreen_change_button = QPushButton(_("Change..."))
+        homescreen_clear_button = QPushButton(_("Reset"))
+        homescreen_change_button.clicked.connect(change_homescreen)
+        try:
+            import PIL
+        except ImportError:
+            homescreen_change_button.setDisabled(True)
+            homescreen_change_button.setToolTip(
+                _("Required package 'PIL' is not available - Please install it.")
+            )
+        homescreen_clear_button.clicked.connect(clear_homescreen)
+        homescreen_msg = QLabel(_("You can set the homescreen on your "
+                                  "device to personalize it.  You must "
+                                  "choose a {} x {} monochrome black and "
+                                  "white image.").format(hs_rows, hs_cols))
+        homescreen_msg.setWordWrap(True)
+        settings_glayout.addWidget(homescreen_label, 4, 0)
+        settings_glayout.addWidget(homescreen_change_button, 4, 1)
+        settings_glayout.addWidget(homescreen_clear_button, 4, 2)
+        settings_glayout.addWidget(homescreen_msg, 5, 1, 1, -1)
+
+        # Settings tab - Session Timeout
+        timeout_label = QLabel(_("Session Timeout"))
+        timeout_minutes = QLabel()
+        timeout_slider = QSlider(Qt.Horizontal)
+        timeout_slider.setRange(1, 60)
+        timeout_slider.setSingleStep(1)
+        timeout_slider.setTickInterval(5)
+        timeout_slider.setTickPosition(QSlider.TicksBelow)
+        timeout_slider.setTracking(True)
+        timeout_msg = QLabel(
+            _("Clear the session after the specified period "
+              "of inactivity.  Once a session has timed out, "
+              "your PIN and passphrase (if enabled) must be "
+              "re-entered to use the device."))
+        timeout_msg.setWordWrap(True)
+        timeout_slider.setSliderPosition(config.get_session_timeout() // 60)
+        slider_moved()
+        timeout_slider.valueChanged.connect(slider_moved)
+        timeout_slider.sliderReleased.connect(slider_released)
+        settings_glayout.addWidget(timeout_label, 6, 0)
+        settings_glayout.addWidget(timeout_slider, 6, 1, 1, 3)
+        settings_glayout.addWidget(timeout_minutes, 6, 4)
+        settings_glayout.addWidget(timeout_msg, 7, 1, 1, -1)
+        settings_layout.addLayout(settings_glayout)
+        settings_layout.addStretch(1)
+
+        # Advanced tab
+        advanced_tab = QWidget()
+        advanced_layout = QVBoxLayout(advanced_tab)
+        advanced_glayout = QGridLayout()
+
+        # Advanced tab - clear PIN
+        clear_pin_button = QPushButton(_("Disable PIN"))
+        clear_pin_button.clicked.connect(clear_pin)
+        clear_pin_warning = QLabel(
+            _("If you disable your PIN, anyone with physical access to your "
+              "{} device can spend your bitcoins.").format(plugin.device))
+        clear_pin_warning.setWordWrap(True)
+        clear_pin_warning.setStyleSheet("color: red")
+        advanced_glayout.addWidget(clear_pin_button, 0, 2)
+        advanced_glayout.addWidget(clear_pin_warning, 1, 0, 1, 5)
+
+        # Advanced tab - toggle passphrase protection
+        passphrase_button = QPushButton()
+        passphrase_button.clicked.connect(toggle_passphrase)
+        passphrase_msg = WWLabel(PASSPHRASE_HELP)
+        passphrase_warning = WWLabel(PASSPHRASE_NOT_PIN)
+        passphrase_warning.setStyleSheet("color: red")
+        advanced_glayout.addWidget(passphrase_button, 3, 2)
+        advanced_glayout.addWidget(passphrase_msg, 4, 0, 1, 5)
+        advanced_glayout.addWidget(passphrase_warning, 5, 0, 1, 5)
+
+        # Advanced tab - wipe device
+        wipe_device_button = QPushButton(_("Wipe Device"))
+        wipe_device_button.clicked.connect(wipe_device)
+        wipe_device_msg = QLabel(
+            _("Wipe the device, removing all data from it.  The firmware "
+              "is left unchanged."))
+        wipe_device_msg.setWordWrap(True)
+        wipe_device_warning = QLabel(
+            _("Only wipe a device if you have the recovery seed written down "
+              "and the device wallet(s) are empty, otherwise the bitcoins "
+              "will be lost forever."))
+        wipe_device_warning.setWordWrap(True)
+        wipe_device_warning.setStyleSheet("color: red")
+        advanced_glayout.addWidget(wipe_device_button, 6, 2)
+        advanced_glayout.addWidget(wipe_device_msg, 7, 0, 1, 5)
+        advanced_glayout.addWidget(wipe_device_warning, 8, 0, 1, 5)
+        advanced_layout.addLayout(advanced_glayout)
+        advanced_layout.addStretch(1)
+
+        tabs = QTabWidget(self)
+        tabs.addTab(info_tab, _("Information"))
+        tabs.addTab(settings_tab, _("Settings"))
+        tabs.addTab(advanced_tab, _("Advanced"))
+        dialog_vbox = QVBoxLayout(self)
+        dialog_vbox.addWidget(tabs)
+        dialog_vbox.addLayout(Buttons(CloseButton(self)))
+
+        # Update information
+        invoke_client(None)

electrum/plugins/safe_t/safe_t.py

@@ -0,0 +1,509 @@
+from binascii import hexlify, unhexlify
+import traceback
+import sys
+
+from electrum.util import bfh, bh2u, versiontuple, UserCancelled
+from electrum.bitcoin import (b58_address_to_hash160, xpub_from_pubkey, deserialize_xpub,
+                              TYPE_ADDRESS, TYPE_SCRIPT, is_address)
+from electrum import constants
+from electrum.i18n import _
+from electrum.plugin import BasePlugin, Device
+from electrum.transaction import deserialize, Transaction
+from electrum.keystore import Hardware_KeyStore, is_xpubkey, parse_xpubkey, xtype_from_derivation
+from electrum.base_wizard import ScriptTypeNotSupported
+
+from ..hw_wallet import HW_PluginBase
+from ..hw_wallet.plugin import is_any_tx_output_on_change_branch, trezor_validate_op_return_output_and_get_data
+
+
+# Safe-T mini initialization methods
+TIM_NEW, TIM_RECOVER, TIM_MNEMONIC, TIM_PRIVKEY = range(0, 4)
+
+# script "generation"
+SCRIPT_GEN_LEGACY, SCRIPT_GEN_P2SH_SEGWIT, SCRIPT_GEN_NATIVE_SEGWIT = range(0, 3)
+
+
+class SafeTKeyStore(Hardware_KeyStore):
+    hw_type = 'safe_t'
+    device = 'Safe-T mini'
+
+    def get_derivation(self):
+        return self.derivation
+
+    def get_script_gen(self):
+        xtype = xtype_from_derivation(self.derivation)
+        if xtype in ('p2wpkh', 'p2wsh'):
+            return SCRIPT_GEN_NATIVE_SEGWIT
+        elif xtype in ('p2wpkh-p2sh', 'p2wsh-p2sh'):
+            return SCRIPT_GEN_P2SH_SEGWIT
+        else:
+            return SCRIPT_GEN_LEGACY
+
+    def get_client(self, force_pair=True):
+        return self.plugin.get_client(self, force_pair)
+
+    def decrypt_message(self, sequence, message, password):
+        raise RuntimeError(_('Encryption and decryption are not implemented by {}').format(self.device))
+
+    def sign_message(self, sequence, message, password):
+        client = self.get_client()
+        address_path = self.get_derivation() + "/%d/%d"%sequence
+        address_n = client.expand_path(address_path)
+        msg_sig = client.sign_message(self.plugin.get_coin_name(), address_n, message)
+        return msg_sig.signature
+
+    def sign_transaction(self, tx, password):
+        if tx.is_complete():
+            return
+        # previous transactions used as inputs
+        prev_tx = {}
+        # path of the xpubs that are involved
+        xpub_path = {}
+        for txin in tx.inputs():
+            pubkeys, x_pubkeys = tx.get_sorted_pubkeys(txin)
+            tx_hash = txin['prevout_hash']
+            if txin.get('prev_tx') is None and not Transaction.is_segwit_input(txin):
+                raise Exception(_('Offline signing with {} is not supported for legacy inputs.').format(self.device))
+            prev_tx[tx_hash] = txin['prev_tx']
+            for x_pubkey in x_pubkeys:
+                if not is_xpubkey(x_pubkey):
+                    continue
+                xpub, s = parse_xpubkey(x_pubkey)
+                if xpub == self.get_master_public_key():
+                    xpub_path[xpub] = self.get_derivation()
+
+        self.plugin.sign_transaction(self, tx, prev_tx, xpub_path)
+
+
+class SafeTPlugin(HW_PluginBase):
+    # Derived classes provide:
+    #
+    #  class-static variables: client_class, firmware_URL, handler_class,
+    #     libraries_available, libraries_URL, minimum_firmware,
+    #     wallet_class, types
+
+    firmware_URL = 'https://safe-t.io'
+    libraries_URL = 'https://github.com/archos-safe-t/python-safet'
+    minimum_firmware = (1, 0, 5)
+    keystore_class = SafeTKeyStore
+    minimum_library = (0, 1, 0)
+    SUPPORTED_XTYPES = ('standard', 'p2wpkh-p2sh', 'p2wpkh', 'p2wsh-p2sh', 'p2wsh')
+
+    MAX_LABEL_LEN = 32
+
+    def __init__(self, parent, config, name):
+        HW_PluginBase.__init__(self, parent, config, name)
+
+        try:
+            # Minimal test if python-safet is installed
+            import safetlib
+            try:
+                library_version = safetlib.__version__
+            except AttributeError:
+                # python-safet only introduced __version__ in 0.1.0
+                library_version = 'unknown'
+            if library_version == 'unknown' or \
+                    versiontuple(library_version) < self.minimum_library:
+                self.libraries_available_message = (
+                        _("Library version for '{}' is too old.").format(name)
+                        + '\nInstalled: {}, Needed: {}'
+                        .format(library_version, self.minimum_library))
+                self.print_stderr(self.libraries_available_message)
+                raise ImportError()
+            self.libraries_available = True
+        except ImportError:
+            self.libraries_available = False
+            return
+
+        from . import client
+        from . import transport
+        import safetlib.messages
+        self.client_class = client.SafeTClient
+        self.types = safetlib.messages
+        self.DEVICE_IDS = ('Safe-T mini',)
+
+        self.transport_handler = transport.SafeTTransport()
+        self.device_manager().register_enumerate_func(self.enumerate)
+
+    def enumerate(self):
+        devices = self.transport_handler.enumerate_devices()
+        return [Device(d.get_path(), -1, d.get_path(), 'Safe-T mini', 0) for d in devices]
+
+    def create_client(self, device, handler):
+        try:
+            self.print_error("connecting to device at", device.path)
+            transport = self.transport_handler.get_transport(device.path)
+        except BaseException as e:
+            self.print_error("cannot connect at", device.path, str(e))
+            return None
+
+        if not transport:
+            self.print_error("cannot connect at", device.path)
+            return
+
+        self.print_error("connected to device at", device.path)
+        client = self.client_class(transport, handler, self)
+
+        # Try a ping for device sanity
+        try:
+            client.ping('t')
+        except BaseException as e:
+            self.print_error("ping failed", str(e))
+            return None
+
+        if not client.atleast_version(*self.minimum_firmware):
+            msg = (_('Outdated {} firmware for device labelled {}. Please '
+                     'download the updated firmware from {}')
+                   .format(self.device, client.label(), self.firmware_URL))
+            self.print_error(msg)
+            if handler:
+                handler.show_error(msg)
+            else:
+                raise Exception(msg)
+            return None
+
+        return client
+
+    def get_client(self, keystore, force_pair=True):
+        devmgr = self.device_manager()
+        handler = keystore.handler
+        with devmgr.hid_lock:
+            client = devmgr.client_for_keystore(self, handler, keystore, force_pair)
+        # returns the client for a given keystore. can use xpub
+        if client:
+            client.used()
+        return client
+
+    def get_coin_name(self):
+        return "Testnet" if constants.net.TESTNET else "Bitcoin"
+
+    def initialize_device(self, device_id, wizard, handler):
+        # Initialization method
+        msg = _("Choose how you want to initialize your {}.\n\n"
+                "The first two methods are secure as no secret information "
+                "is entered into your computer.\n\n"
+                "For the last two methods you input secrets on your keyboard "
+                "and upload them to your {}, and so you should "
+                "only do those on a computer you know to be trustworthy "
+                "and free of malware."
+        ).format(self.device, self.device)
+        choices = [
+            # Must be short as QT doesn't word-wrap radio button text
+            (TIM_NEW, _("Let the device generate a completely new seed randomly")),
+            (TIM_RECOVER, _("Recover from a seed you have previously written down")),
+            (TIM_MNEMONIC, _("Upload a BIP39 mnemonic to generate the seed")),
+            (TIM_PRIVKEY, _("Upload a master private key"))
+        ]
+        def f(method):
+            import threading
+            settings = self.request_safe_t_init_settings(wizard, method, self.device)
+            t = threading.Thread(target=self._initialize_device_safe, args=(settings, method, device_id, wizard, handler))
+            t.setDaemon(True)
+            t.start()
+            exit_code = wizard.loop.exec_()
+            if exit_code != 0:
+                # this method (initialize_device) was called with the expectation
+                # of leaving the device in an initialized state when finishing.
+                # signal that this is not the case:
+                raise UserCancelled()
+        wizard.choice_dialog(title=_('Initialize Device'), message=msg, choices=choices, run_next=f)
+
+    def _initialize_device_safe(self, settings, method, device_id, wizard, handler):
+        exit_code = 0
+        try:
+            self._initialize_device(settings, method, device_id, wizard, handler)
+        except UserCancelled:
+            exit_code = 1
+        except BaseException as e:
+            traceback.print_exc(file=sys.stderr)
+            handler.show_error(str(e))
+            exit_code = 1
+        finally:
+            wizard.loop.exit(exit_code)
+
+    def _initialize_device(self, settings, method, device_id, wizard, handler):
+        item, label, pin_protection, passphrase_protection = settings
+
+        if method == TIM_RECOVER:
+            handler.show_error(_(
+                "You will be asked to enter 24 words regardless of your "
+                "seed's actual length.  If you enter a word incorrectly or "
+                "misspell it, you cannot change it or go back - you will need "
+                "to start again from the beginning.\n\nSo please enter "
+                "the words carefully!"),
+                blocking=True)
+
+        language = 'english'
+        devmgr = self.device_manager()
+        client = devmgr.client_by_id(device_id)
+
+        if method == TIM_NEW:
+            strength = 64 * (item + 2)  # 128, 192 or 256
+            u2f_counter = 0
+            skip_backup = False
+            client.reset_device(True, strength, passphrase_protection,
+                                pin_protection, label, language,
+                                u2f_counter, skip_backup)
+        elif method == TIM_RECOVER:
+            word_count = 6 * (item + 2)  # 12, 18 or 24
+            client.step = 0
+            client.recovery_device(word_count, passphrase_protection,
+                                       pin_protection, label, language)
+        elif method == TIM_MNEMONIC:
+            pin = pin_protection  # It's the pin, not a boolean
+            client.load_device_by_mnemonic(str(item), pin,
+                                           passphrase_protection,
+                                           label, language)
+        else:
+            pin = pin_protection  # It's the pin, not a boolean
+            client.load_device_by_xprv(item, pin, passphrase_protection,
+                                       label, language)
+
+    def _make_node_path(self, xpub, address_n):
+        _, depth, fingerprint, child_num, chain_code, key = deserialize_xpub(xpub)
+        node = self.types.HDNodeType(
+            depth=depth,
+            fingerprint=int.from_bytes(fingerprint, 'big'),
+            child_num=int.from_bytes(child_num, 'big'),
+            chain_code=chain_code,
+            public_key=key,
+        )
+        return self.types.HDNodePathType(node=node, address_n=address_n)
+
+    def setup_device(self, device_info, wizard, purpose):
+        devmgr = self.device_manager()
+        device_id = device_info.device.id_
+        client = devmgr.client_by_id(device_id)
+        if client is None:
+            raise Exception(_('Failed to create a client for this device.') + '\n' +
+                            _('Make sure it is in the correct state.'))
+        # fixme: we should use: client.handler = wizard
+        client.handler = self.create_handler(wizard)
+        if not device_info.initialized:
+            self.initialize_device(device_id, wizard, client.handler)
+        client.get_xpub('m', 'standard')
+        client.used()
+
+    def get_xpub(self, device_id, derivation, xtype, wizard):
+        if xtype not in self.SUPPORTED_XTYPES:
+            raise ScriptTypeNotSupported(_('This type of script is not supported with {}.').format(self.device))
+        devmgr = self.device_manager()
+        client = devmgr.client_by_id(device_id)
+        client.handler = wizard
+        xpub = client.get_xpub(derivation, xtype)
+        client.used()
+        return xpub
+
+    def get_safet_input_script_type(self, script_gen, is_multisig):
+        if script_gen == SCRIPT_GEN_NATIVE_SEGWIT:
+            return self.types.InputScriptType.SPENDWITNESS
+        elif script_gen == SCRIPT_GEN_P2SH_SEGWIT:
+            return self.types.InputScriptType.SPENDP2SHWITNESS
+        else:
+            if is_multisig:
+                return self.types.InputScriptType.SPENDMULTISIG
+            else:
+                return self.types.InputScriptType.SPENDADDRESS
+
+    def sign_transaction(self, keystore, tx, prev_tx, xpub_path):
+        self.prev_tx = prev_tx
+        self.xpub_path = xpub_path
+        client = self.get_client(keystore)
+        inputs = self.tx_inputs(tx, True, keystore.get_script_gen())
+        outputs = self.tx_outputs(keystore.get_derivation(), tx, keystore.get_script_gen())
+        signatures = client.sign_tx(self.get_coin_name(), inputs, outputs, lock_time=tx.locktime)[0]
+        signatures = [(bh2u(x) + '01') for x in signatures]
+        tx.update_signatures(signatures)
+
+    def show_address(self, wallet, address, keystore=None):
+        if keystore is None:
+            keystore = wallet.get_keystore()
+        if not self.show_address_helper(wallet, address, keystore):
+            return
+        client = self.get_client(keystore)
+        if not client.atleast_version(1, 0):
+            keystore.handler.show_error(_("Your device firmware is too old"))
+            return
+        change, index = wallet.get_address_index(address)
+        derivation = keystore.derivation
+        address_path = "%s/%d/%d"%(derivation, change, index)
+        address_n = client.expand_path(address_path)
+        xpubs = wallet.get_master_public_keys()
+        if len(xpubs) == 1:
+            script_gen = keystore.get_script_gen()
+            script_type = self.get_safet_input_script_type(script_gen, is_multisig=False)
+            client.get_address(self.get_coin_name(), address_n, True, script_type=script_type)
+        else:
+            def f(xpub):
+                return self._make_node_path(xpub, [change, index])
+            pubkeys = wallet.get_public_keys(address)
+            # sort xpubs using the order of pubkeys
+            sorted_pubkeys, sorted_xpubs = zip(*sorted(zip(pubkeys, xpubs)))
+            pubkeys = list(map(f, sorted_xpubs))
+            multisig = self.types.MultisigRedeemScriptType(
+               pubkeys=pubkeys,
+               signatures=[b''] * wallet.n,
+               m=wallet.m,
+            )
+            script_gen = keystore.get_script_gen()
+            script_type = self.get_safet_input_script_type(script_gen, is_multisig=True)
+            client.get_address(self.get_coin_name(), address_n, True, multisig=multisig, script_type=script_type)
+
+    def tx_inputs(self, tx, for_sig=False, script_gen=SCRIPT_GEN_LEGACY):
+        inputs = []
+        for txin in tx.inputs():
+            txinputtype = self.types.TxInputType()
+            if txin['type'] == 'coinbase':
+                prev_hash = b"\x00"*32
+                prev_index = 0xffffffff  # signed int -1
+            else:
+                if for_sig:
+                    x_pubkeys = txin['x_pubkeys']
+                    if len(x_pubkeys) == 1:
+                        x_pubkey = x_pubkeys[0]
+                        xpub, s = parse_xpubkey(x_pubkey)
+                        xpub_n = self.client_class.expand_path(self.xpub_path[xpub])
+                        txinputtype._extend_address_n(xpub_n + s)
+                        txinputtype.script_type = self.get_safet_input_script_type(script_gen, is_multisig=False)
+                    else:
+                        def f(x_pubkey):
+                            if is_xpubkey(x_pubkey):
+                                xpub, s = parse_xpubkey(x_pubkey)
+                            else:
+                                xpub = xpub_from_pubkey(0, bfh(x_pubkey))
+                                s = []
+                            return self._make_node_path(xpub, s)
+                        pubkeys = list(map(f, x_pubkeys))
+                        multisig = self.types.MultisigRedeemScriptType(
+                            pubkeys=pubkeys,
+                            signatures=list(map(lambda x: bfh(x)[:-1] if x else b'', txin.get('signatures'))),
+                            m=txin.get('num_sig'),
+                        )
+                        script_type = self.get_safet_input_script_type(script_gen, is_multisig=True)
+                        txinputtype = self.types.TxInputType(
+                            script_type=script_type,
+                            multisig=multisig
+                        )
+                        # find which key is mine
+                        for x_pubkey in x_pubkeys:
+                            if is_xpubkey(x_pubkey):
+                                xpub, s = parse_xpubkey(x_pubkey)
+                                if xpub in self.xpub_path:
+                                    xpub_n = self.client_class.expand_path(self.xpub_path[xpub])
+                                    txinputtype._extend_address_n(xpub_n + s)
+                                    break
+
+                prev_hash = unhexlify(txin['prevout_hash'])
+                prev_index = txin['prevout_n']
+
+            if 'value' in txin:
+                txinputtype.amount = txin['value']
+            txinputtype.prev_hash = prev_hash
+            txinputtype.prev_index = prev_index
+
+            if txin.get('scriptSig') is not None:
+                script_sig = bfh(txin['scriptSig'])
+                txinputtype.script_sig = script_sig
+
+            txinputtype.sequence = txin.get('sequence', 0xffffffff - 1)
+
+            inputs.append(txinputtype)
+
+        return inputs
+
+    def tx_outputs(self, derivation, tx, script_gen=SCRIPT_GEN_LEGACY):
+
+        def create_output_by_derivation(info):
+            index, xpubs, m = info
+            if len(xpubs) == 1:
+                if script_gen == SCRIPT_GEN_NATIVE_SEGWIT:
+                    script_type = self.types.OutputScriptType.PAYTOWITNESS
+                elif script_gen == SCRIPT_GEN_P2SH_SEGWIT:
+                    script_type = self.types.OutputScriptType.PAYTOP2SHWITNESS
+                else:
+                    script_type = self.types.OutputScriptType.PAYTOADDRESS
+                address_n = self.client_class.expand_path(derivation + "/%d/%d" % index)
+                txoutputtype = self.types.TxOutputType(
+                    amount=amount,
+                    script_type=script_type,
+                    address_n=address_n,
+                )
+            else:
+                if script_gen == SCRIPT_GEN_NATIVE_SEGWIT:
+                    script_type = self.types.OutputScriptType.PAYTOWITNESS
+                elif script_gen == SCRIPT_GEN_P2SH_SEGWIT:
+                    script_type = self.types.OutputScriptType.PAYTOP2SHWITNESS
+                else:
+                    script_type = self.types.OutputScriptType.PAYTOMULTISIG
+                address_n = self.client_class.expand_path("/%d/%d" % index)
+                pubkeys = [self._make_node_path(xpub, address_n) for xpub in xpubs]
+                multisig = self.types.MultisigRedeemScriptType(
+                    pubkeys=pubkeys,
+                    signatures=[b''] * len(pubkeys),
+                    m=m)
+                txoutputtype = self.types.TxOutputType(
+                    multisig=multisig,
+                    amount=amount,
+                    address_n=self.client_class.expand_path(derivation + "/%d/%d" % index),
+                    script_type=script_type)
+            return txoutputtype
+
+        def create_output_by_address():
+            txoutputtype = self.types.TxOutputType()
+            txoutputtype.amount = amount
+            if _type == TYPE_SCRIPT:
+                txoutputtype.script_type = self.types.OutputScriptType.PAYTOOPRETURN
+                txoutputtype.op_return_data = trezor_validate_op_return_output_and_get_data(o)
+            elif _type == TYPE_ADDRESS:
+                txoutputtype.script_type = self.types.OutputScriptType.PAYTOADDRESS
+                txoutputtype.address = address
+            return txoutputtype
+
+        outputs = []
+        has_change = False
+        any_output_on_change_branch = is_any_tx_output_on_change_branch(tx)
+
+        for o in tx.outputs():
+            _type, address, amount = o.type, o.address, o.value
+            use_create_by_derivation = False
+
+            info = tx.output_info.get(address)
+            if info is not None and not has_change:
+                index, xpubs, m = info
+                on_change_branch = index[0] == 1
+                # prioritise hiding outputs on the 'change' branch from user
+                # because no more than one change address allowed
+                # note: ^ restriction can be removed once we require fw
+                # that has https://github.com/trezor/trezor-mcu/pull/306
+                if on_change_branch == any_output_on_change_branch:
+                    use_create_by_derivation = True
+                    has_change = True
+
+            if use_create_by_derivation:
+                txoutputtype = create_output_by_derivation(info)
+            else:
+                txoutputtype = create_output_by_address()
+            outputs.append(txoutputtype)
+
+        return outputs
+
+    def electrum_tx_to_txtype(self, tx):
+        t = self.types.TransactionType()
+        if tx is None:
+            # probably for segwit input and we don't need this prev txn
+            return t
+        d = deserialize(tx.raw)
+        t.version = d['version']
+        t.lock_time = d['lockTime']
+        inputs = self.tx_inputs(tx)
+        t._extend_inputs(inputs)
+        for vout in d['outputs']:
+            o = t._add_bin_outputs()
+            o.amount = vout['value']
+            o.script_pubkey = bfh(vout['scriptPubKey'])
+        return t
+
+    # This function is called from the TREZOR libraries (via tx_api)
+    def get_tx(self, tx_hash):
+        tx = self.prev_tx[tx_hash]
+        return self.electrum_tx_to_txtype(tx)

electrum/plugins/safe_t/transport.py

@@ -0,0 +1,95 @@
+from electrum.util import PrintError
+
+
+class SafeTTransport(PrintError):
+
+    @staticmethod
+    def all_transports():
+        """Reimplemented safetlib.transport.all_transports so that we can
+        enable/disable specific transports.
+        """
+        try:
+            # only to detect safetlib version
+            from safetlib.transport import all_transports
+        except ImportError:
+            # old safetlib. compat for safetlib < 0.9.2
+            transports = []
+            #try:
+            #    from safetlib.transport_bridge import BridgeTransport
+            #    transports.append(BridgeTransport)
+            #except BaseException:
+            #    pass
+            try:
+                from safetlib.transport_hid import HidTransport
+                transports.append(HidTransport)
+            except BaseException:
+                pass
+            try:
+                from safetlib.transport_udp import UdpTransport
+                transports.append(UdpTransport)
+            except BaseException:
+                pass
+            try:
+                from safetlib.transport_webusb import WebUsbTransport
+                transports.append(WebUsbTransport)
+            except BaseException:
+                pass
+        else:
+            # new safetlib.
+            transports = []
+            #try:
+            #    from safetlib.transport.bridge import BridgeTransport
+            #    transports.append(BridgeTransport)
+            #except BaseException:
+            #    pass
+            try:
+                from safetlib.transport.hid import HidTransport
+                transports.append(HidTransport)
+            except BaseException:
+                pass
+            try:
+                from safetlib.transport.udp import UdpTransport
+                transports.append(UdpTransport)
+            except BaseException:
+                pass
+            try:
+                from safetlib.transport.webusb import WebUsbTransport
+                transports.append(WebUsbTransport)
+            except BaseException:
+                pass
+            return transports
+        return transports
+
+    def enumerate_devices(self):
+        """Just like safetlib.transport.enumerate_devices,
+        but with exception catching, so that transports can fail separately.
+        """
+        devices = []
+        for transport in self.all_transports():
+            try:
+                new_devices = transport.enumerate()
+            except BaseException as e:
+                self.print_error('enumerate failed for {}. error {}'
+                                 .format(transport.__name__, str(e)))
+            else:
+                devices.extend(new_devices)
+        return devices
+
+    def get_transport(self, path=None):
+        """Reimplemented safetlib.transport.get_transport,
+        (1) for old safetlib
+        (2) to be able to disable specific transports
+        (3) to call our own enumerate_devices that catches exceptions
+        """
+        if path is None:
+            try:
+                return self.enumerate_devices()[0]
+            except IndexError:
+                raise Exception("No Safe-T mini found") from None
+
+        def match_prefix(a, b):
+            return a.startswith(b) or b.startswith(a)
+        transports = [t for t in self.all_transports() if match_prefix(path, t.PATH_PREFIX)]
+        if transports:
+            return transports[0].find_by_path(path)
+        raise Exception("Unknown path prefix '%s'" % path)

electrum/plugins/trezor/trezor.py

@@ -13,7 +13,7 @@ from electrum.keystore import Hardware_KeyStore, is_xpubkey, parse_xpubkey, xtyp
 from electrum.base_wizard import ScriptTypeNotSupported
 
 from ..hw_wallet import HW_PluginBase
-from ..hw_wallet.plugin import is_any_tx_output_on_change_branch
+from ..hw_wallet.plugin import is_any_tx_output_on_change_branch, trezor_validate_op_return_output_and_get_data
 
 
 # TREZOR initialization methods
@@ -157,7 +157,10 @@ class TrezorPlugin(HW_PluginBase):
                      'download the updated firmware from {}')
                    .format(self.device, client.label(), self.firmware_URL))
             self.print_error(msg)
-            handler.show_error(msg)
+            if handler:
+                handler.show_error(msg)
+            else:
+                raise Exception(msg)
             return None
 
         return client
@@ -362,7 +365,7 @@ class TrezorPlugin(HW_PluginBase):
         for txin in tx.inputs():
             txinputtype = self.types.TxInputType()
             if txin['type'] == 'coinbase':
-                prev_hash = "\0"*32
+                prev_hash = b"\x00"*32
                 prev_index = 0xffffffff  # signed int -1
             else:
                 if for_sig:
@@ -461,7 +464,7 @@ class TrezorPlugin(HW_PluginBase):
             txoutputtype.amount = amount
             if _type == TYPE_SCRIPT:
                 txoutputtype.script_type = self.types.OutputScriptType.PAYTOOPRETURN
-                txoutputtype.op_return_data = address[2:]
+                txoutputtype.op_return_data = trezor_validate_op_return_output_and_get_data(o)
             elif _type == TYPE_ADDRESS:
                 txoutputtype.script_type = self.types.OutputScriptType.PAYTOADDRESS
                 txoutputtype.address = address
@@ -471,7 +474,8 @@ class TrezorPlugin(HW_PluginBase):
         has_change = False
         any_output_on_change_branch = is_any_tx_output_on_change_branch(tx)
 
-        for _type, address, amount in tx.outputs():
+        for o in tx.outputs():
+            _type, address, amount = o.type, o.address, o.value
             use_create_by_derivation = False
 
             info = tx.output_info.get(address)

electrum/plugins/trustedcoin/trustedcoin.py

@@ -33,6 +33,7 @@ from urllib.parse import quote
 
 from electrum import bitcoin, ecc, constants, keystore, version
 from electrum.bitcoin import *
+from electrum.transaction import TxOutput
 from electrum.mnemonic import Mnemonic
 from electrum.wallet import Multisig_Wallet, Deterministic_Wallet
 from electrum.i18n import _
@@ -273,7 +274,7 @@ class Wallet_2fa(Multisig_Wallet):
         fee = self.extra_fee(config) if not is_sweep else 0
         if fee:
             address = self.billing_info['billing_address']
-            fee_output = (TYPE_ADDRESS, address, fee)
+            fee_output = TxOutput(TYPE_ADDRESS, address, fee)
             try:
                 tx = mk_tx(outputs + [fee_output])
             except NotEnoughFunds:
@@ -395,9 +396,9 @@ class TrustedCoinPlugin(BasePlugin):
     def get_tx_extra_fee(self, wallet, tx):
         if type(wallet) != Wallet_2fa:
             return
-        for _type, addr, amount in tx.outputs():
-            if _type == TYPE_ADDRESS and wallet.is_billing_address(addr):
-                return addr, amount
+        for o in tx.outputs():
+            if o.type == TYPE_ADDRESS and wallet.is_billing_address(o.address):
+                return o.address, o.value
 
     def finish_requesting(func):
         def f(self, *args, **kwargs):

electrum/simple_config.py

@@ -4,7 +4,8 @@ import time
 import os
 import stat
 from decimal import Decimal
-from typing import Union
+from typing import Union, Optional
+from numbers import Real
 
 from copy import deepcopy
 
@@ -295,35 +296,51 @@ class SimpleConfig(PrintError):
             return fee
         return get_fee_within_limits
 
-    @impose_hard_limits_on_fee
-    def eta_to_fee(self, slider_pos) -> Union[int, None]:
+    def eta_to_fee(self, slider_pos) -> Optional[int]:
         """Returns fee in sat/kbyte."""
         slider_pos = max(slider_pos, 0)
         slider_pos = min(slider_pos, len(FEE_ETA_TARGETS))
         if slider_pos < len(FEE_ETA_TARGETS):
-            target_blocks = FEE_ETA_TARGETS[slider_pos]
-            fee = self.fee_estimates.get(target_blocks)
+            num_blocks = FEE_ETA_TARGETS[slider_pos]
+            fee = self.eta_target_to_fee(num_blocks)
         else:
-            fee = self.fee_estimates.get(2)
-            if fee is not None:
-                fee += fee/2
-                fee = int(fee)
+            fee = self.eta_target_to_fee(1)
         return fee
 
-    def fee_to_depth(self, target_fee):
+    @impose_hard_limits_on_fee
+    def eta_target_to_fee(self, num_blocks: int) -> Optional[int]:
+        """Returns fee in sat/kbyte."""
+        if num_blocks == 1:
+            fee = self.fee_estimates.get(2)
+            if fee is not None:
+                fee += fee / 2
+                fee = int(fee)
+        else:
+            fee = self.fee_estimates.get(num_blocks)
+        return fee
+
+    def fee_to_depth(self, target_fee: Real) -> int:
+        """For a given sat/vbyte fee, returns an estimate of how deep
+        it would be in the current mempool in vbytes.
+        Pessimistic == overestimates the depth.
+        """
         depth = 0
         for fee, s in self.mempool_fees:
             depth += s
             if fee <= target_fee:
                 break
-        else:
-            return 0
         return depth
 
-    @impose_hard_limits_on_fee
     def depth_to_fee(self, slider_pos) -> int:
         """Returns fee in sat/kbyte."""
         target = self.depth_target(slider_pos)
+        return self.depth_target_to_fee(target)
+
+    @impose_hard_limits_on_fee
+    def depth_target_to_fee(self, target: int) -> int:
+        """Returns fee in sat/kbyte.
+        target: desired mempool depth in vbytes
+        """
         depth = 0
         for fee, s in self.mempool_fees:
             depth += s
@@ -331,6 +348,10 @@ class SimpleConfig(PrintError):
                 break
         else:
             return 0
+        # add one sat/byte as currently that is
+        # the max precision of the histogram
+        fee += 1
+        # convert to sat/kbyte
         return fee * 1000
 
     def depth_target(self, slider_pos):

electrum/storage.py

@@ -29,7 +29,7 @@ import json
 import copy
 import re
 import stat
-import pbkdf2, hmac, hashlib
+import hmac, hashlib
 import base64
 import zlib
 from collections import defaultdict
@@ -44,7 +44,7 @@ from .keystore import bip44_derivation
 
 OLD_SEED_VERSION = 4        # electrum versions < 2.0
 NEW_SEED_VERSION = 11       # electrum versions >= 2.0
-FINAL_SEED_VERSION = 17     # electrum >= 2.7 will set this to prevent
+FINAL_SEED_VERSION = 18     # electrum >= 2.7 will set this to prevent
                             # old versions from overwriting new format
 
 
@@ -67,15 +67,84 @@ def get_derivation_used_for_hw_device_encryption():
 # storage encryption version
 STO_EV_PLAINTEXT, STO_EV_USER_PW, STO_EV_XPUB_PW = range(0, 3)
 
-class WalletStorage(PrintError):
 
-    def __init__(self, path, manual_upgrades=False):
-        self.print_error("wallet path", path)
-        self.manual_upgrades = manual_upgrades
-        self.lock = threading.RLock()
+class JsonDB(PrintError):
+
+    def __init__(self, path):
+        self.db_lock = threading.RLock()
         self.data = {}
         self.path = path
         self.modified = False
+
+    def get(self, key, default=None):
+        with self.db_lock:
+            v = self.data.get(key)
+            if v is None:
+                v = default
+            else:
+                v = copy.deepcopy(v)
+        return v
+
+    def put(self, key, value):
+        try:
+            json.dumps(key, cls=util.MyEncoder)
+            json.dumps(value, cls=util.MyEncoder)
+        except:
+            self.print_error("json error: cannot save", key)
+            return
+        with self.db_lock:
+            if value is not None:
+                if self.data.get(key) != value:
+                    self.modified = True
+                    self.data[key] = copy.deepcopy(value)
+            elif key in self.data:
+                self.modified = True
+                self.data.pop(key)
+
+    @profiler
+    def write(self):
+        with self.db_lock:
+            self._write()
+
+    def _write(self):
+        if threading.currentThread().isDaemon():
+            self.print_error('warning: daemon thread cannot write db')
+            return
+        if not self.modified:
+            return
+        s = json.dumps(self.data, indent=4, sort_keys=True, cls=util.MyEncoder)
+        s = self.encrypt_before_writing(s)
+
+        temp_path = "%s.tmp.%s" % (self.path, os.getpid())
+        with open(temp_path, "w", encoding='utf-8') as f:
+            f.write(s)
+            f.flush()
+            os.fsync(f.fileno())
+
+        mode = os.stat(self.path).st_mode if os.path.exists(self.path) else stat.S_IREAD | stat.S_IWRITE
+        # perform atomic write on POSIX systems
+        try:
+            os.rename(temp_path, self.path)
+        except:
+            os.remove(self.path)
+            os.rename(temp_path, self.path)
+        os.chmod(self.path, mode)
+        self.print_error("saved", self.path)
+        self.modified = False
+
+    def encrypt_before_writing(self, plaintext: str) -> str:
+        return plaintext
+
+    def file_exists(self):
+        return self.path and os.path.exists(self.path)
+
+
+class WalletStorage(JsonDB):
+
+    def __init__(self, path, manual_upgrades=False):
+        self.print_error("wallet path", path)
+        JsonDB.__init__(self, path)
+        self.manual_upgrades = manual_upgrades
         self.pubkey = None
         if self.file_exists():
             with open(self.path, "r", encoding='utf-8') as f:
@@ -160,12 +229,9 @@ class WalletStorage(PrintError):
         except:
             return STO_EV_PLAINTEXT
 
-    def file_exists(self):
-        return self.path and os.path.exists(self.path)
-
     @staticmethod
     def get_eckey_from_password(password):
-        secret = pbkdf2.PBKDF2(password, '', iterations=1024, macmodule=hmac, digestmodule=hashlib.sha512).read(64)
+        secret = hashlib.pbkdf2_hmac('sha512', password.encode('utf-8'), b'', iterations=1024)
         ec_key = ecc.ECPrivkey.from_arbitrary_size_secret(secret)
         return ec_key
 
@@ -189,6 +255,17 @@ class WalletStorage(PrintError):
         s = s.decode('utf8')
         self.load_data(s)
 
+    def encrypt_before_writing(self, plaintext: str) -> str:
+        s = plaintext
+        if self.pubkey:
+            s = bytes(s, 'utf8')
+            c = zlib.compress(s)
+            enc_magic = self._get_encryption_magic()
+            public_key = ecc.ECPubkey(bfh(self.pubkey))
+            s = public_key.encrypt_message(c, enc_magic)
+            s = s.decode('utf8')
+        return s
+
     def check_password(self, password):
         """Raises an InvalidPassword exception on invalid password"""
         if not self.is_encrypted():
@@ -211,71 +288,9 @@ class WalletStorage(PrintError):
             self.pubkey = None
             self._encryption_version = STO_EV_PLAINTEXT
         # make sure next storage.write() saves changes
-        with self.lock:
+        with self.db_lock:
             self.modified = True
 
-    def get(self, key, default=None):
-        with self.lock:
-            v = self.data.get(key)
-            if v is None:
-                v = default
-            else:
-                v = copy.deepcopy(v)
-        return v
-
-    def put(self, key, value):
-        try:
-            json.dumps(key, cls=util.MyEncoder)
-            json.dumps(value, cls=util.MyEncoder)
-        except:
-            self.print_error("json error: cannot save", key)
-            return
-        with self.lock:
-            if value is not None:
-                if self.data.get(key) != value:
-                    self.modified = True
-                    self.data[key] = copy.deepcopy(value)
-            elif key in self.data:
-                self.modified = True
-                self.data.pop(key)
-
-    @profiler
-    def write(self):
-        with self.lock:
-            self._write()
-
-    def _write(self):
-        if threading.currentThread().isDaemon():
-            self.print_error('warning: daemon thread cannot write wallet')
-            return
-        if not self.modified:
-            return
-        s = json.dumps(self.data, indent=4, sort_keys=True, cls=util.MyEncoder)
-        if self.pubkey:
-            s = bytes(s, 'utf8')
-            c = zlib.compress(s)
-            enc_magic = self._get_encryption_magic()
-            public_key = ecc.ECPubkey(bfh(self.pubkey))
-            s = public_key.encrypt_message(c, enc_magic)
-            s = s.decode('utf8')
-
-        temp_path = "%s.tmp.%s" % (self.path, os.getpid())
-        with open(temp_path, "w", encoding='utf-8') as f:
-            f.write(s)
-            f.flush()
-            os.fsync(f.fileno())
-
-        mode = os.stat(self.path).st_mode if os.path.exists(self.path) else stat.S_IREAD | stat.S_IWRITE
-        # perform atomic write on POSIX systems
-        try:
-            os.rename(temp_path, self.path)
-        except:
-            os.remove(self.path)
-            os.rename(temp_path, self.path)
-        os.chmod(self.path, mode)
-        self.print_error("saved", self.path)
-        self.modified = False
-
     def requires_split(self):
         d = self.get('accounts', {})
         return len(d) > 1
@@ -304,7 +319,7 @@ class WalletStorage(PrintError):
             storage2.upgrade()
             storage2.write()
             result = [storage1.path, storage2.path]
-        elif wallet_type in ['bip44', 'trezor', 'keepkey', 'ledger', 'btchip', 'digitalbitbox']:
+        elif wallet_type in ['bip44', 'trezor', 'keepkey', 'ledger', 'btchip', 'digitalbitbox', 'safe_t']:
             mpk = storage.get('master_public_keys')
             for k in d.keys():
                 i = int(k)
@@ -341,6 +356,7 @@ class WalletStorage(PrintError):
         self.convert_version_15()
         self.convert_version_16()
         self.convert_version_17()
+        self.convert_version_18()
 
         self.put('seed_version', FINAL_SEED_VERSION)  # just to be sure
         self.write()
@@ -401,7 +417,7 @@ class WalletStorage(PrintError):
             self.put('wallet_type', 'standard')
             self.put('keystore', d)
 
-        elif wallet_type in ['trezor', 'keepkey', 'ledger', 'digitalbitbox']:
+        elif wallet_type in ['trezor', 'keepkey', 'ledger', 'digitalbitbox', 'safe_t']:
             xpub = xpubs["x/0'"]
             derivation = self.get('derivation', bip44_derivation(0))
             d = {
@@ -555,6 +571,15 @@ class WalletStorage(PrintError):
 
         self.put('seed_version', 17)
 
+    def convert_version_18(self):
+        # delete verified_tx3 as its structure changed
+        if not self._is_upgrade_method_needed(17, 17):
+            return
+
+        self.put('verified_tx3', None)
+
+        self.put('seed_version', 18)
+
     def convert_imported(self):
         if not self._is_upgrade_method_needed(0, 13):
             return
@@ -637,7 +662,7 @@ class WalletStorage(PrintError):
             # version 1.9.8 created v6 wallets when an incorrect seed was entered in the restore dialog
             msg += '\n\nThis file was created because of a bug in version 1.9.8.'
             if self.get('master_public_keys') is None and self.get('master_private_keys') is None and self.get('imported_keys') is None:
-                # pbkdf2 was not included with the binaries, and wallet creation aborted.
+                # pbkdf2 (at that time an additional dependency) was not included with the binaries, and wallet creation aborted.
                 msg += "\nIt does not contain any keys, and can safely be removed."
             else:
                 # creation was complete if electrum was run from source

electrum/tests/test_simple_config.py

@@ -110,6 +110,43 @@ class Test_SimpleConfig(SequentialTestCase):
         result.pop('config_version', None)
         self.assertEqual({"something": "a"}, result)
 
+    def test_depth_target_to_fee(self):
+        config = SimpleConfig(self.options)
+        config.mempool_fees = [[49, 100110], [10, 121301], [6, 153731], [5, 125872], [1, 36488810]]
+        self.assertEqual( 2 * 1000, config.depth_target_to_fee(1000000))
+        self.assertEqual( 6 * 1000, config.depth_target_to_fee( 500000))
+        self.assertEqual( 7 * 1000, config.depth_target_to_fee( 250000))
+        self.assertEqual(11 * 1000, config.depth_target_to_fee( 200000))
+        self.assertEqual(50 * 1000, config.depth_target_to_fee( 100000))
+        config.mempool_fees = []
+        self.assertEqual( 1 * 1000, config.depth_target_to_fee(10 ** 5))
+        self.assertEqual( 1 * 1000, config.depth_target_to_fee(10 ** 6))
+        self.assertEqual( 1 * 1000, config.depth_target_to_fee(10 ** 7))
+        config.mempool_fees = [[1, 36488810]]
+        self.assertEqual( 2 * 1000, config.depth_target_to_fee(10 ** 5))
+        self.assertEqual( 2 * 1000, config.depth_target_to_fee(10 ** 6))
+        self.assertEqual( 2 * 1000, config.depth_target_to_fee(10 ** 7))
+        self.assertEqual( 1 * 1000, config.depth_target_to_fee(10 ** 8))
+        config.mempool_fees = [[5, 125872], [1, 36488810]]
+        self.assertEqual( 6 * 1000, config.depth_target_to_fee(10 ** 5))
+        self.assertEqual( 2 * 1000, config.depth_target_to_fee(10 ** 6))
+        self.assertEqual( 2 * 1000, config.depth_target_to_fee(10 ** 7))
+        self.assertEqual( 1 * 1000, config.depth_target_to_fee(10 ** 8))
+
+    def test_fee_to_depth(self):
+        config = SimpleConfig(self.options)
+        config.mempool_fees = [[49, 100000], [10, 120000], [6, 150000], [5, 125000], [1, 36000000]]
+        self.assertEqual(100000, config.fee_to_depth(500))
+        self.assertEqual(100000, config.fee_to_depth(50))
+        self.assertEqual(100000, config.fee_to_depth(49))
+        self.assertEqual(220000, config.fee_to_depth(48))
+        self.assertEqual(220000, config.fee_to_depth(10))
+        self.assertEqual(370000, config.fee_to_depth(9))
+        self.assertEqual(370000, config.fee_to_depth(6.5))
+        self.assertEqual(370000, config.fee_to_depth(6))
+        self.assertEqual(495000, config.fee_to_depth(5.5))
+        self.assertEqual(36495000, config.fee_to_depth(0.5))
+
 
 class TestUserConfig(SequentialTestCase):
 

electrum/tests/test_wallet_vertical.py

@@ -7,8 +7,10 @@ from typing import Sequence
 from electrum import storage, bitcoin, keystore, constants
 from electrum import Transaction
 from electrum import SimpleConfig
-from electrum.wallet import TX_HEIGHT_UNCONFIRMED, TX_HEIGHT_UNCONF_PARENT, sweep, Multisig_Wallet, Standard_Wallet, Imported_Wallet
+from electrum.address_synchronizer import TX_HEIGHT_UNCONFIRMED, TX_HEIGHT_UNCONF_PARENT
+from electrum.wallet import sweep, Multisig_Wallet, Standard_Wallet, Imported_Wallet
 from electrum.util import bfh, bh2u
+from electrum.transaction import TxOutput
 
 from electrum.plugins.trustedcoin import trustedcoin
 
@@ -17,6 +19,11 @@ from . import SequentialTestCase
 from .test_bitcoin import needs_test_with_all_ecc_implementations
 
 
+_UNICODE_HORROR_HEX = 'e282bf20f09f988020f09f98882020202020e3818620e38191e3819fe381be20e3828fe3828b2077cda2cda2cd9d68cda16fcda2cda120ccb8cda26bccb5cd9f6eccb4cd98c7ab77ccb8cc9b73cd9820cc80cc8177cd98cda2e1b8a9ccb561d289cca1cda27420cca7cc9568cc816fccb572cd8fccb5726f7273cca120ccb6cda1cda06cc4afccb665cd9fcd9f20ccb6cd9d696ecda220cd8f74cc9568ccb7cca1cd9f6520cd9fcd9f64cc9b61cd9c72cc95cda16bcca2cca820cda168ccb465cd8f61ccb7cca2cca17274cc81cd8f20ccb4ccb7cda0c3b2ccb5ccb666ccb82075cca7cd986ec3adcc9bcd9c63cda2cd8f6fccb7cd8f64ccb8cda265cca1cd9d3fcd9e'
+UNICODE_HORROR = bfh(_UNICODE_HORROR_HEX).decode('utf-8')
+# '₿ 😀 😈     う けたま わる w͢͢͝h͡o͢͡ ̸͢k̵͟n̴͘ǫw̸̛s͘ ̀́w͘͢ḩ̵a҉̡͢t ̧̕h́o̵r͏̵rors̡ ̶͡͠lį̶e͟͟ ̶͝in͢ ͏t̕h̷̡͟e ͟͟d̛a͜r̕͡k̢̨ ͡h̴e͏a̷̢̡rt́͏ ̴̷͠ò̵̶f̸ u̧͘ní̛͜c͢͏o̷͏d̸͢e̡͝?͞'
+
+
 class WalletIntegrityHelper:
 
     gap_limit = 1  # make tests run faster
@@ -67,7 +74,6 @@ class WalletIntegrityHelper:
         return w
 
 
-# TODO passphrase/seed_extension
 class TestWalletKeystoreAddressIntegrityForMainnet(SequentialTestCase):
 
     @needs_test_with_all_ecc_implementations
@@ -110,6 +116,26 @@ class TestWalletKeystoreAddressIntegrityForMainnet(SequentialTestCase):
         self.assertEqual(w.get_receiving_addresses()[0], 'bc1q3g5tmkmlvxryhh843v4dz026avatc0zzr6h3af')
         self.assertEqual(w.get_change_addresses()[0], 'bc1qdy94n2q5qcp0kg7v9yzwe6wvfkhnvyzje7nx2p')
 
+    @needs_test_with_all_ecc_implementations
+    @mock.patch.object(storage.WalletStorage, '_write')
+    def test_electrum_seed_segwit_passphrase(self, mock_write):
+        seed_words = 'bitter grass shiver impose acquire brush forget axis eager alone wine silver'
+        self.assertEqual(bitcoin.seed_type(seed_words), 'segwit')
+
+        ks = keystore.from_seed(seed_words, UNICODE_HORROR, False)
+
+        WalletIntegrityHelper.check_seeded_keystore_sanity(self, ks)
+        self.assertTrue(isinstance(ks, keystore.BIP32_KeyStore))
+
+        self.assertEqual(ks.xprv, 'zprvAZDmEQiCLUcZXPfrBXoksCD2R6RMAzAre7SUyBotibisy9c7vGhLYvHaP3d9rYU12DKAWdZfscPNA7qEPgTkCDqX5sE93ryAJAQvkDbfLxU')
+        self.assertEqual(ks.xpub, 'zpub6nD7dvF6ArArjskKHZLmEL9ky8FqaSti1LN5maDWGwFrqwwGTp1b6ic4EHwciFNaYDmCXcQYxXSiF9BjcLCMPcaYkVN2nQD6QjYQ8vpSR3Z')
+
+        w = WalletIntegrityHelper.create_standard_wallet(ks)
+        self.assertEqual(w.txin_type, 'p2wpkh')
+
+        self.assertEqual(w.get_receiving_addresses()[0], 'bc1qx94dutas7ysn2my645cyttujrms5d9p57f6aam')
+        self.assertEqual(w.get_change_addresses()[0], 'bc1qcywwsy87sdp8vz5rfjh3sxdv6rt95kujdqq38g')
+
     @needs_test_with_all_ecc_implementations
     @mock.patch.object(storage.WalletStorage, '_write')
     def test_electrum_seed_old(self, mock_write):
@@ -182,6 +208,25 @@ class TestWalletKeystoreAddressIntegrityForMainnet(SequentialTestCase):
         self.assertEqual(w.get_receiving_addresses()[0], '16j7Dqk3Z9DdTdBtHcCVLaNQy9MTgywUUo')
         self.assertEqual(w.get_change_addresses()[0], '1GG5bVeWgAp5XW7JLCphse14QaC4qiHyWn')
 
+    @needs_test_with_all_ecc_implementations
+    @mock.patch.object(storage.WalletStorage, '_write')
+    def test_bip39_seed_bip44_standard_passphrase(self, mock_write):
+        seed_words = 'treat dwarf wealth gasp brass outside high rent blood crowd make initial'
+        self.assertEqual(keystore.bip39_is_checksum_valid(seed_words), (True, True))
+
+        ks = keystore.from_bip39_seed(seed_words, UNICODE_HORROR, "m/44'/0'/0'")
+
+        self.assertTrue(isinstance(ks, keystore.BIP32_KeyStore))
+
+        self.assertEqual(ks.xprv, 'xprv9z8izheguGnLopSqkY7GcGFrP2Gu6rzBvvHo6uB9B8DWJhsows6WDZAsbBTaP3ncP2AVbTQphyEQkahrB9s1L7ihZtfz5WGQPMbXwsUtSik')
+        self.assertEqual(ks.xpub, 'xpub6D85QDBajeLe2JXJrZeGyQCaw47PWKi3J9DPuHakjTkVBWCxVQQkmMVMSSfnw39tj9FntbozpRtb1AJ8ubjeVSBhyK4M5mzdvsXZzKPwodT')
+
+        w = WalletIntegrityHelper.create_standard_wallet(ks)
+        self.assertEqual(w.txin_type, 'p2pkh')
+
+        self.assertEqual(w.get_receiving_addresses()[0], '1F88g2naBMhDB7pYFttPWGQgryba3hPevM')
+        self.assertEqual(w.get_change_addresses()[0], '1H4QD1rg2zQJ4UjuAVJr5eW1fEM8WMqyxh')
+
     @needs_test_with_all_ecc_implementations
     @mock.patch.object(storage.WalletStorage, '_write')
     def test_bip39_seed_bip49_p2sh_segwit(self, mock_write):
@@ -488,7 +533,7 @@ class TestWalletSending(TestCaseForTestnet):
         wallet1.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED)
 
         # wallet1 -> wallet2
-        outputs = [(bitcoin.TYPE_ADDRESS, wallet2.get_receiving_address(), 250000)]
+        outputs = [TxOutput(bitcoin.TYPE_ADDRESS, wallet2.get_receiving_address(), 250000)]
         tx = wallet1.mktx(outputs=outputs, password=None, config=self.config, fee=5000)
 
         self.assertTrue(tx.is_complete())
@@ -508,7 +553,7 @@ class TestWalletSending(TestCaseForTestnet):
         wallet2.receive_tx_callback(tx.txid(), tx, TX_HEIGHT_UNCONFIRMED)
 
         # wallet2 -> wallet1
-        outputs = [(bitcoin.TYPE_ADDRESS, wallet1.get_receiving_address(), 100000)]
+        outputs = [TxOutput(bitcoin.TYPE_ADDRESS, wallet1.get_receiving_address(), 100000)]
         tx = wallet2.mktx(outputs=outputs, password=None, config=self.config, fee=5000)
 
         self.assertTrue(tx.is_complete())
@@ -561,7 +606,7 @@ class TestWalletSending(TestCaseForTestnet):
         wallet1a.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED)
 
         # wallet1 -> wallet2
-        outputs = [(bitcoin.TYPE_ADDRESS, wallet2.get_receiving_address(), 370000)]
+        outputs = [TxOutput(bitcoin.TYPE_ADDRESS, wallet2.get_receiving_address(), 370000)]
         tx = wallet1a.mktx(outputs=outputs, password=None, config=self.config, fee=5000)
         tx = Transaction(tx.serialize())  # simulates moving partial txn between cosigners
         self.assertFalse(tx.is_complete())
@@ -584,7 +629,7 @@ class TestWalletSending(TestCaseForTestnet):
         wallet2.receive_tx_callback(tx.txid(), tx, TX_HEIGHT_UNCONFIRMED)
 
         # wallet2 -> wallet1
-        outputs = [(bitcoin.TYPE_ADDRESS, wallet1a.get_receiving_address(), 100000)]
+        outputs = [TxOutput(bitcoin.TYPE_ADDRESS, wallet1a.get_receiving_address(), 100000)]
         tx = wallet2.mktx(outputs=outputs, password=None, config=self.config, fee=5000)
 
         self.assertTrue(tx.is_complete())
@@ -652,7 +697,7 @@ class TestWalletSending(TestCaseForTestnet):
         wallet1a.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED)
 
         # wallet1 -> wallet2
-        outputs = [(bitcoin.TYPE_ADDRESS, wallet2a.get_receiving_address(), 165000)]
+        outputs = [TxOutput(bitcoin.TYPE_ADDRESS, wallet2a.get_receiving_address(), 165000)]
         tx = wallet1a.mktx(outputs=outputs, password=None, config=self.config, fee=5000)
         txid = tx.txid()
         tx = Transaction(tx.serialize())  # simulates moving partial txn between cosigners
@@ -678,7 +723,7 @@ class TestWalletSending(TestCaseForTestnet):
         wallet2a.receive_tx_callback(tx.txid(), tx, TX_HEIGHT_UNCONFIRMED)
 
         # wallet2 -> wallet1
-        outputs = [(bitcoin.TYPE_ADDRESS, wallet1a.get_receiving_address(), 100000)]
+        outputs = [TxOutput(bitcoin.TYPE_ADDRESS, wallet1a.get_receiving_address(), 100000)]
         tx = wallet2a.mktx(outputs=outputs, password=None, config=self.config, fee=5000)
         txid = tx.txid()
         tx = Transaction(tx.serialize())  # simulates moving partial txn between cosigners
@@ -732,7 +777,7 @@ class TestWalletSending(TestCaseForTestnet):
         wallet1a.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED)
 
         # wallet1 -> wallet2
-        outputs = [(bitcoin.TYPE_ADDRESS, wallet2.get_receiving_address(), 1000000)]
+        outputs = [TxOutput(bitcoin.TYPE_ADDRESS, wallet2.get_receiving_address(), 1000000)]
         tx = wallet1a.mktx(outputs=outputs, password=None, config=self.config, fee=5000)
 
         self.assertTrue(tx.is_complete())
@@ -752,7 +797,7 @@ class TestWalletSending(TestCaseForTestnet):
         wallet2.receive_tx_callback(tx.txid(), tx, TX_HEIGHT_UNCONFIRMED)
 
         # wallet2 -> wallet1
-        outputs = [(bitcoin.TYPE_ADDRESS, wallet1a.get_receiving_address(), 300000)]
+        outputs = [TxOutput(bitcoin.TYPE_ADDRESS, wallet1a.get_receiving_address(), 300000)]
         tx = wallet2.mktx(outputs=outputs, password=None, config=self.config, fee=5000)
 
         self.assertTrue(tx.is_complete())
@@ -788,7 +833,7 @@ class TestWalletSending(TestCaseForTestnet):
         wallet.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED)
 
         # create tx
-        outputs = [(bitcoin.TYPE_ADDRESS, '2N1VTMMFb91SH9SNRAkT7z8otP5eZEct4KL', 2500000)]
+        outputs = [TxOutput(bitcoin.TYPE_ADDRESS, '2N1VTMMFb91SH9SNRAkT7z8otP5eZEct4KL', 2500000)]
         coins = wallet.get_spendable_coins(domain=None, config=self.config)
         tx = wallet.make_unsigned_transaction(coins, outputs, config=self.config, fixed_fee=5000)
         tx.set_rbf(True)
@@ -874,7 +919,7 @@ class TestWalletSending(TestCaseForTestnet):
         wallet.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED)
 
         # create tx
-        outputs = [(bitcoin.TYPE_ADDRESS, '2N1VTMMFb91SH9SNRAkT7z8otP5eZEct4KL', 2500000)]
+        outputs = [TxOutput(bitcoin.TYPE_ADDRESS, '2N1VTMMFb91SH9SNRAkT7z8otP5eZEct4KL', 2500000)]
         coins = wallet.get_spendable_coins(domain=None, config=self.config)
         tx = wallet.make_unsigned_transaction(coins, outputs, config=self.config, fixed_fee=5000)
         tx.set_rbf(True)
@@ -1004,7 +1049,7 @@ class TestWalletOfflineSigning(TestCaseForTestnet):
         wallet_online.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED)
 
         # create unsigned tx
-        outputs = [(bitcoin.TYPE_ADDRESS, 'tb1qp0mv2sxsyxxfj5gl0332f9uyez93su9cf26757', 2500000)]
+        outputs = [TxOutput(bitcoin.TYPE_ADDRESS, 'tb1qp0mv2sxsyxxfj5gl0332f9uyez93su9cf26757', 2500000)]
         tx = wallet_online.mktx(outputs=outputs, password=None, config=self.config, fee=5000)
         tx.set_rbf(True)
         tx.locktime = 1325340
@@ -1044,7 +1089,7 @@ class TestWalletOfflineSigning(TestCaseForTestnet):
         wallet_online.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED)
 
         # create unsigned tx
-        outputs = [(bitcoin.TYPE_ADDRESS, 'tb1qp0mv2sxsyxxfj5gl0332f9uyez93su9cf26757', 2500000)]
+        outputs = [TxOutput(bitcoin.TYPE_ADDRESS, 'tb1qp0mv2sxsyxxfj5gl0332f9uyez93su9cf26757', 2500000)]
         tx = wallet_online.mktx(outputs=outputs, password=None, config=self.config, fee=5000)
         tx.set_rbf(True)
         tx.locktime = 1325341
@@ -1085,7 +1130,7 @@ class TestWalletOfflineSigning(TestCaseForTestnet):
         wallet_online.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED)
 
         # create unsigned tx
-        outputs = [(bitcoin.TYPE_ADDRESS, 'tb1qp0mv2sxsyxxfj5gl0332f9uyez93su9cf26757', 2500000)]
+        outputs = [TxOutput(bitcoin.TYPE_ADDRESS, 'tb1qp0mv2sxsyxxfj5gl0332f9uyez93su9cf26757', 2500000)]
         tx = wallet_online.mktx(outputs=outputs, password=None, config=self.config, fee=5000)
         tx.set_rbf(True)
         tx.locktime = 1325341
@@ -1121,7 +1166,7 @@ class TestWalletOfflineSigning(TestCaseForTestnet):
         wallet_online.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED)
 
         # create unsigned tx
-        outputs = [(bitcoin.TYPE_ADDRESS, 'tb1quk7ahlhr3qmjndy0uvu9y9hxfesrtahtta9ghm', 2500000)]
+        outputs = [TxOutput(bitcoin.TYPE_ADDRESS, 'tb1quk7ahlhr3qmjndy0uvu9y9hxfesrtahtta9ghm', 2500000)]
         tx = wallet_online.mktx(outputs=outputs, password=None, config=self.config, fee=5000)
         tx.set_rbf(True)
         tx.locktime = 1325340
@@ -1155,7 +1200,7 @@ class TestWalletOfflineSigning(TestCaseForTestnet):
         wallet_online.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED)
 
         # create unsigned tx
-        outputs = [(bitcoin.TYPE_ADDRESS, 'tb1quk7ahlhr3qmjndy0uvu9y9hxfesrtahtta9ghm', 2500000)]
+        outputs = [TxOutput(bitcoin.TYPE_ADDRESS, 'tb1quk7ahlhr3qmjndy0uvu9y9hxfesrtahtta9ghm', 2500000)]
         tx = wallet_online.mktx(outputs=outputs, password=None, config=self.config, fee=5000)
         tx.set_rbf(True)
         tx.locktime = 1325340
@@ -1189,7 +1234,7 @@ class TestWalletOfflineSigning(TestCaseForTestnet):
         wallet_online.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED)
 
         # create unsigned tx
-        outputs = [(bitcoin.TYPE_ADDRESS, 'tb1quk7ahlhr3qmjndy0uvu9y9hxfesrtahtta9ghm', 2500000)]
+        outputs = [TxOutput(bitcoin.TYPE_ADDRESS, 'tb1quk7ahlhr3qmjndy0uvu9y9hxfesrtahtta9ghm', 2500000)]
         tx = wallet_online.mktx(outputs=outputs, password=None, config=self.config, fee=5000)
         tx.set_rbf(True)
         tx.locktime = 1325340
@@ -1226,7 +1271,7 @@ class TestWalletOfflineSigning(TestCaseForTestnet):
         wallet_online.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED)
 
         # create unsigned tx
-        outputs = [(bitcoin.TYPE_ADDRESS, 'tb1quk7ahlhr3qmjndy0uvu9y9hxfesrtahtta9ghm', 2500000)]
+        outputs = [TxOutput(bitcoin.TYPE_ADDRESS, 'tb1quk7ahlhr3qmjndy0uvu9y9hxfesrtahtta9ghm', 2500000)]
         tx = wallet_online.mktx(outputs=outputs, password=None, config=self.config, fee=5000)
         tx.set_rbf(True)
         tx.locktime = 1325340
@@ -1263,7 +1308,7 @@ class TestWalletOfflineSigning(TestCaseForTestnet):
         wallet_online.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED)
 
         # create unsigned tx
-        outputs = [(bitcoin.TYPE_ADDRESS, 'tb1quk7ahlhr3qmjndy0uvu9y9hxfesrtahtta9ghm', 2500000)]
+        outputs = [TxOutput(bitcoin.TYPE_ADDRESS, 'tb1quk7ahlhr3qmjndy0uvu9y9hxfesrtahtta9ghm', 2500000)]
         tx = wallet_online.mktx(outputs=outputs, password=None, config=self.config, fee=5000)
         tx.set_rbf(True)
         tx.locktime = 1325340
@@ -1300,7 +1345,7 @@ class TestWalletOfflineSigning(TestCaseForTestnet):
         wallet_online.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED)
 
         # create unsigned tx
-        outputs = [(bitcoin.TYPE_ADDRESS, 'tb1quk7ahlhr3qmjndy0uvu9y9hxfesrtahtta9ghm', 2500000)]
+        outputs = [TxOutput(bitcoin.TYPE_ADDRESS, 'tb1quk7ahlhr3qmjndy0uvu9y9hxfesrtahtta9ghm', 2500000)]
         tx = wallet_online.mktx(outputs=outputs, password=None, config=self.config, fee=5000)
         tx.set_rbf(True)
         tx.locktime = 1325340
@@ -1349,7 +1394,7 @@ class TestWalletOfflineSigning(TestCaseForTestnet):
         wallet_online.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED)
 
         # create unsigned tx
-        outputs = [(bitcoin.TYPE_ADDRESS, '2MuCQQHJNnrXzQzuqfUCfAwAjPqpyEHbgue', 2500000)]
+        outputs = [TxOutput(bitcoin.TYPE_ADDRESS, '2MuCQQHJNnrXzQzuqfUCfAwAjPqpyEHbgue', 2500000)]
         tx = wallet_online.mktx(outputs=outputs, password=None, config=self.config, fee=5000)
         tx.set_rbf(True)
         tx.locktime = 1325503
@@ -1406,7 +1451,7 @@ class TestWalletOfflineSigning(TestCaseForTestnet):
         wallet_online.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED)
 
         # create unsigned tx
-        outputs = [(bitcoin.TYPE_ADDRESS, '2N8CtJRwxb2GCaiWWdSHLZHHLoZy53CCyxf', 2500000)]
+        outputs = [TxOutput(bitcoin.TYPE_ADDRESS, '2N8CtJRwxb2GCaiWWdSHLZHHLoZy53CCyxf', 2500000)]
         tx = wallet_online.mktx(outputs=outputs, password=None, config=self.config, fee=5000)
         tx.set_rbf(True)
         tx.locktime = 1325504
@@ -1465,7 +1510,7 @@ class TestWalletOfflineSigning(TestCaseForTestnet):
         wallet_online.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED)
 
         # create unsigned tx
-        outputs = [(bitcoin.TYPE_ADDRESS, '2MyoZVy8T1t94yLmyKu8DP1SmbWvnxbkwRA', 2500000)]
+        outputs = [TxOutput(bitcoin.TYPE_ADDRESS, '2MyoZVy8T1t94yLmyKu8DP1SmbWvnxbkwRA', 2500000)]
         tx = wallet_online.mktx(outputs=outputs, password=None, config=self.config, fee=5000)
         tx.set_rbf(True)
         tx.locktime = 1325505

electrum/transaction.py

@@ -27,7 +27,7 @@
 
 # Note: The deserialization code originally comes from ABE.
 
-from typing import Sequence, Union
+from typing import Sequence, Union, NamedTuple
 
 from .util import print_error, profiler
 
@@ -59,6 +59,10 @@ class NotRecognizedRedeemScript(Exception):
     pass
 
 
+TxOutput = NamedTuple("TxOutput", [('type', int), ('address', str), ('value', Union[int, str])])
+# ^ value is str when the output is set to max: '!'
+
+
 class BCDataStream(object):
     def __init__(self):
         self.input = None
@@ -672,7 +676,7 @@ class Transaction:
 
         `signatures` is expected to be a list of sigs with signatures[i]
         intended for self._inputs[i].
-        This is used by the Trezor and KeepKey plugins.
+        This is used by the Trezor, KeepKey an Safe-T plugins.
         """
         if self.is_complete():
             return
@@ -721,7 +725,7 @@ class Transaction:
             return
         d = deserialize(self.raw, force_full_parse)
         self._inputs = d['inputs']
-        self._outputs = [(x['type'], x['address'], x['value']) for x in d['outputs']]
+        self._outputs = [TxOutput(x['type'], x['address'], x['value']) for x in d['outputs']]
         self.locktime = d['lockTime']
         self.version = d['version']
         self.is_partial_originally = d['partial']
@@ -1180,17 +1184,17 @@ class Transaction:
 
     def get_outputs(self):
         """convert pubkeys to addresses"""
-        o = []
-        for type, x, v in self.outputs():
-            if type == TYPE_ADDRESS:
-                addr = x
-            elif type == TYPE_PUBKEY:
+        outputs = []
+        for o in self.outputs():
+            if o.type == TYPE_ADDRESS:
+                addr = o.address
+            elif o.type == TYPE_PUBKEY:
                 # TODO do we really want this conversion? it's not really that address after all
-                addr = bitcoin.public_key_to_p2pkh(bfh(x))
+                addr = bitcoin.public_key_to_p2pkh(bfh(o.address))
             else:
-                addr = 'SCRIPT ' + x
-            o.append((addr,v))      # consider using yield (addr, v)
-        return o
+                addr = 'SCRIPT ' + o.address
+            outputs.append((addr, o.value))      # consider using yield (addr, v)
+        return outputs
 
     def get_output_addresses(self):
         return [addr for addr, val in self.get_outputs()]

electrum/util.py

@@ -23,6 +23,7 @@
 import binascii
 import os, sys, re, json
 from collections import defaultdict
+from typing import NamedTuple
 from datetime import datetime
 import decimal
 from decimal import Decimal
@@ -31,6 +32,8 @@ import urllib
 import threading
 import hmac
 import stat
+import inspect
+from locale import localeconv
 
 from .i18n import _
 
@@ -118,6 +121,8 @@ class UserCancelled(Exception):
     pass
 
 class Satoshis(object):
+    __slots__ = ('value',)
+
     def __new__(cls, value):
         self = super(Satoshis, cls).__new__(cls)
         self.value = value
@@ -130,6 +135,8 @@ class Satoshis(object):
         return format_satoshis(self.value) + " BTC"
 
 class Fiat(object):
+    __slots__ = ('value', 'ccy')
+
     def __new__(cls, value, ccy):
         self = super(Fiat, cls).__new__(cls)
         self.ccy = ccy
@@ -164,12 +171,14 @@ class MyEncoder(json.JSONEncoder):
 
 class PrintError(object):
     '''A handy base class'''
+    verbosity_filter = ''
+
     def diagnostic_name(self):
         return self.__class__.__name__
 
     def print_error(self, *msg):
-        # only prints with --verbose flag
-        print_error("[%s]" % self.diagnostic_name(), *msg)
+        if self.verbosity_filter in verbosity or verbosity == '*':
+            print_error("[%s]" % self.diagnostic_name(), *msg)
 
     def print_stderr(self, *msg):
         print_stderr("[%s]" % self.diagnostic_name(), *msg)
@@ -213,6 +222,7 @@ class DebugMem(ThreadJob):
 
 class DaemonThread(threading.Thread, PrintError):
     """ daemon thread that terminates cleanly """
+    verbosity_filter = 'd'
 
     def __init__(self):
         threading.Thread.__init__(self)
@@ -263,15 +273,14 @@ class DaemonThread(threading.Thread, PrintError):
         self.print_error("stopped")
 
 
-# TODO: disable
-is_verbose = True
+verbosity = '*'
 def set_verbosity(b):
-    global is_verbose
-    is_verbose = b
+    global verbosity
+    verbosity = b
 
 
 def print_error(*args):
-    if not is_verbose: return
+    if not verbosity: return
     print_stderr(*args)
 
 def print_stderr(*args):
@@ -307,14 +316,24 @@ def constant_time_compare(val1, val2):
 
 # decorator that prints execution time
 def profiler(func):
-    def do_profile(func, args, kw_args):
-        n = func.__name__
+    def get_func_name(args):
+        arg_names_from_sig = inspect.getfullargspec(func).args
+        # prepend class name if there is one (and if we can find it)
+        if len(arg_names_from_sig) > 0 and len(args) > 0 \
+                and arg_names_from_sig[0] in ('self', 'cls', 'klass'):
+            classname = args[0].__class__.__name__
+        else:
+            classname = ''
+        name = '{}.{}'.format(classname, func.__name__) if classname else func.__name__
+        return name
+    def do_profile(args, kw_args):
+        name = get_func_name(args)
         t0 = time.time()
         o = func(*args, **kw_args)
         t = time.time() - t0
-        print_error("[profiler]", n, "%.4f"%t)
+        print_error("[profiler]", name, "%.4f"%t)
         return o
-    return lambda *args, **kw_args: do_profile(func, args, kw_args)
+    return lambda *args, **kw_args: do_profile(args, kw_args)
 
 
 def android_ext_dir():
@@ -463,8 +482,10 @@ def format_satoshis_plain(x, decimal_point = 8):
     return "{:.8f}".format(Decimal(x) / scale_factor).rstrip('0').rstrip('.')
 
 
+DECIMAL_POINT = localeconv()['decimal_point']
+
+
 def format_satoshis(x, num_zeros=0, decimal_point=8, precision=None, is_diff=False, whitespaces=False):
-    from locale import localeconv
     if x is None:
         return 'unknown'
     if precision is None:
@@ -474,7 +495,7 @@ def format_satoshis(x, num_zeros=0, decimal_point=8, precision=None, is_diff=Fal
         decimal_format = '+' + decimal_format
     result = ("{:" + decimal_format + "f}").format(x / pow (10, decimal_point)).rstrip('0')
     integer_part, fract_part = result.split(".")
-    dp = localeconv()['decimal_point']
+    dp = DECIMAL_POINT
     if len(fract_part) < num_zeros:
         fract_part += "0" * (num_zeros - len(fract_part))
     result = integer_part + dp + fract_part
@@ -901,3 +922,13 @@ def make_dir(path, allow_symlink=True):
             raise Exception('Dangling link: ' + path)
         os.mkdir(path)
         os.chmod(path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
+
+
+TxMinedStatus = NamedTuple("TxMinedStatus", [("height", int),
+                                             ("conf", int),
+                                             ("timestamp", int),
+                                             ("header_hash", str)])
+VerifiedTxInfo = NamedTuple("VerifiedTxInfo", [("height", int),
+                                               ("timestamp", int),
+                                               ("txpos", int),
+                                               ("header_hash", str)])

electrum/verifier.py

@@ -20,12 +20,19 @@
 # ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
-from .util import ThreadJob, bh2u
+
+from typing import Sequence, Optional
+
+from .util import ThreadJob, bh2u, VerifiedTxInfo
 from .bitcoin import Hash, hash_decode, hash_encode
 from .transaction import Transaction
+from .blockchain import hash_header
 
 
-class InnerNodeOfSpvProofIsValidTx(Exception): pass
+class MerkleVerificationFailure(Exception): pass
+class MissingBlockHeader(MerkleVerificationFailure): pass
+class MerkleRootMismatch(MerkleVerificationFailure): pass
+class InnerNodeOfSpvProofIsValidTx(MerkleVerificationFailure): pass
 
 
 class SPV(ThreadJob):
@@ -85,44 +92,41 @@ class SPV(ThreadJob):
         tx_hash = params[0]
         tx_height = merkle.get('block_height')
         pos = merkle.get('pos')
-        try:
-            merkle_root = self.hash_merkle_root(merkle['merkle'], tx_hash, pos)
-        except InnerNodeOfSpvProofIsValidTx:
-            self.print_error("merkle verification failed for {} (inner node looks like tx)"
-                             .format(tx_hash))
-            return
+        merkle_branch = merkle.get('merkle')
         header = self.network.blockchain().read_header(tx_height)
-        # FIXME: if verification fails below,
-        # we should make a fresh connection to a server to
-        # recover from this, as this TX will now never verify
-        if not header:
-            self.print_error(
-                "merkle verification failed for {} (missing header {})"
-                .format(tx_hash, tx_height))
-            return
-        if header.get('merkle_root') != merkle_root:
-            self.print_error(
-                "merkle verification failed for {} (merkle root mismatch {} != {})"
-                .format(tx_hash, header.get('merkle_root'), merkle_root))
+        try:
+            verify_tx_is_in_block(tx_hash, merkle_branch, pos, header, tx_height)
+        except MerkleVerificationFailure as e:
+            self.print_error(str(e))
+            # FIXME: we should make a fresh connection to a server
+            # to recover from this, as this TX will now never verify
             return
         # we passed all the tests
-        self.merkle_roots[tx_hash] = merkle_root
+        self.merkle_roots[tx_hash] = header.get('merkle_root')
         try:
             # note: we could pop in the beginning, but then we would request
             # this proof again in case of verification failure from the same server
             self.requested_merkle.remove(tx_hash)
         except KeyError: pass
         self.print_error("verified %s" % tx_hash)
-        self.wallet.add_verified_tx(tx_hash, (tx_height, header.get('timestamp'), pos))
+        header_hash = hash_header(header)
+        vtx_info = VerifiedTxInfo(tx_height, header.get('timestamp'), pos, header_hash)
+        self.wallet.add_verified_tx(tx_hash, vtx_info)
         if self.is_up_to_date() and self.wallet.is_up_to_date():
             self.wallet.save_verified_tx(write=True)
 
     @classmethod
-    def hash_merkle_root(cls, merkle_s, target_hash, pos):
-        h = hash_decode(target_hash)
-        for i in range(len(merkle_s)):
-            item = merkle_s[i]
-            h = Hash(hash_decode(item) + h) if ((pos >> i) & 1) else Hash(h + hash_decode(item))
+    def hash_merkle_root(cls, merkle_branch: Sequence[str], tx_hash: str, leaf_pos_in_tree: int):
+        """Return calculated merkle root."""
+        try:
+            h = hash_decode(tx_hash)
+            merkle_branch_bytes = [hash_decode(item) for item in merkle_branch]
+            int(leaf_pos_in_tree)  # raise if invalid
+        except Exception as e:
+            raise MerkleVerificationFailure(e)
+
+        for i, item in enumerate(merkle_branch_bytes):
+            h = Hash(item + h) if ((leaf_pos_in_tree >> i) & 1) else Hash(h + item)
             cls._raise_if_valid_tx(bh2u(h))
         return hash_encode(h)
 
@@ -141,7 +145,7 @@ class SPV(ThreadJob):
             raise InnerNodeOfSpvProofIsValidTx()
 
     def undo_verifications(self):
-        height = self.blockchain.get_checkpoint()
+        height = self.blockchain.get_forkpoint()
         tx_hashes = self.wallet.undo_verifications(self.blockchain, height)
         for tx_hash in tx_hashes:
             self.print_error("redoing", tx_hash)
@@ -156,3 +160,16 @@ class SPV(ThreadJob):
 
     def is_up_to_date(self):
         return not self.requested_merkle
+
+
+def verify_tx_is_in_block(tx_hash: str, merkle_branch: Sequence[str],
+                          leaf_pos_in_tree: int, block_header: Optional[dict],
+                          block_height: int) -> None:
+    """Raise MerkleVerificationFailure if verification fails."""
+    if not block_header:
+        raise MissingBlockHeader("merkle verification failed for {} (missing header {})"
+                                 .format(tx_hash, block_height))
+    calc_merkle_root = SPV.hash_merkle_root(merkle_branch, tx_hash, leaf_pos_in_tree)
+    if block_header.get('merkle_root') != calc_merkle_root:
+        raise MerkleRootMismatch("merkle verification failed for {} ({} != {})".format(
+            tx_hash, block_header.get('merkle_root'), calc_merkle_root))

icons.qrc

@@ -31,6 +31,8 @@
     <file>icons/qrcode.png</file>
     <file>icons/qrcode_white.png</file>
     <file>icons/preferences.png</file>
+    <file>icons/safe-t_unpaired.png</file>
+    <file>icons/safe-t.png</file>
     <file>icons/seed.png</file>
     <file>icons/status_connected.png</file>
     <file>icons/status_connected_proxy.png</file>

run_electrum

@@ -46,7 +46,6 @@ def check_imports():
         import ecdsa
         import requests
         import qrcode
-        import pbkdf2
         import google.protobuf
         import jsonrpclib
     except ImportError as e:
@@ -356,7 +355,7 @@ if __name__ == '__main__':
     # config is an object passed to the various constructors (wallet, interface, gui)
     if is_android:
         config_options = {
-            'verbose': True,
+            'verbosity': '',
             'cmd': 'gui',
             'gui': 'kivy',
         }
@@ -377,7 +376,7 @@ if __name__ == '__main__':
         config_options['electrum_path'] = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'electrum_data')
 
     # kivy sometimes freezes when we write to sys.stderr
-    set_verbosity(config_options.get('verbose') and config_options.get('gui')!='kivy')
+    set_verbosity(config_options.get('verbosity') if config_options.get('gui') != 'kivy' else '')
 
     # check uri
     uri = config_options.get('url')

setup.py

@@ -2,7 +2,7 @@
 
 # python setup.py sdist --format=zip,gztar
 
-from setuptools import setup
+from setuptools import setup, find_packages
 import os
 import sys
 import platform
@@ -43,7 +43,6 @@ if platform.system() in ['Linux', 'FreeBSD', 'DragonFly']:
 extras_require = {
     'hardware': requirements_hw,
     'fast': ['pycryptodomex'],
-    ':python_version < "3.5"': ['typing>=3.0.0'],
 }
 extras_require['full'] = extras_require['hardware'] + extras_require['fast']
 
@@ -58,28 +57,15 @@ setup(
         'electrum.gui',
         'electrum.gui.qt',
         'electrum.plugins',
-        'electrum.plugins.audio_modem',
-        'electrum.plugins.cosigner_pool',
-        'electrum.plugins.email_requests',
-        'electrum.plugins.greenaddress_instant',
-        'electrum.plugins.hw_wallet',
-        'electrum.plugins.keepkey',
-        'electrum.plugins.labels',
-        'electrum.plugins.ledger',
-        'electrum.plugins.revealer',
-        'electrum.plugins.trezor',
-        'electrum.plugins.digitalbitbox',
-        'electrum.plugins.trustedcoin',
-        'electrum.plugins.virtualkeyboard',
-    ],
+    ] + [('electrum.plugins.'+pkg) for pkg in find_packages('electrum/plugins')],
     package_dir={
         'electrum': 'electrum'
     },
     package_data={
         '': ['*.txt', '*.json', '*.ttf', '*.otf'],
         'electrum': [
-            'electrum/wordlist/*.txt',
-            'electrum/locale/*/LC_MESSAGES/electrum.mo',
+            'wordlist/*.txt',
+            'locale/*/LC_MESSAGES/electrum.mo',
         ],
     },
     scripts=['electrum/electrum'],

tox.ini

@@ -6,7 +6,7 @@ deps=
 	pytest
 	coverage
 commands=
-	coverage run --source=electrum -m py.test -v
+	coverage run --source=electrum '--omit=electrum/gui/*,electrum/plugins/*,electrum/scripts/*,electrum/tests/*' -m py.test -v
 	coverage report
 extras=
 	fast