diff --git a/.travis.yml b/.travis.yml index a68aaed8..63dde547 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,7 +33,7 @@ jobs: install: - sudo docker build --no-cache -t electrum-wine-builder-img ./contrib/build-wine/docker/ script: - - sudo docker run --name electrum-wine-builder-cont -v $PWD:/opt/electrum --rm --workdir /opt/electrum/contrib/build-wine electrum-wine-builder-img ./build.sh $TRAVIS_COMMIT + - sudo docker run --name electrum-wine-builder-cont -v $PWD:/opt/wine64/drive_c/electrum --rm --workdir /opt/wine64/drive_c/electrum/contrib/build-wine electrum-wine-builder-img ./build.sh after_success: true - os: osx language: c @@ -53,4 +53,4 @@ jobs: script: - ./contrib/deterministic-build/check_submodules.sh after_success: true - if: tag IS present \ No newline at end of file + if: tag IS present diff --git a/MANIFEST.in b/MANIFEST.in index b028b14c..d09f4f5d 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,15 +3,16 @@ include README.rst include electrum.conf.sample include electrum.desktop include *.py -include electrum +include run_electrum include contrib/requirements/requirements.txt include contrib/requirements/requirements-hw.txt -recursive-include lib *.py -recursive-include gui *.py -recursive-include plugins *.py recursive-include packages *.py recursive-include packages cacert.pem include icons.qrc -recursive-include icons * -recursive-include scripts * +graft icons +graft electrum +prune electrum/tests + +global-exclude __pycache__ +global-exclude *.py[co] diff --git a/README.rst b/README.rst index 4a201e81..c7665604 100644 --- a/README.rst +++ b/README.rst @@ -15,9 +15,9 @@ Electrum - Lightweight Bitcoin client .. image:: https://coveralls.io/repos/github/spesmilo/electrum/badge.svg?branch=master :target: https://coveralls.io/github/spesmilo/electrum?branch=master :alt: Test coverage statistics -.. image:: https://img.shields.io/badge/help-translating-blue.svg +.. image:: https://d322cqt584bo4o.cloudfront.net/electrum/localized.svg :target: https://crowdin.com/project/electrum - :alt: Help translating Electrum online + :alt: Help translate Electrum online diff --git a/contrib/build-osx/osx.spec b/contrib/build-osx/osx.spec index 5e2e6d4d..0cd01c66 100644 --- a/contrib/build-osx/osx.spec +++ b/contrib/build-osx/osx.spec @@ -23,6 +23,7 @@ block_cipher = None # see https://github.com/pyinstaller/pyinstaller/issues/2005 hiddenimports = [] hiddenimports += collect_submodules('trezorlib') +hiddenimports += collect_submodules('safetlib') hiddenimports += collect_submodules('btchip') hiddenimports += collect_submodules('keepkeylib') hiddenimports += collect_submodules('websocket') @@ -33,10 +34,11 @@ datas = [ (electrum+'electrum/locale', PYPKG + '/locale') ] datas += collect_data_files('trezorlib') +datas += collect_data_files('safetlib') datas += collect_data_files('btchip') datas += collect_data_files('keepkeylib') -# Add libusb so Trezor will work +# Add libusb so Trezor and Safe-T mini will work binaries = [(electrum + "contrib/build-osx/libusb-1.0.dylib", ".")] binaries += [(electrum + "contrib/build-osx/libsecp256k1.0.dylib", ".")] @@ -57,6 +59,8 @@ a = Analysis([electrum+ MAIN_SCRIPT, electrum+'electrum/plugins/email_requests/qt.py', electrum+'electrum/plugins/trezor/client.py', electrum+'electrum/plugins/trezor/qt.py', + electrum+'electrum/plugins/safe_t/client.py', + electrum+'electrum/plugins/safe_t/qt.py', electrum+'electrum/plugins/keepkey/qt.py', electrum+'electrum/plugins/ledger/qt.py', ], diff --git a/contrib/build-wine/build-electrum-git.sh b/contrib/build-wine/build-electrum-git.sh index bf0a06eb..f98a77a1 100755 --- a/contrib/build-wine/build-electrum-git.sh +++ b/contrib/build-wine/build-electrum-git.sh @@ -19,29 +19,13 @@ set -e mkdir -p tmp cd tmp -if [ -d ./electrum ]; then - rm ./electrum -rf -fi - -git clone https://github.com/spesmilo/electrum -b master - -pushd electrum -if [ ! -z "$1" ]; then - # a commit/tag/branch was specified - if ! git cat-file -e "$1" 2> /dev/null - then # can't find target - # try pull requests - git config --local --add remote.origin.fetch '+refs/pull/*/merge:refs/remotes/origin/pr/*' - git fetch --all - fi - git checkout $1 -fi +pushd $WINEPREFIX/drive_c/electrum # Load electrum-icons and electrum-locale for this release git submodule init git submodule update -VERSION=`git describe --tags --dirty` +VERSION=`git describe --tags --dirty || printf 'custom'` echo "Last commit: $VERSION" pushd ./contrib/deterministic-build/electrum-locale @@ -59,11 +43,9 @@ popd find -exec touch -d '2000-11-11T11:11:11+00:00' {} + popd -rm -rf $WINEPREFIX/drive_c/electrum -cp -r electrum $WINEPREFIX/drive_c/electrum -cp electrum/LICENCE . -cp -r ./electrum/contrib/deterministic-build/electrum-locale/locale $WINEPREFIX/drive_c/electrum/electrum/ -cp ./electrum/contrib/deterministic-build/electrum-icons/icons_rc.py $WINEPREFIX/drive_c/electrum/electrum/gui/qt/ +cp $WINEPREFIX/drive_c/electrum/LICENCE . +cp -r $WINEPREFIX/drive_c/electrum/contrib/deterministic-build/electrum-locale/locale $WINEPREFIX/drive_c/electrum/electrum/ +cp $WINEPREFIX/drive_c/electrum/contrib/deterministic-build/electrum-icons/icons_rc.py $WINEPREFIX/drive_c/electrum/electrum/gui/qt/ # Install frozen dependencies $PYTHON -m pip install -r ../../deterministic-build/requirements.txt diff --git a/contrib/build-wine/build.sh b/contrib/build-wine/build.sh index ef4012b4..01ca071f 100755 --- a/contrib/build-wine/build.sh +++ b/contrib/build-wine/build.sh @@ -2,10 +2,6 @@ # Lucky number export PYTHONHASHSEED=22 -if [ ! -z "$1" ]; then - to_build="$1" -fi - here=$(dirname "$0") test -n "$here" -a -d "$here" || exit @@ -28,5 +24,5 @@ find -exec touch -d '2000-11-11T11:11:11+00:00' {} + popd ls -l /opt/wine64/drive_c/python* -$here/build-electrum-git.sh $to_build && \ +$here/build-electrum-git.sh && \ echo "Done." diff --git a/contrib/build-wine/deterministic.spec b/contrib/build-wine/deterministic.spec index 6561c62c..ae3fba80 100644 --- a/contrib/build-wine/deterministic.spec +++ b/contrib/build-wine/deterministic.spec @@ -18,6 +18,7 @@ home = 'C:\\electrum\\' # see https://github.com/pyinstaller/pyinstaller/issues/2005 hiddenimports = [] hiddenimports += collect_submodules('trezorlib') +hiddenimports += collect_submodules('safetlib') hiddenimports += collect_submodules('btchip') hiddenimports += collect_submodules('keepkeylib') hiddenimports += collect_submodules('websocket') @@ -37,6 +38,7 @@ datas = [ ('C:\\Program Files (x86)\\ZBar\\bin\\', '.') ] datas += collect_data_files('trezorlib') +datas += collect_data_files('safetlib') datas += collect_data_files('btchip') datas += collect_data_files('keepkeylib') @@ -54,6 +56,8 @@ a = Analysis([home+'run_electrum', home+'electrum/plugins/email_requests/qt.py', home+'electrum/plugins/trezor/client.py', home+'electrum/plugins/trezor/qt.py', + home+'electrum/plugins/safe_t/client.py', + home+'electrum/plugins/safe_t/qt.py', home+'electrum/plugins/keepkey/qt.py', home+'electrum/plugins/ledger/qt.py', #home+'packages/requests/utils.py' diff --git a/contrib/build-wine/docker/Dockerfile b/contrib/build-wine/docker/Dockerfile index 0d2a63b1..f46e9812 100644 --- a/contrib/build-wine/docker/Dockerfile +++ b/contrib/build-wine/docker/Dockerfile @@ -8,7 +8,7 @@ RUN dpkg --add-architecture i386 && \ wget=1.19.4-1ubuntu2.1 \ gnupg2=2.2.4-1ubuntu1.1 \ dirmngr=2.2.4-1ubuntu1.1 \ - software-properties-common=0.96.24.32.3 \ + software-properties-common=0.96.24.32.4 \ && \ wget -nc https://dl.winehq.org/wine-builds/Release.key && \ apt-key add Release.key && \ diff --git a/contrib/build-wine/docker/README.md b/contrib/build-wine/docker/README.md index ae7e0365..7cab0f58 100644 --- a/contrib/build-wine/docker/README.md +++ b/contrib/build-wine/docker/README.md @@ -25,14 +25,14 @@ folder. 3. Build Windows binaries ``` - $ TARGET=master + $ git checkout $REV $ sudo docker run \ --name electrum-wine-builder-cont \ - -v .:/opt/electrum \ + -v $PWD:/opt/wine64/drive_c/electrum \ --rm \ - --workdir /opt/electrum/contrib/build-wine \ + --workdir /opt/wine64/drive_c/electrum/contrib/build-wine \ electrum-wine-builder-img \ - ./build.sh $TARGET + ./build.sh ``` 4. The generated binaries are in `./contrib/build-wine/dist`. diff --git a/contrib/build-wine/electrum.nsi b/contrib/build-wine/electrum.nsi index 59e90d75..8d2482e2 100644 --- a/contrib/build-wine/electrum.nsi +++ b/contrib/build-wine/electrum.nsi @@ -72,7 +72,7 @@ !define MUI_ABORTWARNING !define MUI_ABORTWARNING_TEXT "Are you sure you wish to abort the installation of ${PRODUCT_NAME}?" - !define MUI_ICON "tmp\electrum\icons\electrum.ico" + !define MUI_ICON "c:\electrum\icons\electrum.ico" ;-------------------------------- ;Pages @@ -111,7 +111,7 @@ Section ;Files to pack into the installer File /r "dist\electrum\*.*" - File "..\..\icons\electrum.ico" + File "c:\electrum\icons\electrum.ico" ;Store installation folder WriteRegStr HKCU "Software\${PRODUCT_NAME}" "" $INSTDIR diff --git a/contrib/build-wine/prepare-wine.sh b/contrib/build-wine/prepare-wine.sh index 95ef980d..ffa31e62 100755 --- a/contrib/build-wine/prepare-wine.sh +++ b/contrib/build-wine/prepare-wine.sh @@ -79,11 +79,6 @@ retry() { here=$(dirname $(readlink -e $0)) set -e -# Clean up Wine environment -echo "Cleaning $WINEPREFIX" -rm -rf $WINEPREFIX -echo "done" - wine 'wineboot' # HACK to work around https://bugs.winehq.org/show_bug.cgi?id=42474#c22 @@ -101,8 +96,7 @@ KEYRING_PYTHON_DEV="keyring-electrum-build-python-dev.gpg" for server in $(shuf -e ha.pool.sks-keyservers.net \ hkp://p80.pool.sks-keyservers.net:80 \ keyserver.ubuntu.com \ - hkp://keyserver.ubuntu.com:80 \ - pgp.mit.edu) ; do + hkp://keyserver.ubuntu.com:80) ; do retry gpg --no-default-keyring --keyring $KEYRING_PYTHON_DEV --keyserver "$server" --recv-keys $KEYLIST_PYTHON_DEV \ && break || : ; done diff --git a/contrib/deterministic-build/requirements-hw.txt b/contrib/deterministic-build/requirements-hw.txt index 29fae2fd..f06b43e1 100644 --- a/contrib/deterministic-build/requirements-hw.txt +++ b/contrib/deterministic-build/requirements-hw.txt @@ -94,6 +94,9 @@ pyblake2==1.1.2 \ requests==2.19.1 \ --hash=sha256:63b52e3c866428a224f97cab011de738c36aec0185aa91cfacd418b5d58911d1 \ --hash=sha256:ec22d826a36ed72a7358ff3fe56cbd4ba69dd7a6718ffd450ff0e9df7a47ce6a +safet==0.1.3 \ + --hash=sha256:ba80fe9f6ba317ab9514a8726cd3792e68eb46dd419f380d48ae4a0ccae646dc \ + --hash=sha256:e5d8e6a87c8bdf1cefd07004181b93fd7631557fdab09d143ba8d1b29291d6dc setuptools==40.0.0 \ --hash=sha256:012adb8e25fbfd64c652e99e7bab58799a3aaf05d39ab38561f69190a909015f \ --hash=sha256:d68abee4eed409fbe8c302ac4d8429a1ffef912cd047a903b5701c024048dd49 diff --git a/contrib/deterministic-build/requirements.txt b/contrib/deterministic-build/requirements.txt index 67d1fafd..1b226f9d 100644 --- a/contrib/deterministic-build/requirements.txt +++ b/contrib/deterministic-build/requirements.txt @@ -16,8 +16,6 @@ idna==2.7 \ jsonrpclib-pelix==0.3.1 \ --hash=sha256:5417b1508d5a50ec64f6e5b88907f111155d52607b218ff3ba9a777afb2e49e3 \ --hash=sha256:bd89a6093bc4d47dc8a096197aacb827359944a4533be5193f3845f57b9f91b4 -pbkdf2==1.3 \ - --hash=sha256:ac6397369f128212c43064a2b4878038dab78dab41875364554aaf2a684e6979 pip==10.0.1 \ --hash=sha256:717cdffb2833be8409433a93746744b59505f42146e8d37de6c62b430e25d6d7 \ --hash=sha256:f2bd08e0cd1b06e10218feaf6fef299f473ba706582eb3bd9d52203fdbd7ee68 @@ -56,6 +54,10 @@ setuptools==40.0.0 \ six==1.11.0 \ --hash=sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9 \ --hash=sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb +typing==3.6.4 \ + --hash=sha256:3a887b021a77b292e151afb75323dea88a7bc1b3dfa92176cff8e44c8b68bddf \ + --hash=sha256:b2c689d54e1144bbcfd191b0832980a21c2dbcf7b5ff7a66248a60c90e951eb8 \ + --hash=sha256:d400a9344254803a2368533e4533a4200d21eb7b6b729c173bc38201a74db3f2 urllib3==1.23 \ --hash=sha256:a68ac5e15e76e7e5dd2b8f94007233e01effe3e50e8daddf69acfd81cb686baf \ --hash=sha256:b5725a0bd4ba422ab0e66e89e030c806576753ea3ee08554382c14e685d117b5 diff --git a/contrib/make_locale b/contrib/make_locale index 052f5d87..3c28d570 100755 --- a/contrib/make_locale +++ b/contrib/make_locale @@ -54,7 +54,7 @@ if crowdin_api_key: # Download & unzip print('Download translations') -s = requests.request('GET', 'https://crowdin.com/download/project/' + crowdin_identifier + '.zip').content +s = requests.request('GET', 'https://crowdin.com/backend/download/project/' + crowdin_identifier + '.zip').content zfobj = zipfile.ZipFile(io.BytesIO(s)) print('Unzip translations') diff --git a/contrib/requirements/requirements-hw.txt b/contrib/requirements/requirements-hw.txt index e8c7b2e9..4fe6477f 100644 --- a/contrib/requirements/requirements-hw.txt +++ b/contrib/requirements/requirements-hw.txt @@ -1,5 +1,6 @@ Cython>=0.27 trezor[hidapi]>=0.9.0 +safet[hidapi]>=0.1.0 keepkey btchip-python websocket-client diff --git a/contrib/requirements/requirements.txt b/contrib/requirements/requirements.txt index 26b1b3d3..99b859c3 100644 --- a/contrib/requirements/requirements.txt +++ b/contrib/requirements/requirements.txt @@ -1,6 +1,5 @@ pyaes>=0.1a1 ecdsa>=0.9 -pbkdf2 requests qrcode protobuf @@ -8,3 +7,4 @@ dnspython jsonrpclib-pelix PySocks>=1.6.6 qdarkstyle<3.0 +typing>=3.0.0 diff --git a/electrum/__init__.py b/electrum/__init__.py index d8df161f..48a60c15 100644 --- a/electrum/__init__.py +++ b/electrum/__init__.py @@ -1,6 +1,6 @@ from .version import ELECTRUM_VERSION from .util import format_satoshis, print_msg, print_error, set_verbosity -from .wallet import Synchronizer, Wallet +from .wallet import Wallet from .storage import WalletStorage from .coinchooser import COIN_CHOOSERS from .network import Network, pick_random_server diff --git a/electrum/address_synchronizer.py b/electrum/address_synchronizer.py new file mode 100644 index 00000000..2d6a70cc --- /dev/null +++ b/electrum/address_synchronizer.py @@ -0,0 +1,789 @@ +# Electrum - lightweight Bitcoin client +# Copyright (C) 2018 The Electrum Developers +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation files +# (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, +# publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS +# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import threading +import itertools +from collections import defaultdict + +from . import bitcoin +from .bitcoin import COINBASE_MATURITY, TYPE_ADDRESS, TYPE_PUBKEY +from .util import PrintError, profiler, bfh, VerifiedTxInfo, TxMinedStatus +from .transaction import Transaction, TxOutput +from .synchronizer import Synchronizer +from .verifier import SPV +from .blockchain import hash_header +from .i18n import _ + +TX_HEIGHT_LOCAL = -2 +TX_HEIGHT_UNCONF_PARENT = -1 +TX_HEIGHT_UNCONFIRMED = 0 + +class AddTransactionException(Exception): + pass + + +class UnrelatedTransactionException(AddTransactionException): + def __str__(self): + return _("Transaction is unrelated to this wallet.") + + +class AddressSynchronizer(PrintError): + """ + inherited by wallet + """ + + def __init__(self, storage): + self.storage = storage + self.network = None + # verifier (SPV) and synchronizer are started in start_threads + self.synchronizer = None + self.verifier = None + # locks: if you need to take multiple ones, acquire them in the order they are defined here! + self.lock = threading.RLock() + self.transaction_lock = threading.RLock() + # address -> list(txid, height) + self.history = storage.get('addr_history',{}) + # Verified transactions. txid -> VerifiedTxInfo. Access with self.lock. + verified_tx = storage.get('verified_tx3', {}) + self.verified_tx = {} + for txid, (height, timestamp, txpos, header_hash) in verified_tx.items(): + self.verified_tx[txid] = VerifiedTxInfo(height, timestamp, txpos, header_hash) + # Transactions pending verification. txid -> tx_height. Access with self.lock. + self.unverified_tx = defaultdict(int) + # true when synchronized + self.up_to_date = False + # thread local storage for caching stuff + self.threadlocal_cache = threading.local() + + self.load_and_cleanup() + + def load_and_cleanup(self): + self.load_transactions() + self.load_local_history() + self.check_history() + self.load_unverified_transactions() + self.remove_local_transactions_we_dont_have() + + def is_mine(self, address): + return address in self.history + + def get_addresses(self): + return sorted(self.history.keys()) + + def get_address_history(self, addr): + h = [] + # we need self.transaction_lock but get_tx_height will take self.lock + # so we need to take that too here, to enforce order of locks + with self.lock, self.transaction_lock: + related_txns = self._history_local.get(addr, set()) + for tx_hash in related_txns: + tx_height = self.get_tx_height(tx_hash).height + h.append((tx_hash, tx_height)) + return h + + def get_address_history_len(self, addr: str) -> int: + return len(self._history_local.get(addr, ())) + + def get_txin_address(self, txi): + addr = txi.get('address') + if addr and addr != "(pubkey)": + return addr + prevout_hash = txi.get('prevout_hash') + prevout_n = txi.get('prevout_n') + dd = self.txo.get(prevout_hash, {}) + for addr, l in dd.items(): + for n, v, is_cb in l: + if n == prevout_n: + return addr + return None + + def get_txout_address(self, txo: TxOutput): + if txo.type == TYPE_ADDRESS: + addr = txo.address + elif txo.type == TYPE_PUBKEY: + addr = bitcoin.public_key_to_p2pkh(bfh(txo.address)) + else: + addr = None + return addr + + def load_unverified_transactions(self): + # review transactions that are in the history + for addr, hist in self.history.items(): + for tx_hash, tx_height in hist: + # add it in case it was previously unconfirmed + self.add_unverified_tx(tx_hash, tx_height) + + def start_threads(self, network): + self.network = network + if self.network is not None: + self.verifier = SPV(self.network, self) + self.synchronizer = Synchronizer(self, network) + network.add_jobs([self.verifier, self.synchronizer]) + else: + self.verifier = None + self.synchronizer = None + + def stop_threads(self): + if self.network: + self.network.remove_jobs([self.synchronizer, self.verifier]) + self.synchronizer.release() + self.synchronizer = None + self.verifier = None + # Now no references to the synchronizer or verifier + # remain so they will be GC-ed + self.storage.put('stored_height', self.get_local_height()) + self.save_transactions() + self.save_verified_tx() + self.storage.write() + + def add_address(self, address): + if address not in self.history: + self.history[address] = [] + self.set_up_to_date(False) + if self.synchronizer: + self.synchronizer.add(address) + + def get_conflicting_transactions(self, tx): + """Returns a set of transaction hashes from the wallet history that are + directly conflicting with tx, i.e. they have common outpoints being + spent with tx. If the tx is already in wallet history, that will not be + reported as a conflict. + """ + conflicting_txns = set() + with self.transaction_lock: + for txin in tx.inputs(): + if txin['type'] == 'coinbase': + continue + prevout_hash = txin['prevout_hash'] + prevout_n = txin['prevout_n'] + spending_tx_hash = self.spent_outpoints[prevout_hash].get(prevout_n) + if spending_tx_hash is None: + continue + # this outpoint has already been spent, by spending_tx + assert spending_tx_hash in self.transactions + conflicting_txns |= {spending_tx_hash} + txid = tx.txid() + if txid in conflicting_txns: + # this tx is already in history, so it conflicts with itself + if len(conflicting_txns) > 1: + raise Exception('Found conflicting transactions already in wallet history.') + conflicting_txns -= {txid} + return conflicting_txns + + def add_transaction(self, tx_hash, tx, allow_unrelated=False): + assert tx_hash, tx_hash + assert tx, tx + assert tx.is_complete() + # we need self.transaction_lock but get_tx_height will take self.lock + # so we need to take that too here, to enforce order of locks + with self.lock, self.transaction_lock: + # NOTE: returning if tx in self.transactions might seem like a good idea + # BUT we track is_mine inputs in a txn, and during subsequent calls + # of add_transaction tx, we might learn of more-and-more inputs of + # being is_mine, as we roll the gap_limit forward + is_coinbase = tx.inputs()[0]['type'] == 'coinbase' + tx_height = self.get_tx_height(tx_hash).height + if not allow_unrelated: + # note that during sync, if the transactions are not properly sorted, + # it could happen that we think tx is unrelated but actually one of the inputs is is_mine. + # this is the main motivation for allow_unrelated + is_mine = any([self.is_mine(self.get_txin_address(txin)) for txin in tx.inputs()]) + is_for_me = any([self.is_mine(self.get_txout_address(txo)) for txo in tx.outputs()]) + if not is_mine and not is_for_me: + raise UnrelatedTransactionException() + # Find all conflicting transactions. + # In case of a conflict, + # 1. confirmed > mempool > local + # 2. this new txn has priority over existing ones + # When this method exits, there must NOT be any conflict, so + # either keep this txn and remove all conflicting (along with dependencies) + # or drop this txn + conflicting_txns = self.get_conflicting_transactions(tx) + if conflicting_txns: + existing_mempool_txn = any( + self.get_tx_height(tx_hash2).height in (TX_HEIGHT_UNCONFIRMED, TX_HEIGHT_UNCONF_PARENT) + for tx_hash2 in conflicting_txns) + existing_confirmed_txn = any( + self.get_tx_height(tx_hash2).height > 0 + for tx_hash2 in conflicting_txns) + if existing_confirmed_txn and tx_height <= 0: + # this is a non-confirmed tx that conflicts with confirmed txns; drop. + return False + if existing_mempool_txn and tx_height == TX_HEIGHT_LOCAL: + # this is a local tx that conflicts with non-local txns; drop. + return False + # keep this txn and remove all conflicting + to_remove = set() + to_remove |= conflicting_txns + for conflicting_tx_hash in conflicting_txns: + to_remove |= self.get_depending_transactions(conflicting_tx_hash) + for tx_hash2 in to_remove: + self.remove_transaction(tx_hash2) + # add inputs + def add_value_from_prev_output(): + dd = self.txo.get(prevout_hash, {}) + # note: this nested loop takes linear time in num is_mine outputs of prev_tx + for addr, outputs in dd.items(): + # note: instead of [(n, v, is_cb), ...]; we could store: {n -> (v, is_cb)} + for n, v, is_cb in outputs: + if n == prevout_n: + if addr and self.is_mine(addr): + if d.get(addr) is None: + d[addr] = set() + d[addr].add((ser, v)) + return + self.txi[tx_hash] = d = {} + for txi in tx.inputs(): + if txi['type'] == 'coinbase': + continue + prevout_hash = txi['prevout_hash'] + prevout_n = txi['prevout_n'] + ser = prevout_hash + ':%d' % prevout_n + self.spent_outpoints[prevout_hash][prevout_n] = tx_hash + add_value_from_prev_output() + # add outputs + self.txo[tx_hash] = d = {} + for n, txo in enumerate(tx.outputs()): + v = txo[2] + ser = tx_hash + ':%d'%n + addr = self.get_txout_address(txo) + if addr and self.is_mine(addr): + if d.get(addr) is None: + d[addr] = [] + d[addr].append((n, v, is_coinbase)) + # give v to txi that spends me + next_tx = self.spent_outpoints[tx_hash].get(n) + if next_tx is not None: + dd = self.txi.get(next_tx, {}) + if dd.get(addr) is None: + dd[addr] = set() + if (ser, v) not in dd[addr]: + dd[addr].add((ser, v)) + self._add_tx_to_local_history(next_tx) + # add to local history + self._add_tx_to_local_history(tx_hash) + # save + self.transactions[tx_hash] = tx + return True + + def remove_transaction(self, tx_hash): + def remove_from_spent_outpoints(): + # undo spends in spent_outpoints + if tx is not None: # if we have the tx, this branch is faster + for txin in tx.inputs(): + if txin['type'] == 'coinbase': + continue + prevout_hash = txin['prevout_hash'] + prevout_n = txin['prevout_n'] + self.spent_outpoints[prevout_hash].pop(prevout_n, None) + if not self.spent_outpoints[prevout_hash]: + self.spent_outpoints.pop(prevout_hash) + else: # expensive but always works + for prevout_hash, d in list(self.spent_outpoints.items()): + for prevout_n, spending_txid in d.items(): + if spending_txid == tx_hash: + self.spent_outpoints[prevout_hash].pop(prevout_n, None) + if not self.spent_outpoints[prevout_hash]: + self.spent_outpoints.pop(prevout_hash) + # Remove this tx itself; if nothing spends from it. + # It is not so clear what to do if other txns spend from it, but it will be + # removed when those other txns are removed. + if not self.spent_outpoints[tx_hash]: + self.spent_outpoints.pop(tx_hash) + + with self.transaction_lock: + self.print_error("removing tx from history", tx_hash) + tx = self.transactions.pop(tx_hash, None) + remove_from_spent_outpoints() + self._remove_tx_from_local_history(tx_hash) + self.txi.pop(tx_hash, None) + self.txo.pop(tx_hash, None) + + def receive_tx_callback(self, tx_hash, tx, tx_height): + self.add_unverified_tx(tx_hash, tx_height) + self.add_transaction(tx_hash, tx, allow_unrelated=True) + + def receive_history_callback(self, addr, hist, tx_fees): + with self.lock: + old_hist = self.get_address_history(addr) + for tx_hash, height in old_hist: + if (tx_hash, height) not in hist: + # make tx local + self.unverified_tx.pop(tx_hash, None) + self.verified_tx.pop(tx_hash, None) + if self.verifier: + self.verifier.remove_spv_proof_for_tx(tx_hash) + self.history[addr] = hist + + for tx_hash, tx_height in hist: + # add it in case it was previously unconfirmed + self.add_unverified_tx(tx_hash, tx_height) + # if addr is new, we have to recompute txi and txo + tx = self.transactions.get(tx_hash) + if tx is None: + continue + self.add_transaction(tx_hash, tx, allow_unrelated=True) + + # Store fees + self.tx_fees.update(tx_fees) + + @profiler + def load_transactions(self): + # load txi, txo, tx_fees + self.txi = self.storage.get('txi', {}) + for txid, d in list(self.txi.items()): + for addr, lst in d.items(): + self.txi[txid][addr] = set([tuple(x) for x in lst]) + self.txo = self.storage.get('txo', {}) + self.tx_fees = self.storage.get('tx_fees', {}) + tx_list = self.storage.get('transactions', {}) + # load transactions + self.transactions = {} + for tx_hash, raw in tx_list.items(): + tx = Transaction(raw) + self.transactions[tx_hash] = tx + if self.txi.get(tx_hash) is None and self.txo.get(tx_hash) is None: + self.print_error("removing unreferenced tx", tx_hash) + self.transactions.pop(tx_hash) + # load spent_outpoints + _spent_outpoints = self.storage.get('spent_outpoints', {}) + self.spent_outpoints = defaultdict(dict) + for prevout_hash, d in _spent_outpoints.items(): + for prevout_n_str, spending_txid in d.items(): + prevout_n = int(prevout_n_str) + self.spent_outpoints[prevout_hash][prevout_n] = spending_txid + + @profiler + def load_local_history(self): + self._history_local = {} # address -> set(txid) + for txid in itertools.chain(self.txi, self.txo): + self._add_tx_to_local_history(txid) + + @profiler + def check_history(self): + save = False + hist_addrs_mine = list(filter(lambda k: self.is_mine(k), self.history.keys())) + hist_addrs_not_mine = list(filter(lambda k: not self.is_mine(k), self.history.keys())) + for addr in hist_addrs_not_mine: + self.history.pop(addr) + save = True + for addr in hist_addrs_mine: + hist = self.history[addr] + for tx_hash, tx_height in hist: + if self.txi.get(tx_hash) or self.txo.get(tx_hash): + continue + tx = self.transactions.get(tx_hash) + if tx is not None: + self.add_transaction(tx_hash, tx, allow_unrelated=True) + save = True + if save: + self.save_transactions() + + def remove_local_transactions_we_dont_have(self): + txid_set = set(self.txi) | set(self.txo) + for txid in txid_set: + tx_height = self.get_tx_height(txid).height + if tx_height == TX_HEIGHT_LOCAL and txid not in self.transactions: + self.remove_transaction(txid) + + @profiler + def save_transactions(self, write=False): + with self.transaction_lock: + tx = {} + for k,v in self.transactions.items(): + tx[k] = str(v) + self.storage.put('transactions', tx) + self.storage.put('txi', self.txi) + self.storage.put('txo', self.txo) + self.storage.put('tx_fees', self.tx_fees) + self.storage.put('addr_history', self.history) + self.storage.put('spent_outpoints', self.spent_outpoints) + if write: + self.storage.write() + + def save_verified_tx(self, write=False): + with self.lock: + self.storage.put('verified_tx3', self.verified_tx) + if write: + self.storage.write() + + def clear_history(self): + with self.lock: + with self.transaction_lock: + self.txi = {} + self.txo = {} + self.tx_fees = {} + self.spent_outpoints = defaultdict(dict) + self.history = {} + self.verified_tx = {} + self.transactions = {} + self.save_transactions() + + def get_txpos(self, tx_hash): + """Returns (height, txpos) tuple, even if the tx is unverified.""" + with self.lock: + if tx_hash in self.verified_tx: + info = self.verified_tx[tx_hash] + return info.height, info.txpos + elif tx_hash in self.unverified_tx: + height = self.unverified_tx[tx_hash] + return (height, 0) if height > 0 else ((1e9 - height), 0) + else: + return (1e9+1, 0) + + def with_local_height_cached(func): + # get local height only once, as it's relatively expensive. + # take care that nested calls work as expected + def f(self, *args, **kwargs): + orig_val = getattr(self.threadlocal_cache, 'local_height', None) + self.threadlocal_cache.local_height = orig_val or self.get_local_height() + try: + return func(self, *args, **kwargs) + finally: + self.threadlocal_cache.local_height = orig_val + return f + + @with_local_height_cached + def get_history(self, domain=None): + # get domain + if domain is None: + domain = self.history.keys() + domain = set(domain) + # 1. Get the history of each address in the domain, maintain the + # delta of a tx as the sum of its deltas on domain addresses + tx_deltas = defaultdict(int) + for addr in domain: + h = self.get_address_history(addr) + for tx_hash, height in h: + delta = self.get_tx_delta(tx_hash, addr) + if delta is None or tx_deltas[tx_hash] is None: + tx_deltas[tx_hash] = None + else: + tx_deltas[tx_hash] += delta + # 2. create sorted history + history = [] + for tx_hash in tx_deltas: + delta = tx_deltas[tx_hash] + tx_mined_status = self.get_tx_height(tx_hash) + history.append((tx_hash, tx_mined_status, delta)) + history.sort(key = lambda x: self.get_txpos(x[0])) + history.reverse() + # 3. add balance + c, u, x = self.get_balance(domain) + balance = c + u + x + h2 = [] + for tx_hash, tx_mined_status, delta in history: + h2.append((tx_hash, tx_mined_status, delta, balance)) + if balance is None or delta is None: + balance = None + else: + balance -= delta + h2.reverse() + # fixme: this may happen if history is incomplete + if balance not in [None, 0]: + self.print_error("Error: history not synchronized") + return [] + + return h2 + + def _add_tx_to_local_history(self, txid): + with self.transaction_lock: + for addr in itertools.chain(self.txi.get(txid, []), self.txo.get(txid, [])): + cur_hist = self._history_local.get(addr, set()) + cur_hist.add(txid) + self._history_local[addr] = cur_hist + + def _remove_tx_from_local_history(self, txid): + with self.transaction_lock: + for addr in itertools.chain(self.txi.get(txid, []), self.txo.get(txid, [])): + cur_hist = self._history_local.get(addr, set()) + try: + cur_hist.remove(txid) + except KeyError: + pass + else: + self._history_local[addr] = cur_hist + + def add_unverified_tx(self, tx_hash, tx_height): + if tx_hash in self.verified_tx: + if tx_height in (TX_HEIGHT_UNCONFIRMED, TX_HEIGHT_UNCONF_PARENT): + with self.lock: + self.verified_tx.pop(tx_hash) + if self.verifier: + self.verifier.remove_spv_proof_for_tx(tx_hash) + else: + with self.lock: + # tx will be verified only if height > 0 + self.unverified_tx[tx_hash] = tx_height + # to remove pending proof requests: + if self.verifier: + self.verifier.remove_spv_proof_for_tx(tx_hash) + + def add_verified_tx(self, tx_hash: str, info: VerifiedTxInfo): + # Remove from the unverified map and add to the verified map + with self.lock: + self.unverified_tx.pop(tx_hash, None) + self.verified_tx[tx_hash] = info + tx_mined_status = self.get_tx_height(tx_hash) + self.network.trigger_callback('verified', tx_hash, tx_mined_status) + + def get_unverified_txs(self): + '''Returns a map from tx hash to transaction height''' + with self.lock: + return dict(self.unverified_tx) # copy + + def undo_verifications(self, blockchain, height): + '''Used by the verifier when a reorg has happened''' + txs = set() + with self.lock: + for tx_hash, info in list(self.verified_tx.items()): + tx_height = info.height + if tx_height >= height: + header = blockchain.read_header(tx_height) + if not header or hash_header(header) != info.header_hash: + self.verified_tx.pop(tx_hash, None) + # NOTE: we should add these txns to self.unverified_tx, + # but with what height? + # If on the new fork after the reorg, the txn is at the + # same height, we will not get a status update for the + # address. If the txn is not mined or at a diff height, + # we should get a status update. Unless we put tx into + # unverified_tx, it will turn into local. So we put it + # into unverified_tx with the old height, and if we get + # a status update, that will overwrite it. + self.unverified_tx[tx_hash] = tx_height + txs.add(tx_hash) + return txs + + def get_local_height(self): + """ return last known height if we are offline """ + cached_local_height = getattr(self.threadlocal_cache, 'local_height', None) + if cached_local_height is not None: + return cached_local_height + return self.network.get_local_height() if self.network else self.storage.get('stored_height', 0) + + def get_tx_height(self, tx_hash: str) -> TxMinedStatus: + """ Given a transaction, returns (height, conf, timestamp, header_hash) """ + with self.lock: + if tx_hash in self.verified_tx: + info = self.verified_tx[tx_hash] + conf = max(self.get_local_height() - info.height + 1, 0) + return TxMinedStatus(info.height, conf, info.timestamp, info.header_hash) + elif tx_hash in self.unverified_tx: + height = self.unverified_tx[tx_hash] + return TxMinedStatus(height, 0, None, None) + else: + # local transaction + return TxMinedStatus(TX_HEIGHT_LOCAL, 0, None, None) + + def set_up_to_date(self, up_to_date): + with self.lock: + self.up_to_date = up_to_date + if up_to_date: + self.save_transactions(write=True) + # if the verifier is also up to date, persist that too; + # otherwise it will persist its results when it finishes + if self.verifier and self.verifier.is_up_to_date(): + self.save_verified_tx(write=True) + + def is_up_to_date(self): + with self.lock: return self.up_to_date + + def get_num_tx(self, address): + """ return number of transactions where address is involved """ + return len(self.history.get(address, [])) + + def get_tx_delta(self, tx_hash, address): + "effect of tx on address" + delta = 0 + # substract the value of coins sent from address + d = self.txi.get(tx_hash, {}).get(address, []) + for n, v in d: + delta -= v + # add the value of the coins received at address + d = self.txo.get(tx_hash, {}).get(address, []) + for n, v, cb in d: + delta += v + return delta + + def get_tx_value(self, txid): + " effect of tx on the entire domain" + delta = 0 + for addr, d in self.txi.get(txid, {}).items(): + for n, v in d: + delta -= v + for addr, d in self.txo.get(txid, {}).items(): + for n, v, cb in d: + delta += v + return delta + + def get_wallet_delta(self, tx): + """ effect of tx on wallet """ + is_relevant = False # "related to wallet?" + is_mine = False + is_pruned = False + is_partial = False + v_in = v_out = v_out_mine = 0 + for txin in tx.inputs(): + addr = self.get_txin_address(txin) + if self.is_mine(addr): + is_mine = True + is_relevant = True + d = self.txo.get(txin['prevout_hash'], {}).get(addr, []) + for n, v, cb in d: + if n == txin['prevout_n']: + value = v + break + else: + value = None + if value is None: + is_pruned = True + else: + v_in += value + else: + is_partial = True + if not is_mine: + is_partial = False + for addr, value in tx.get_outputs(): + v_out += value + if self.is_mine(addr): + v_out_mine += value + is_relevant = True + if is_pruned: + # some inputs are mine: + fee = None + if is_mine: + v = v_out_mine - v_out + else: + # no input is mine + v = v_out_mine + else: + v = v_out_mine - v_in + if is_partial: + # some inputs are mine, but not all + fee = None + else: + # all inputs are mine + fee = v_in - v_out + if not is_mine: + fee = None + return is_relevant, is_mine, v, fee + + def get_addr_io(self, address): + h = self.get_address_history(address) + received = {} + sent = {} + for tx_hash, height in h: + l = self.txo.get(tx_hash, {}).get(address, []) + for n, v, is_cb in l: + received[tx_hash + ':%d'%n] = (height, v, is_cb) + for tx_hash, height in h: + l = self.txi.get(tx_hash, {}).get(address, []) + for txi, v in l: + sent[txi] = height + return received, sent + + def get_addr_utxo(self, address): + coins, spent = self.get_addr_io(address) + for txi in spent: + coins.pop(txi) + out = {} + for txo, v in coins.items(): + tx_height, value, is_cb = v + prevout_hash, prevout_n = txo.split(':') + x = { + 'address':address, + 'value':value, + 'prevout_n':int(prevout_n), + 'prevout_hash':prevout_hash, + 'height':tx_height, + 'coinbase':is_cb + } + out[txo] = x + return out + + # return the total amount ever received by an address + def get_addr_received(self, address): + received, sent = self.get_addr_io(address) + return sum([v for height, v, is_cb in received.values()]) + + @with_local_height_cached + def get_addr_balance(self, address): + """Return the balance of a bitcoin address: + confirmed and matured, unconfirmed, unmatured + """ + received, sent = self.get_addr_io(address) + c = u = x = 0 + local_height = self.get_local_height() + for txo, (tx_height, v, is_cb) in received.items(): + if is_cb and tx_height + COINBASE_MATURITY > local_height: + x += v + elif tx_height > 0: + c += v + else: + u += v + if txo in sent: + if sent[txo] > 0: + c -= v + else: + u -= v + return c, u, x + + @with_local_height_cached + def get_utxos(self, domain=None, excluded=None, mature=False, confirmed_only=False): + coins = [] + if domain is None: + domain = self.get_addresses() + domain = set(domain) + if excluded: + domain = set(domain) - excluded + for addr in domain: + utxos = self.get_addr_utxo(addr) + for x in utxos.values(): + if confirmed_only and x['height'] <= 0: + continue + if mature and x['coinbase'] and x['height'] + COINBASE_MATURITY > self.get_local_height(): + continue + coins.append(x) + continue + return coins + + def get_balance(self, domain=None): + if domain is None: + domain = self.get_addresses() + domain = set(domain) + cc = uu = xx = 0 + for addr in domain: + c, u, x = self.get_addr_balance(addr) + cc += c + uu += u + xx += x + return cc, uu, xx + + def is_used(self, address): + h = self.history.get(address,[]) + return len(h) != 0 + + def is_empty(self, address): + c, u, x = self.get_addr_balance(address) + return c+u+x == 0 diff --git a/electrum/base_wizard.py b/electrum/base_wizard.py index fd3c8cc3..8e130a7a 100644 --- a/electrum/base_wizard.py +++ b/electrum/base_wizard.py @@ -34,7 +34,7 @@ from .keystore import bip44_derivation, purpose48_derivation from .wallet import Imported_Wallet, Standard_Wallet, Multisig_Wallet, wallet_types, Wallet from .storage import STO_EV_USER_PW, STO_EV_XPUB_PW, get_derivation_used_for_hw_device_encryption from .i18n import _ -from .util import UserCancelled, InvalidPassword +from .util import UserCancelled, InvalidPassword, WalletFileException # hardware device setup purpose HWD_SETUP_NEW_WALLET, HWD_SETUP_DECRYPT_WALLET = range(0, 2) @@ -106,10 +106,20 @@ class BaseWizard(object): self.choice_dialog(title=title, message=message, choices=choices, run_next=self.on_wallet_type) def upgrade_storage(self): + exc = None def on_finished(): - self.wallet = Wallet(self.storage) - self.terminate() - self.waiting_dialog(partial(self.storage.upgrade), _('Upgrading wallet format...'), on_finished=on_finished) + if exc is None: + self.wallet = Wallet(self.storage) + self.terminate() + else: + raise exc + def do_upgrade(): + nonlocal exc + try: + self.storage.upgrade() + except Exception as e: + exc = e + self.waiting_dialog(do_upgrade, _('Upgrading wallet format...'), on_finished=on_finished) def load_2fa(self): self.storage.put('wallet_type', '2fa') @@ -348,7 +358,7 @@ class BaseWizard(object): k = hardware_keystore(d) self.on_keystore(k) - def passphrase_dialog(self, run_next): + def passphrase_dialog(self, run_next, is_restoring=False): title = _('Seed extension') message = '\n'.join([ _('You may extend your seed with custom words.'), @@ -358,7 +368,10 @@ class BaseWizard(object): _('Note that this is NOT your encryption password.'), _('If you do not know what this is, leave this field empty.'), ]) - self.line_dialog(title=title, message=message, warning=warning, default='', test=lambda x:True, run_next=run_next) + warn_issue4566 = is_restoring and self.seed_type == 'bip39' + self.line_dialog(title=title, message=message, warning=warning, + default='', test=lambda x:True, run_next=run_next, + warn_issue4566=warn_issue4566) def restore_from_seed(self): self.opt_bip39 = True @@ -371,10 +384,10 @@ class BaseWizard(object): self.seed_type = 'bip39' if is_bip39 else bitcoin.seed_type(seed) if self.seed_type == 'bip39': f = lambda passphrase: self.on_restore_bip39(seed, passphrase) - self.passphrase_dialog(run_next=f) if is_ext else f('') + self.passphrase_dialog(run_next=f, is_restoring=True) if is_ext else f('') elif self.seed_type in ['standard', 'segwit']: f = lambda passphrase: self.run('create_keystore', seed, passphrase) - self.passphrase_dialog(run_next=f) if is_ext else f('') + self.passphrase_dialog(run_next=f, is_restoring=True) if is_ext else f('') elif self.seed_type == 'old': self.run('create_keystore', seed, '') elif self.seed_type == '2fa': diff --git a/electrum/blockchain.py b/electrum/blockchain.py index 3aaa8067..c4c55782 100644 --- a/electrum/blockchain.py +++ b/electrum/blockchain.py @@ -79,12 +79,12 @@ def read_blockchains(config): l = filter(lambda x: x.startswith('fork_'), os.listdir(fdir)) l = sorted(l, key = lambda x: int(x.split('_')[1])) for filename in l: - checkpoint = int(filename.split('_')[2]) + forkpoint = int(filename.split('_')[2]) parent_id = int(filename.split('_')[1]) - b = Blockchain(config, checkpoint, parent_id) - h = b.read_header(b.checkpoint) + b = Blockchain(config, forkpoint, parent_id) + h = b.read_header(b.forkpoint) if b.parent().can_connect(h, check_height=False): - blockchains[b.checkpoint] = b + blockchains[b.forkpoint] = b else: util.print_error("cannot connect", filename) return blockchains @@ -109,32 +109,39 @@ class Blockchain(util.PrintError): Manages blockchain headers and their verification """ - def __init__(self, config, checkpoint, parent_id): + def __init__(self, config, forkpoint, parent_id): self.config = config - self.catch_up = None # interface catching up - self.checkpoint = checkpoint + self.catch_up = None # interface catching up + self.forkpoint = forkpoint self.checkpoints = constants.net.CHECKPOINTS self.parent_id = parent_id - self.lock = threading.Lock() + assert parent_id != forkpoint + self.lock = threading.RLock() with self.lock: self.update_size() + def with_lock(func): + def func_wrapper(self, *args, **kwargs): + with self.lock: + return func(self, *args, **kwargs) + return func_wrapper + def parent(self): return blockchains[self.parent_id] def get_max_child(self): - children = list(filter(lambda y: y.parent_id==self.checkpoint, blockchains.values())) - return max([x.checkpoint for x in children]) if children else None + children = list(filter(lambda y: y.parent_id==self.forkpoint, blockchains.values())) + return max([x.forkpoint for x in children]) if children else None - def get_checkpoint(self): + def get_forkpoint(self): mc = self.get_max_child() - return mc if mc is not None else self.checkpoint + return mc if mc is not None else self.forkpoint def get_branch_size(self): - return self.height() - self.get_checkpoint() + 1 + return self.height() - self.get_forkpoint() + 1 def get_name(self): - return self.get_hash(self.get_checkpoint()).lstrip('00')[0:10] + return self.get_hash(self.get_forkpoint()).lstrip('00')[0:10] def check_header(self, header): header_hash = hash_header(header) @@ -142,14 +149,14 @@ class Blockchain(util.PrintError): return header_hash == self.get_hash(height) def fork(parent, header): - checkpoint = header.get('block_height') - self = Blockchain(parent.config, checkpoint, parent.checkpoint) + forkpoint = header.get('block_height') + self = Blockchain(parent.config, forkpoint, parent.forkpoint) open(self.path(), 'w+').close() self.save_header(header) return self def height(self): - return self.checkpoint + self.size() - 1 + return self.forkpoint + self.size() - 1 def size(self): with self.lock: @@ -183,44 +190,55 @@ class Blockchain(util.PrintError): def path(self): d = util.get_headers_dir(self.config) - filename = 'blockchain_headers' if self.parent_id is None else os.path.join('forks', 'fork_%d_%d'%(self.parent_id, self.checkpoint)) + filename = 'blockchain_headers' if self.parent_id is None else os.path.join('forks', 'fork_%d_%d'%(self.parent_id, self.forkpoint)) return os.path.join(d, filename) + @with_lock def save_chunk(self, index, chunk): - filename = self.path() - d = (index * 2016 - self.checkpoint) * 80 - if d < 0: - chunk = chunk[-d:] - d = 0 - truncate = index >= len(self.checkpoints) - self.write(chunk, d, truncate) + chunk_within_checkpoint_region = index < len(self.checkpoints) + # chunks in checkpoint region are the responsibility of the 'main chain' + if chunk_within_checkpoint_region and self.parent_id is not None: + main_chain = blockchains[0] + main_chain.save_chunk(index, chunk) + return + + delta_height = (index * 2016 - self.forkpoint) + delta_bytes = delta_height * 80 + # if this chunk contains our forkpoint, only save the part after forkpoint + # (the part before is the responsibility of the parent) + if delta_bytes < 0: + chunk = chunk[-delta_bytes:] + delta_bytes = 0 + truncate = not chunk_within_checkpoint_region + self.write(chunk, delta_bytes, truncate) self.swap_with_parent() + @with_lock def swap_with_parent(self): if self.parent_id is None: return - parent_branch_size = self.parent().height() - self.checkpoint + 1 + parent_branch_size = self.parent().height() - self.forkpoint + 1 if parent_branch_size >= self.size(): return - self.print_error("swap", self.checkpoint, self.parent_id) + self.print_error("swap", self.forkpoint, self.parent_id) parent_id = self.parent_id - checkpoint = self.checkpoint + forkpoint = self.forkpoint parent = self.parent() self.assert_headers_file_available(self.path()) with open(self.path(), 'rb') as f: my_data = f.read() self.assert_headers_file_available(parent.path()) with open(parent.path(), 'rb') as f: - f.seek((checkpoint - parent.checkpoint)*80) + f.seek((forkpoint - parent.forkpoint)*80) parent_data = f.read(parent_branch_size*80) self.write(parent_data, 0) - parent.write(my_data, (checkpoint - parent.checkpoint)*80) + parent.write(my_data, (forkpoint - parent.forkpoint)*80) # store file path for b in blockchains.values(): b.old_path = b.path() # swap parameters self.parent_id = parent.parent_id; parent.parent_id = parent_id - self.checkpoint = parent.checkpoint; parent.checkpoint = checkpoint + self.forkpoint = parent.forkpoint; parent.forkpoint = forkpoint self._size = parent._size; parent._size = parent_branch_size # move files for b in blockchains.values(): @@ -229,8 +247,8 @@ class Blockchain(util.PrintError): self.print_error("renaming", b.old_path, b.path()) os.rename(b.old_path, b.path()) # update pointers - blockchains[self.checkpoint] = self - blockchains[parent.checkpoint] = parent + blockchains[self.forkpoint] = self + blockchains[parent.forkpoint] = parent def assert_headers_file_available(self, path): if os.path.exists(path): @@ -254,23 +272,25 @@ class Blockchain(util.PrintError): os.fsync(f.fileno()) self.update_size() + @with_lock def save_header(self, header): - delta = header.get('block_height') - self.checkpoint + delta = header.get('block_height') - self.forkpoint data = bfh(serialize_header(header)) + # headers are only _appended_ to the end: assert delta == self.size() assert len(data) == 80 self.write(data, delta*80) self.swap_with_parent() def read_header(self, height): - assert self.parent_id != self.checkpoint + assert self.parent_id != self.forkpoint if height < 0: return - if height < self.checkpoint: + if height < self.forkpoint: return self.parent().read_header(height) if height > self.height(): return - delta = height - self.checkpoint + delta = height - self.forkpoint name = self.path() self.assert_headers_file_available(name) with open(name, 'rb') as f: diff --git a/electrum/coinchooser.py b/electrum/coinchooser.py index 0b8128b9..7dc355d5 100644 --- a/electrum/coinchooser.py +++ b/electrum/coinchooser.py @@ -26,7 +26,7 @@ from collections import defaultdict, namedtuple from math import floor, log10 from .bitcoin import sha256, COIN, TYPE_ADDRESS, is_address -from .transaction import Transaction +from .transaction import Transaction, TxOutput from .util import NotEnoughFunds, PrintError @@ -178,7 +178,7 @@ class CoinChooserBase(PrintError): # size of the change output, add it to the transaction. dust = sum(amount for amount in amounts if amount < dust_threshold) amounts = [amount for amount in amounts if amount >= dust_threshold] - change = [(TYPE_ADDRESS, addr, amount) + change = [TxOutput(TYPE_ADDRESS, addr, amount) for addr, amount in zip(change_addrs, amounts)] self.print_error('change:', change) if dust: diff --git a/electrum/commands.py b/electrum/commands.py index a95d74ed..2240f57c 100644 --- a/electrum/commands.py +++ b/electrum/commands.py @@ -38,7 +38,7 @@ from .util import bfh, bh2u, format_satoshis, json_decode, print_error, json_enc from . import bitcoin from .bitcoin import is_address, hash_160, COIN, TYPE_ADDRESS from .i18n import _ -from .transaction import Transaction, multisig_script +from .transaction import Transaction, multisig_script, TxOutput from .paymentrequest import PR_PAID, PR_UNPAID, PR_UNKNOWN, PR_EXPIRED from .plugin import run_hook @@ -226,7 +226,7 @@ class Commands: txin['signatures'] = [None] txin['num_sig'] = 1 - outputs = [(TYPE_ADDRESS, x['address'], int(x['value'])) for x in outputs] + outputs = [TxOutput(TYPE_ADDRESS, x['address'], int(x['value'])) for x in outputs] tx = Transaction.from_io(inputs, outputs, locktime=locktime) tx.sign(keypairs) return tx.as_dict() @@ -415,7 +415,7 @@ class Commands: for address, amount in outputs: address = self._resolver(address) amount = satoshis(amount) - final_outputs.append((TYPE_ADDRESS, address, amount)) + final_outputs.append(TxOutput(TYPE_ADDRESS, address, amount)) coins = self.wallet.get_spendable_coins(domain, self.config) tx = self.wallet.make_unsigned_transaction(coins, final_outputs, self.config, fee, change_addr) @@ -826,10 +826,13 @@ def add_network_options(parser): parser.add_argument("-1", "--oneserver", action="store_true", dest="oneserver", default=None, help="connect to one server only") parser.add_argument("-s", "--server", dest="server", default=None, help="set server host:port:protocol, where protocol is either t (tcp) or s (ssl)") parser.add_argument("-p", "--proxy", dest="proxy", default=None, help="set proxy [type:]host[:port], where type is socks4,socks5 or http") + parser.add_argument("--noonion", action="store_true", dest="noonion", default=None, help="do not try to connect to onion servers") def add_global_options(parser): group = parser.add_argument_group('global options') - group.add_argument("-v", "--verbose", action="store_true", dest="verbose", default=False, help="Show debugging information") + # const is for when no argument is given to verbosity + # default is for when the flag is missing + group.add_argument("-v", "--verbosity", dest="verbosity", help="Set verbosity filter", default='', const='*', nargs='?') group.add_argument("-D", "--dir", dest="electrum_path", help="electrum directory") group.add_argument("-P", "--portable", action="store_true", dest="portable", default=False, help="Use local 'electrum_data' directory") group.add_argument("-w", "--wallet", dest="wallet_path", help="wallet path") diff --git a/electrum/gui/kivy/main_window.py b/electrum/gui/kivy/main_window.py index 5414543f..ba8e7da2 100644 --- a/electrum/gui/kivy/main_window.py +++ b/electrum/gui/kivy/main_window.py @@ -10,12 +10,12 @@ import threading from electrum.bitcoin import TYPE_ADDRESS from electrum.storage import WalletStorage from electrum.wallet import Wallet -from electrum.i18n import _ from electrum.paymentrequest import InvoiceStore from electrum.util import profiler, InvalidPassword from electrum.plugin import run_hook from electrum.util import format_satoshis, format_satoshis_plain from electrum.paymentrequest import PR_UNPAID, PR_PAID, PR_UNKNOWN, PR_EXPIRED +from .i18n import _ from kivy.app import App from kivy.core.window import Window @@ -88,7 +88,7 @@ class ElectrumWindow(App): balance = StringProperty('') fiat_balance = StringProperty('') is_fiat = BooleanProperty(False) - blockchain_checkpoint = NumericProperty(0) + blockchain_forkpoint = NumericProperty(0) auto_connect = BooleanProperty(False) def on_auto_connect(self, instance, x): @@ -159,6 +159,9 @@ class ElectrumWindow(App): Logger.info("on_history") self._trigger_update_history() + def on_fee_histogram(self, *args): + self._trigger_update_history() + def _get_bu(self): decimal_point = self.electrum_config.get('decimal_point', 5) return decimal_point_to_base_unit_name(decimal_point) @@ -391,9 +394,16 @@ class ElectrumWindow(App): popup.export = self.export_private_keys popup.open() - def qr_dialog(self, title, data, show_text=False): + def qr_dialog(self, title, data, show_text=False, text_for_clipboard=None): from .uix.dialogs.qr_dialog import QRDialog - popup = QRDialog(title, data, show_text) + def on_qr_failure(): + popup.dismiss() + msg = _('Failed to display QR code.') + if text_for_clipboard: + msg += '\n' + _('Text copied to clipboard.') + self._clipboard.copy(text_for_clipboard) + Clock.schedule_once(lambda dt: self.show_info(msg)) + popup = QRDialog(title, data, show_text, on_qr_failure) popup.open() def scan_qr(self, on_complete): @@ -476,6 +486,7 @@ class ElectrumWindow(App): interests = ['updated', 'status', 'new_transaction', 'verified', 'interfaces'] self.network.register_callback(self.on_network_event, interests) self.network.register_callback(self.on_fee, ['fee']) + self.network.register_callback(self.on_fee_histogram, ['fee_histogram']) self.network.register_callback(self.on_quotes, ['on_quotes']) self.network.register_callback(self.on_history, ['on_history']) # load wallet @@ -643,7 +654,7 @@ class ElectrumWindow(App): self.num_nodes = len(self.network.get_interfaces()) self.num_chains = len(self.network.get_blockchains()) chain = self.network.blockchain() - self.blockchain_checkpoint = chain.get_checkpoint() + self.blockchain_forkpoint = chain.get_forkpoint() self.blockchain_name = chain.get_name() interface = self.network.interface if interface: @@ -702,13 +713,14 @@ class ElectrumWindow(App): self.fiat_balance = self.fx.format_amount(c+u+x) + ' [size=22dp]%s[/size]'% self.fx.ccy def get_max_amount(self): + from electrum.transaction import TxOutput if run_hook('abort_send', self): return '' inputs = self.wallet.get_spendable_coins(None, self.electrum_config) if not inputs: return '' addr = str(self.send_screen.screen.address) or self.wallet.dummy_address() - outputs = [(TYPE_ADDRESS, addr, '!')] + outputs = [TxOutput(TYPE_ADDRESS, addr, '!')] try: tx = self.wallet.make_unsigned_transaction(inputs, outputs, self.electrum_config) except NoDynamicFeeEstimates as e: @@ -875,6 +887,7 @@ class ElectrumWindow(App): self.wallet.invoices.save() self.update_tab('invoices') else: + msg = msg[:500] if msg else _('There was an error broadcasting the transaction.') self.show_error(msg) if self.network and self.network.is_connected(): diff --git a/electrum/gui/kivy/uix/dialogs/__init__.py b/electrum/gui/kivy/uix/dialogs/__init__.py index 9f8f7f2e..352675a3 100644 --- a/electrum/gui/kivy/uix/dialogs/__init__.py +++ b/electrum/gui/kivy/uix/dialogs/__init__.py @@ -206,9 +206,9 @@ class OutputList(RecycleView): def update(self, outputs): res = [] - for (type, address, amount) in outputs: - value = self.app.format_amount_and_units(amount) - res.append({'address': address, 'value': value}) + for o in outputs: + value = self.app.format_amount_and_units(o.value) + res.append({'address': o.address, 'value': value}) self.data = res diff --git a/electrum/gui/kivy/uix/dialogs/addresses.py b/electrum/gui/kivy/uix/dialogs/addresses.py index 5b90f2f7..b1fc40cd 100644 --- a/electrum/gui/kivy/uix/dialogs/addresses.py +++ b/electrum/gui/kivy/uix/dialogs/addresses.py @@ -136,14 +136,14 @@ class AddressesDialog(Factory.Popup): for address in _list: label = wallet.labels.get(address, '') balance = sum(wallet.get_addr_balance(address)) - is_used = wallet.is_used(address) - if self.show_used == 1 and (balance or is_used): + is_used_and_empty = wallet.is_used(address) and balance == 0 + if self.show_used == 1 and (balance or is_used_and_empty): continue if self.show_used == 2 and balance == 0: continue - if self.show_used == 3 and not is_used: + if self.show_used == 3 and not is_used_and_empty: continue - card = self.get_card(address, balance, is_used, label) + card = self.get_card(address, balance, is_used_and_empty, label) if search and not self.ext_search(card, search): continue cards.append(card) diff --git a/electrum/gui/kivy/uix/dialogs/crash_reporter.py b/electrum/gui/kivy/uix/dialogs/crash_reporter.py index 8603d806..04582b95 100644 --- a/electrum/gui/kivy/uix/dialogs/crash_reporter.py +++ b/electrum/gui/kivy/uix/dialogs/crash_reporter.py @@ -9,9 +9,9 @@ from kivy.lang import Builder from kivy.uix.label import Label from kivy.utils import platform +from electrum.gui.kivy.i18n import _ from electrum.base_crash_reporter import BaseCrashReporter -from electrum.i18n import _ Builder.load_string(''' diff --git a/electrum/gui/kivy/uix/dialogs/installwizard.py b/electrum/gui/kivy/uix/dialogs/installwizard.py index d5b51e07..04af3734 100644 --- a/electrum/gui/kivy/uix/dialogs/installwizard.py +++ b/electrum/gui/kivy/uix/dialogs/installwizard.py @@ -957,7 +957,12 @@ class InstallWizard(BaseWizard, Widget): # on completion hide message Clock.schedule_once(lambda dt: app.info_bubble.hide(now=True), -1) if on_finished: - Clock.schedule_once(lambda dt: on_finished(), -1) + def protected_on_finished(): + try: + on_finished() + except Exception as e: + self.show_error(str(e)) + Clock.schedule_once(lambda dt: protected_on_finished(), -1) app = App.get_running_app() app.show_info_bubble( diff --git a/electrum/gui/kivy/uix/dialogs/qr_dialog.py b/electrum/gui/kivy/uix/dialogs/qr_dialog.py index abc5a6de..b12eb6ce 100644 --- a/electrum/gui/kivy/uix/dialogs/qr_dialog.py +++ b/electrum/gui/kivy/uix/dialogs/qr_dialog.py @@ -36,11 +36,12 @@ Builder.load_string(''' ''') class QRDialog(Factory.Popup): - def __init__(self, title, data, show_text): + def __init__(self, title, data, show_text, failure_cb=None): Factory.Popup.__init__(self) self.title = title self.data = data self.show_text = show_text + self.failure_cb = failure_cb def on_open(self): - self.ids.qr.set_data(self.data) + self.ids.qr.set_data(self.data, self.failure_cb) diff --git a/electrum/gui/kivy/uix/dialogs/tx_dialog.py b/electrum/gui/kivy/uix/dialogs/tx_dialog.py index 5a179fe0..a99acb94 100644 --- a/electrum/gui/kivy/uix/dialogs/tx_dialog.py +++ b/electrum/gui/kivy/uix/dialogs/tx_dialog.py @@ -179,6 +179,7 @@ class TxDialog(Factory.Popup): def show_qr(self): from electrum.bitcoin import base_encode, bfh - text = bfh(str(self.tx)) + raw_tx = str(self.tx) + text = bfh(raw_tx) text = base_encode(text, base=43) - self.app.qr_dialog(_("Raw Transaction"), text) + self.app.qr_dialog(_("Raw Transaction"), text, text_for_clipboard=raw_tx) diff --git a/electrum/gui/kivy/uix/qrcodewidget.py b/electrum/gui/kivy/uix/qrcodewidget.py index 1989c5a0..56e20cdd 100644 --- a/electrum/gui/kivy/uix/qrcodewidget.py +++ b/electrum/gui/kivy/uix/qrcodewidget.py @@ -5,6 +5,7 @@ from threading import Thread from functools import partial import qrcode +from qrcode import exceptions from kivy.uix.floatlayout import FloatLayout from kivy.graphics.texture import Texture @@ -50,15 +51,23 @@ class QRCodeWidget(FloatLayout): self.data = None self.qr = None self._qrtexture = None + self.failure_cb = None def on_data(self, instance, value): if not (self.canvas or value): return - self.update_qr() + try: + self.update_qr() + except qrcode.exceptions.DataOverflowError: + if self.failure_cb: + self.failure_cb() + else: + raise - def set_data(self, data): + def set_data(self, data, failure_cb=None): if self.data == data: return + self.failure_cb = failure_cb MinSize = 210 if len(data) < 128 else 500 self.setMinimumSize((MinSize, MinSize)) self.data = data diff --git a/electrum/gui/kivy/uix/screens.py b/electrum/gui/kivy/uix/screens.py index 94807913..73930aef 100644 --- a/electrum/gui/kivy/uix/screens.py +++ b/electrum/gui/kivy/uix/screens.py @@ -20,6 +20,7 @@ from kivy.utils import platform from electrum.util import profiler, parse_URI, format_time, InvalidPassword, NotEnoughFunds, Fiat from electrum import bitcoin +from electrum.transaction import TxOutput from electrum.util import timestamp_to_datetime from electrum.paymentrequest import PR_UNPAID, PR_PAID, PR_UNKNOWN, PR_EXPIRED from electrum.plugin import run_hook @@ -131,8 +132,8 @@ class HistoryScreen(CScreen): d = LabelDialog(_('Enter Transaction Label'), text, callback) d.open() - def get_card(self, tx_hash, height, conf, timestamp, value, balance): - status, status_str = self.app.wallet.get_tx_status(tx_hash, height, conf, timestamp) + def get_card(self, tx_hash, tx_mined_status, value, balance): + status, status_str = self.app.wallet.get_tx_status(tx_hash, tx_mined_status) icon = "atlas://electrum/gui/kivy/theming/light/" + TX_ICONS[status] label = self.app.wallet.get_label(tx_hash) if tx_hash else _('Pruned transaction outputs') ri = {} @@ -141,7 +142,7 @@ class HistoryScreen(CScreen): ri['icon'] = icon ri['date'] = status_str ri['message'] = label - ri['confirmations'] = conf + ri['confirmations'] = tx_mined_status.conf if value is not None: ri['is_mine'] = value < 0 if value < 0: value = - value @@ -158,7 +159,6 @@ class HistoryScreen(CScreen): return history = reversed(self.app.wallet.get_history()) history_card = self.screen.ids.history_container - count = 0 history_card.data = [self.get_card(*item) for item in history] @@ -257,7 +257,7 @@ class SendScreen(CScreen): except: self.app.show_error(_('Invalid amount') + ':\n' + self.screen.amount) return - outputs = [(bitcoin.TYPE_ADDRESS, address, amount)] + outputs = [TxOutput(bitcoin.TYPE_ADDRESS, address, amount)] message = self.screen.message amount = sum(map(lambda x:x[2], outputs)) if self.app.electrum_config.get('use_rbf'): diff --git a/electrum/gui/kivy/uix/ui_screens/network.kv b/electrum/gui/kivy/uix/ui_screens/network.kv index f499618a..99fb8366 100644 --- a/electrum/gui/kivy/uix/ui_screens/network.kv +++ b/electrum/gui/kivy/uix/ui_screens/network.kv @@ -46,7 +46,7 @@ Popup: CardSeparator SettingsItem: - title: _('Fork detected at block {}').format(app.blockchain_checkpoint) if app.num_chains>1 else _('No fork detected') + title: _('Fork detected at block {}').format(app.blockchain_forkpoint) if app.num_chains>1 else _('No fork detected') fork_description: (_('You are following branch') if app.auto_connect else _("Your server is on branch")) + ' ' + app.blockchain_name description: self.fork_description if app.num_chains>1 else _('Connected nodes are on the same chain') action: app.choose_blockchain_dialog diff --git a/electrum/gui/qt/address_list.py b/electrum/gui/qt/address_list.py index 7217940f..fce1004c 100644 --- a/electrum/gui/qt/address_list.py +++ b/electrum/gui/qt/address_list.py @@ -93,20 +93,20 @@ class AddressList(MyTreeWidget): else: addr_list = self.wallet.get_addresses() self.clear() + fx = self.parent.fx for address in addr_list: - num = len(self.wallet.get_address_history(address)) - is_used = self.wallet.is_used(address) + num = self.wallet.get_address_history_len(address) label = self.wallet.labels.get(address, '') c, u, x = self.wallet.get_addr_balance(address) balance = c + u + x - if self.show_used == 1 and (balance or is_used): + is_used_and_empty = self.wallet.is_used(address) and balance == 0 + if self.show_used == 1 and (balance or is_used_and_empty): continue if self.show_used == 2 and balance == 0: continue - if self.show_used == 3 and not is_used: + if self.show_used == 3 and not is_used_and_empty: continue balance_text = self.parent.format_amount(balance, whitespaces=True) - fx = self.parent.fx # create item if fx and fx.get_fiat_address_config(): rate = fx.exchange_rate() diff --git a/electrum/gui/qt/history_list.py b/electrum/gui/qt/history_list.py index 83ab3fdb..79fe4d3a 100644 --- a/electrum/gui/qt/history_list.py +++ b/electrum/gui/qt/history_list.py @@ -26,10 +26,10 @@ import webbrowser import datetime -from electrum.wallet import AddTransactionException, TX_HEIGHT_LOCAL +from electrum.address_synchronizer import TX_HEIGHT_LOCAL from .util import * from electrum.i18n import _ -from electrum.util import block_explorer_URL, profiler, print_error +from electrum.util import block_explorer_URL, profiler, print_error, TxMinedStatus try: from electrum.plot import plot_history, NothingToPlotException @@ -229,6 +229,9 @@ class HistoryList(MyTreeWidget, AcceptFileDragDrop): current_tx = item.data(0, Qt.UserRole) if item else None self.clear() if fx: fx.history_used_spot = False + blue_brush = QBrush(QColor("#1E1EFF")) + red_brush = QBrush(QColor("#BC1E1E")) + monospace_font = QFont(MONOSPACE_FONT) for tx_item in self.transactions: tx_hash = tx_item['txid'] height = tx_item['height'] @@ -237,7 +240,8 @@ class HistoryList(MyTreeWidget, AcceptFileDragDrop): value = tx_item['value'].value balance = tx_item['balance'].value label = tx_item['label'] - status, status_str = self.wallet.get_tx_status(tx_hash, height, conf, timestamp) + tx_mined_status = TxMinedStatus(height, conf, timestamp, None) + status, status_str = self.wallet.get_tx_status(tx_hash, tx_mined_status) has_invoice = self.wallet.invoices.paid.get(tx_hash) icon = self.icon_cache.get(":icons/" + TX_ICONS[status]) v_str = self.parent.format_amount(value, is_diff=True, whitespaces=True) @@ -262,12 +266,12 @@ class HistoryList(MyTreeWidget, AcceptFileDragDrop): if i>3: item.setTextAlignment(i, Qt.AlignRight | Qt.AlignVCenter) if i!=2: - item.setFont(i, QFont(MONOSPACE_FONT)) + item.setFont(i, monospace_font) if value and value < 0: - item.setForeground(3, QBrush(QColor("#BC1E1E"))) - item.setForeground(4, QBrush(QColor("#BC1E1E"))) + item.setForeground(3, red_brush) + item.setForeground(4, red_brush) if fiat_value and not tx_item['fiat_default']: - item.setForeground(6, QBrush(QColor("#1E1EFF"))) + item.setForeground(6, blue_brush) if tx_hash: item.setData(0, Qt.UserRole, tx_hash) self.insertTopLevelItem(0, item) @@ -304,10 +308,11 @@ class HistoryList(MyTreeWidget, AcceptFileDragDrop): label = self.wallet.get_label(txid) item.setText(3, label) - def update_item(self, tx_hash, height, conf, timestamp): + def update_item(self, tx_hash, tx_mined_status): if self.wallet is None: return - status, status_str = self.wallet.get_tx_status(tx_hash, height, conf, timestamp) + conf = tx_mined_status.conf + status, status_str = self.wallet.get_tx_status(tx_hash, tx_mined_status) icon = self.icon_cache.get(":icons/" + TX_ICONS[status]) items = self.findItems(tx_hash, Qt.UserRole|Qt.MatchContains|Qt.MatchRecursive, column=1) if items: @@ -332,7 +337,7 @@ class HistoryList(MyTreeWidget, AcceptFileDragDrop): column_title = self.headerItem().text(column) column_data = item.text(column) tx_URL = block_explorer_URL(self.config, 'tx', tx_hash) - height, conf, timestamp = self.wallet.get_tx_height(tx_hash) + height = self.wallet.get_tx_height(tx_hash).height tx = self.wallet.transactions.get(tx_hash) is_relevant, is_mine, v, fee = self.wallet.get_wallet_delta(tx) is_unconfirmed = height <= 0 diff --git a/electrum/gui/qt/installwizard.py b/electrum/gui/qt/installwizard.py index 79647235..cd45e6f3 100644 --- a/electrum/gui/qt/installwizard.py +++ b/electrum/gui/qt/installwizard.py @@ -32,6 +32,12 @@ WIF_HELP_TEXT = (_('WIF keys are typed in Electrum, based on script type.') + '\ 'p2wpkh-p2sh:KxZcY47uGp9a... \t-> 3NhNeZQXF...\n' + 'p2wpkh:KxZcY47uGp9a... \t-> bc1q3fjfk...') # note: full key is KxZcY47uGp9aVQAb6VVvuBs8SwHKgkSR2DbZUzjDzXf2N2GPhG9n +MSG_PASSPHRASE_WARN_ISSUE4566 = _("Warning") + ": "\ + + _("You have multiple consecutive whitespaces or leading/trailing " + "whitespaces in your passphrase.") + " " \ + + _("This is discouraged.") + " " \ + + _("Due to a bug, old versions of Electrum will NOT be creating the " + "same wallet as newer versions or other software.") class CosignWidget(QWidget): @@ -550,17 +556,24 @@ class InstallWizard(QDialog, MessageBoxMixin, BaseWizard): @wizard_dialog def line_dialog(self, run_next, title, message, default, test, warning='', - presets=()): + presets=(), warn_issue4566=False): vbox = QVBoxLayout() vbox.addWidget(WWLabel(message)) line = QLineEdit() line.setText(default) def f(text): self.next_button.setEnabled(test(text)) + if warn_issue4566: + text_whitespace_normalised = ' '.join(text.split()) + warn_issue4566_label.setVisible(text != text_whitespace_normalised) line.textEdited.connect(f) vbox.addWidget(line) vbox.addWidget(WWLabel(warning)) + warn_issue4566_label = WWLabel(MSG_PASSPHRASE_WARN_ISSUE4566) + warn_issue4566_label.setVisible(False) + vbox.addWidget(warn_issue4566_label) + for preset in presets: button = QPushButton(preset[0]) button.clicked.connect(lambda __, text=preset[1]: line.setText(text)) @@ -570,7 +583,7 @@ class InstallWizard(QDialog, MessageBoxMixin, BaseWizard): vbox.addLayout(hbox) self.exec_layout(vbox, title, next_enabled=test(default)) - return ' '.join(line.text().split()) + return line.text() @wizard_dialog def show_xpub_dialog(self, xpub, run_next): diff --git a/electrum/gui/qt/main_window.py b/electrum/gui/qt/main_window.py index ac2c5282..75667603 100644 --- a/electrum/gui/qt/main_window.py +++ b/electrum/gui/qt/main_window.py @@ -50,8 +50,9 @@ from electrum.util import (format_time, format_satoshis, format_fee_satoshis, export_meta, import_meta, bh2u, bfh, InvalidPassword, base_units, base_units_list, base_unit_name_to_decimal_point, decimal_point_to_base_unit_name, quantize_feerate) -from electrum.transaction import Transaction -from electrum.wallet import Multisig_Wallet, AddTransactionException, CannotBumpFee +from electrum.transaction import Transaction, TxOutput +from electrum.address_synchronizer import AddTransactionException +from electrum.wallet import Multisig_Wallet, CannotBumpFee from .amountedit import AmountEdit, BTCAmountEdit, MyLineEdit, FeerateEdit from .qrcodewidget import QRCodeWidget, QRDialog @@ -1305,7 +1306,7 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError): outputs = self.payto_e.get_outputs(self.is_max) if not outputs: _type, addr = self.get_payto_or_dummy() - outputs = [(_type, addr, amount)] + outputs = [TxOutput(_type, addr, amount)] is_sweep = bool(self.tx_external_keypairs) make_tx = lambda fee_est: \ self.wallet.make_unsigned_transaction( @@ -1484,14 +1485,14 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError): self.show_error(_('No outputs')) return - for _type, addr, amount in outputs: - if addr is None: + for o in outputs: + if o.address is None: self.show_error(_('Bitcoin Address is None')) return - if _type == TYPE_ADDRESS and not bitcoin.is_address(addr): + if o.type == TYPE_ADDRESS and not bitcoin.is_address(o.address): self.show_error(_('Invalid Bitcoin Address')) return - if amount is None: + if o.value is None: self.show_error(_('Invalid Amount')) return diff --git a/electrum/gui/qt/network_dialog.py b/electrum/gui/qt/network_dialog.py index f9a1f7da..ecc7695b 100644 --- a/electrum/gui/qt/network_dialog.py +++ b/electrum/gui/qt/network_dialog.py @@ -106,9 +106,9 @@ class NodesListWidget(QTreeWidget): b = network.blockchains[k] name = b.get_name() if n_chains >1: - x = QTreeWidgetItem([name + '@%d'%b.get_checkpoint(), '%d'%b.height()]) + x = QTreeWidgetItem([name + '@%d'%b.get_forkpoint(), '%d'%b.height()]) x.setData(0, Qt.UserRole, 1) - x.setData(1, Qt.UserRole, b.checkpoint) + x.setData(1, Qt.UserRole, b.forkpoint) else: x = self for i in items: @@ -357,9 +357,9 @@ class NetworkChoiceLayout(object): chains = self.network.get_blockchains() if len(chains)>1: chain = self.network.blockchain() - checkpoint = chain.get_checkpoint() + forkpoint = chain.get_forkpoint() name = chain.get_name() - msg = _('Chain split detected at block {0}').format(checkpoint) + '\n' + msg = _('Chain split detected at block {0}').format(forkpoint) + '\n' msg += (_('You are following branch') if auto_connect else _('Your server is on branch'))+ ' ' + name msg += ' (%d %s)' % (chain.get_branch_size(), _('blocks')) else: diff --git a/electrum/gui/qt/paytoedit.py b/electrum/gui/qt/paytoedit.py index 716a5f58..376303b9 100644 --- a/electrum/gui/qt/paytoedit.py +++ b/electrum/gui/qt/paytoedit.py @@ -29,13 +29,13 @@ from decimal import Decimal from electrum import bitcoin from electrum.util import bfh +from electrum.transaction import TxOutput from .qrtextedit import ScanQRTextEdit from .completion_text_edit import CompletionTextEdit from . import util -RE_ADDRESS = '[1-9A-HJ-NP-Za-km-z]{26,}' -RE_ALIAS = '(.*?)\s*\<([1-9A-HJ-NP-Za-km-z]{26,})\>' +RE_ALIAS = '(.*?)\s*\<([0-9A-Za-z]{1,})\>' frozen_style = "QWidget { background-color:none; border:none;}" normal_style = "QPlainTextEdit { }" @@ -78,7 +78,7 @@ class PayToEdit(CompletionTextEdit, ScanQRTextEdit): x, y = line.split(',') out_type, out = self.parse_output(x) amount = self.parse_amount(y) - return out_type, out, amount + return TxOutput(out_type, out, amount) def parse_output(self, x): try: @@ -140,16 +140,16 @@ class PayToEdit(CompletionTextEdit, ScanQRTextEdit): is_max = False for i, line in enumerate(lines): try: - _type, to_address, amount = self.parse_address_and_amount(line) + output = self.parse_address_and_amount(line) except: self.errors.append((i, line.strip())) continue - outputs.append((_type, to_address, amount)) - if amount == '!': + outputs.append(output) + if output.value == '!': is_max = True else: - total += amount + total += output.value self.win.is_max = is_max self.outputs = outputs @@ -175,7 +175,7 @@ class PayToEdit(CompletionTextEdit, ScanQRTextEdit): amount = self.amount_edit.get_amount() _type, addr = self.payto_address - self.outputs = [(_type, addr, amount)] + self.outputs = [TxOutput(_type, addr, amount)] return self.outputs[:] diff --git a/electrum/gui/qt/seed_dialog.py b/electrum/gui/qt/seed_dialog.py index 1006dfd9..86fc15ec 100644 --- a/electrum/gui/qt/seed_dialog.py +++ b/electrum/gui/qt/seed_dialog.py @@ -207,5 +207,6 @@ class SeedDialog(WindowModalDialog): title = _("Your wallet generation seed is:") slayout = SeedLayout(title=title, seed=seed, msg=True, passphrase=passphrase) vbox.addLayout(slayout) - run_hook('set_seed', seed, slayout.seed_e) + has_extension = True if passphrase else False + run_hook('set_seed', seed, has_extension, slayout.seed_e) vbox.addLayout(Buttons(CloseButton(self))) diff --git a/electrum/gui/qt/transaction_dialog.py b/electrum/gui/qt/transaction_dialog.py index fe7624da..4e8e6e67 100644 --- a/electrum/gui/qt/transaction_dialog.py +++ b/electrum/gui/qt/transaction_dialog.py @@ -31,13 +31,15 @@ from PyQt5.QtCore import * from PyQt5.QtGui import * from PyQt5.QtWidgets import * +import qrcode +from qrcode import exceptions + from electrum.bitcoin import base_encode from electrum.i18n import _ from electrum.plugin import run_hook from electrum import simple_config from electrum.util import bfh -from electrum.wallet import AddTransactionException from electrum.transaction import SerializationError from .util import * @@ -184,8 +186,11 @@ class TxDialog(QDialog, MessageBoxMixin): text = base_encode(text, base=43) try: self.main_window.show_qrcode(text, 'Transaction', parent=self) + except qrcode.exceptions.DataOverflowError: + self.show_error(_('Failed to display QR code.') + '\n' + + _('Transaction is too large in size.')) except Exception as e: - self.show_message(str(e)) + self.show_error(_('Failed to display QR code.') + '\n' + str(e)) def sign(self): def sign_done(success): diff --git a/electrum/gui/stdio.py b/electrum/gui/stdio.py index a4756eae..d155efb0 100644 --- a/electrum/gui/stdio.py +++ b/electrum/gui/stdio.py @@ -4,6 +4,7 @@ _ = lambda x:x from electrum import WalletStorage, Wallet from electrum.util import format_satoshis, set_verbosity from electrum.bitcoin import is_address, COIN, TYPE_ADDRESS +from electrum.transaction import TxOutput import getpass, datetime # minimal fdisk like gui for console usage @@ -87,9 +88,9 @@ class ElectrumGui: + "%d"%(width[2]+delta)+"s"+"%"+"%d"%(width[3]+delta)+"s" messages = [] - for item in self.wallet.get_history(): - tx_hash, height, conf, timestamp, delta, balance = item - if conf: + for tx_hash, tx_mined_status, delta, balance in self.wallet.get_history(): + if tx_mined_status.conf: + timestamp = tx_mined_status.timestamp try: time_str = datetime.datetime.fromtimestamp(timestamp).isoformat(' ')[:-3] except Exception: @@ -189,7 +190,8 @@ class ElectrumGui: if c == "n": return try: - tx = self.wallet.mktx([(TYPE_ADDRESS, self.str_recipient, amount)], password, self.config, fee) + tx = self.wallet.mktx([TxOutput(TYPE_ADDRESS, self.str_recipient, amount)], + password, self.config, fee) except Exception as e: print(str(e)) return diff --git a/electrum/gui/text.py b/electrum/gui/text.py index ee051563..61e3ac1d 100644 --- a/electrum/gui/text.py +++ b/electrum/gui/text.py @@ -6,6 +6,7 @@ import getpass import electrum from electrum.util import format_satoshis, set_verbosity from electrum.bitcoin import is_address, COIN, TYPE_ADDRESS +from electrum.transaction import TxOutput from .. import Wallet, WalletStorage _ = lambda x:x @@ -109,9 +110,9 @@ class ElectrumGui: b = 0 self.history = [] - for item in self.wallet.get_history(): - tx_hash, height, conf, timestamp, value, balance = item - if conf: + for tx_hash, tx_mined_status, value, balance in self.wallet.get_history(): + if tx_mined_status.conf: + timestamp = tx_mined_status.timestamp try: time_str = datetime.datetime.fromtimestamp(timestamp).isoformat(' ')[:-3] except Exception: @@ -340,7 +341,8 @@ class ElectrumGui: else: password = None try: - tx = self.wallet.mktx([(TYPE_ADDRESS, self.str_recipient, amount)], password, self.config, fee) + tx = self.wallet.mktx([TxOutput(TYPE_ADDRESS, self.str_recipient, amount)], + password, self.config, fee) except Exception as e: self.show_message(str(e)) return diff --git a/electrum/interface.py b/electrum/interface.py index 98305bb1..357d913a 100644 --- a/electrum/interface.py +++ b/electrum/interface.py @@ -59,6 +59,7 @@ def Connection(server, queue, config_path): class TcpConnection(threading.Thread, util.PrintError): + verbosity_filter = 'i' def __init__(self, server, queue, config_path): threading.Thread.__init__(self) diff --git a/electrum/keystore.py b/electrum/keystore.py index a8e068bb..c822be13 100644 --- a/electrum/keystore.py +++ b/electrum/keystore.py @@ -552,13 +552,12 @@ def bip39_normalize_passphrase(passphrase): return normalize('NFKD', passphrase or '') def bip39_to_seed(mnemonic, passphrase): - import pbkdf2, hashlib, hmac + import hashlib, hmac PBKDF2_ROUNDS = 2048 mnemonic = normalize('NFKD', ' '.join(mnemonic.split())) passphrase = bip39_normalize_passphrase(passphrase) - return pbkdf2.PBKDF2(mnemonic, 'mnemonic' + passphrase, - iterations = PBKDF2_ROUNDS, macmodule = hmac, - digestmodule = hashlib.sha512).read(64) + return hashlib.pbkdf2_hmac('sha512', mnemonic.encode('utf-8'), + b'mnemonic' + passphrase.encode('utf-8'), iterations = PBKDF2_ROUNDS) # returns tuple (is_checksum_valid, is_wordlist_valid) def bip39_is_checksum_valid(mnemonic): diff --git a/electrum/mnemonic.py b/electrum/mnemonic.py index 17c2cafc..a22913ae 100644 --- a/electrum/mnemonic.py +++ b/electrum/mnemonic.py @@ -30,7 +30,6 @@ import unicodedata import string import ecdsa -import pbkdf2 from .util import print_error from .bitcoin import is_old_seed, is_new_seed @@ -131,7 +130,7 @@ class Mnemonic(object): PBKDF2_ROUNDS = 2048 mnemonic = normalize_text(mnemonic) passphrase = normalize_text(passphrase) - return pbkdf2.PBKDF2(mnemonic, 'electrum' + passphrase, iterations = PBKDF2_ROUNDS, macmodule = hmac, digestmodule = hashlib.sha512).read(64) + return hashlib.pbkdf2_hmac('sha512', mnemonic.encode('utf-8'), b'electrum' + passphrase.encode('utf-8'), iterations = PBKDF2_ROUNDS) def mnemonic_encode(self, i): n = len(self.wordlist) diff --git a/electrum/network.py b/electrum/network.py index b836fbc7..344d60f0 100644 --- a/electrum/network.py +++ b/electrum/network.py @@ -89,6 +89,10 @@ def filter_version(servers): return {k: v for k, v in servers.items() if is_recent(v.get('version'))} +def filter_noonion(servers): + return {k: v for k, v in servers.items() if not k.endswith('.onion')} + + def filter_protocol(hostmap, protocol='s'): '''Filters the hostmap for those implementing protocol. The result is a list in serialized form.''' @@ -170,6 +174,7 @@ class Network(util.DaemonThread): get_parameters(), get_server_height(), get_status_value(), is_connected(), set_parameters(), stop() """ + verbosity_filter = 'n' def __init__(self, config=None): if config is None: @@ -408,6 +413,8 @@ class Network(util.DaemonThread): continue if host not in out: out[host] = {protocol: port} + if self.config.get('noonion'): + out = filter_noonion(out) return out @with_interface_lock @@ -875,13 +882,17 @@ class Network(util.DaemonThread): if not connect: self.connection_down(interface.server) return - # If not finished, get the next chunk - if index >= len(blockchain.checkpoints) and blockchain.height() < interface.tip: - self.request_chunk(interface, index+1) + if index >= len(blockchain.checkpoints): + # If not finished, get the next chunk + if blockchain.height() < interface.tip: + self.request_chunk(interface, index+1) + else: + interface.mode = 'default' + interface.print_error('catch up done', blockchain.height()) + blockchain.catch_up = None else: - interface.mode = 'default' - interface.print_error('catch up done', blockchain.height()) - blockchain.catch_up = None + # the verifier must have asked for this chunk + pass self.notify('updated') def on_get_header(self, interface, response): @@ -892,6 +903,7 @@ class Network(util.DaemonThread): self.connection_down(interface.server) return height = header.get('block_height') + #interface.print_error('got header', height, blockchain.hash_header(header)) if interface.request != height: interface.print_error("unsolicited header",interface.request, height) self.connection_down(interface.server) @@ -906,6 +918,9 @@ class Network(util.DaemonThread): next_height = height + 1 interface.blockchain.catch_up = interface.server elif chain: + # FIXME should await "initial chunk download". + # binary search will NOT do the correct thing if we don't yet + # have all headers up to the fork height interface.print_error("binary search") interface.mode = 'binary' interface.blockchain = chain @@ -947,9 +962,9 @@ class Network(util.DaemonThread): elif branch.parent().check_header(header): interface.print_error('reorg', interface.bad, interface.tip) interface.blockchain = branch.parent() - next_height = None + next_height = interface.bad else: - interface.print_error('checkpoint conflicts with existing fork', branch.path()) + interface.print_error('forkpoint conflicts with existing fork', branch.path()) branch.write(b'', 0) branch.save_header(interface.bad_header) interface.mode = 'catch_up' @@ -965,10 +980,12 @@ class Network(util.DaemonThread): with self.blockchains_lock: self.blockchains[interface.bad] = b interface.blockchain = b - interface.print_error("new chain", b.checkpoint) + interface.print_error("new chain", b.forkpoint) interface.mode = 'catch_up' - next_height = interface.bad + 1 - interface.blockchain.catch_up = interface.server + maybe_next_height = interface.bad + 1 + if maybe_next_height <= interface.tip: + next_height = maybe_next_height + interface.blockchain.catch_up = interface.server else: assert bh == interface.good if interface.blockchain.catch_up is None and bh < interface.tip: @@ -1081,6 +1098,7 @@ class Network(util.DaemonThread): except InvalidHeader: self.connection_down(interface.server) return + #interface.print_error('notified of header', height, blockchain.hash_header(header)) if height < self.max_checkpoint(): self.connection_down(interface.server) return @@ -1125,7 +1143,7 @@ class Network(util.DaemonThread): @with_interface_lock def blockchain(self): if self.interface and self.interface.blockchain is not None: - self.blockchain_index = self.interface.blockchain.checkpoint + self.blockchain_index = self.interface.blockchain.forkpoint return self.blockchains[self.blockchain_index] @with_interface_lock diff --git a/electrum/paymentrequest.py b/electrum/paymentrequest.py index 03ffe602..7010b93f 100644 --- a/electrum/paymentrequest.py +++ b/electrum/paymentrequest.py @@ -42,6 +42,7 @@ from .util import print_error, bh2u, bfh from .util import export_meta, import_meta from .bitcoin import TYPE_ADDRESS +from .transaction import TxOutput REQUEST_HEADERS = {'Accept': 'application/bitcoin-paymentrequest', 'User-Agent': 'Electrum'} ACK_HEADERS = {'Content-Type':'application/bitcoin-payment','Accept':'application/bitcoin-paymentack','User-Agent':'Electrum'} @@ -123,7 +124,7 @@ class PaymentRequest: self.outputs = [] for o in self.details.outputs: addr = transaction.get_address_from_output_script(o.script)[1] - self.outputs.append((TYPE_ADDRESS, addr, o.amount)) + self.outputs.append(TxOutput(TYPE_ADDRESS, addr, o.amount)) self.memo = self.details.memo self.payment_url = self.details.payment_url @@ -225,8 +226,8 @@ class PaymentRequest: def get_address(self): o = self.outputs[0] - assert o[0] == TYPE_ADDRESS - return o[1] + assert o.type == TYPE_ADDRESS + return o.address def get_requestor(self): return self.requestor if self.requestor else self.get_address() diff --git a/electrum/plugin.py b/electrum/plugin.py index c5223dc3..4ea8e53d 100644 --- a/electrum/plugin.py +++ b/electrum/plugin.py @@ -26,7 +26,6 @@ from collections import namedtuple import traceback import sys import os -import imp import pkgutil import time import threading @@ -43,6 +42,7 @@ hooks = {} class Plugins(DaemonThread): + verbosity_filter = 'p' @profiler def __init__(self, config, is_local, gui_name): diff --git a/electrum/plugins/cosigner_pool/qt.py b/electrum/plugins/cosigner_pool/qt.py index 6efe73b5..3db937ab 100644 --- a/electrum/plugins/cosigner_pool/qt.py +++ b/electrum/plugins/cosigner_pool/qt.py @@ -38,6 +38,7 @@ from electrum.wallet import Multisig_Wallet from electrum.util import bh2u, bfh from electrum.gui.qt.transaction_dialog import show_transaction +from electrum.gui.qt.util import WaitingDialog import sys import traceback @@ -170,20 +171,26 @@ class Plugin(BasePlugin): return cosigner_xpub in xpub_set def do_send(self, tx): + def on_success(result): + window.show_message(_("Your transaction was sent to the cosigning pool.") + '\n' + + _("Open your cosigner wallet to retrieve it.")) + def on_failure(exc_info): + e = exc_info[1] + try: traceback.print_exception(*exc_info) + except OSError: pass + window.show_error(_("Failed to send transaction to cosigning pool") + ':\n' + str(e)) + for window, xpub, K, _hash in self.cosigner_list: if not self.cosigner_can_sign(tx, xpub): continue + # construct message raw_tx_bytes = bfh(str(tx)) public_key = ecc.ECPubkey(K) message = public_key.encrypt_message(raw_tx_bytes).decode('ascii') - try: - server.put(_hash, message) - except Exception as e: - traceback.print_exc(file=sys.stdout) - window.show_error(_("Failed to send transaction to cosigning pool") + ':\n' + str(e)) - return - window.show_message(_("Your transaction was sent to the cosigning pool.") + '\n' + - _("Open your cosigner wallet to retrieve it.")) + # send message + task = lambda: server.put(_hash, message) + msg = _('Sending transaction to cosigning pool...') + WaitingDialog(window, msg, task, on_success, on_failure) def on_receive(self, keyhash, message): self.print_error("signal arrived for", keyhash) diff --git a/electrum/plugins/digitalbitbox/digitalbitbox.py b/electrum/plugins/digitalbitbox/digitalbitbox.py index ed8c05f8..441d33b2 100644 --- a/electrum/plugins/digitalbitbox/digitalbitbox.py +++ b/electrum/plugins/digitalbitbox/digitalbitbox.py @@ -120,8 +120,8 @@ class DigitalBitbox_Client(): def stretch_key(self, key): - import pbkdf2, hmac - return to_hexstr(pbkdf2.PBKDF2(key, b'Digital Bitbox', iterations = 20480, macmodule = hmac, digestmodule = hashlib.sha512).read(64)) + import hmac + return to_hexstr(hashlib.pbkdf2_hmac('sha512', key.encode('utf-8'), b'Digital Bitbox', iterations = 20480)) def backup_password_dialog(self): @@ -534,9 +534,9 @@ class DigitalBitbox_KeyStore(Hardware_KeyStore): self.give_error("No matching x_key for sign_transaction") # should never happen # Build pubkeyarray from outputs - for _type, address, amount in tx.outputs(): - assert _type == TYPE_ADDRESS - info = tx.output_info.get(address) + for o in tx.outputs(): + assert o.type == TYPE_ADDRESS + info = tx.output_info.get(o.address) if info is not None: index, xpubs, m = info changePath = self.get_derivation() + "/%d/%d" % index diff --git a/electrum/plugins/hw_wallet/plugin.py b/electrum/plugins/hw_wallet/plugin.py index a7f1b76e..9d2e32ac 100644 --- a/electrum/plugins/hw_wallet/plugin.py +++ b/electrum/plugins/hw_wallet/plugin.py @@ -26,7 +26,9 @@ from electrum.plugin import BasePlugin, hook from electrum.i18n import _ -from electrum.bitcoin import is_address +from electrum.bitcoin import is_address, TYPE_SCRIPT +from electrum.util import bfh +from electrum.transaction import opcodes, TxOutput class HW_PluginBase(BasePlugin): @@ -87,3 +89,15 @@ def is_any_tx_output_on_change_branch(tx): if index[0] == 1: return True return False + + +def trezor_validate_op_return_output_and_get_data(output: TxOutput) -> bytes: + if output.type != TYPE_SCRIPT: + raise Exception("Unexpected output type: {}".format(output.type)) + script = bfh(output.address) + if not (script[0] == opcodes.OP_RETURN and + script[1] == len(script) - 2 and script[1] <= 75): + raise Exception(_("Only OP_RETURN scripts, with one constant push, are supported.")) + if output.value != 0: + raise Exception(_("Amount for OP_RETURN output must be zero.")) + return script[2:] diff --git a/electrum/plugins/keepkey/keepkey.py b/electrum/plugins/keepkey/keepkey.py index 053cfe39..66a208a4 100644 --- a/electrum/plugins/keepkey/keepkey.py +++ b/electrum/plugins/keepkey/keepkey.py @@ -15,7 +15,7 @@ from electrum.wallet import Standard_Wallet from electrum.base_wizard import ScriptTypeNotSupported from ..hw_wallet import HW_PluginBase -from ..hw_wallet.plugin import is_any_tx_output_on_change_branch +from ..hw_wallet.plugin import is_any_tx_output_on_change_branch, trezor_validate_op_return_output_and_get_data # TREZOR initialization methods @@ -141,7 +141,10 @@ class KeepKeyPlugin(HW_PluginBase): 'download the updated firmware from {}') .format(self.device, client.label(), self.firmware_URL)) self.print_error(msg) - handler.show_error(msg) + if handler: + handler.show_error(msg) + else: + raise Exception(msg) return None return client @@ -288,7 +291,7 @@ class KeepKeyPlugin(HW_PluginBase): for txin in tx.inputs(): txinputtype = self.types.TxInputType() if txin['type'] == 'coinbase': - prev_hash = "\0"*32 + prev_hash = b"\x00"*32 prev_index = 0xffffffff # signed int -1 else: if for_sig: @@ -379,7 +382,7 @@ class KeepKeyPlugin(HW_PluginBase): txoutputtype.amount = amount if _type == TYPE_SCRIPT: txoutputtype.script_type = self.types.PAYTOOPRETURN - txoutputtype.op_return_data = address[2:] + txoutputtype.op_return_data = trezor_validate_op_return_output_and_get_data(o) elif _type == TYPE_ADDRESS: if is_segwit_address(address): txoutputtype.script_type = self.types.PAYTOWITNESS @@ -398,7 +401,8 @@ class KeepKeyPlugin(HW_PluginBase): has_change = False any_output_on_change_branch = is_any_tx_output_on_change_branch(tx) - for _type, address, amount in tx.outputs(): + for o in tx.outputs(): + _type, address, amount = o.type, o.address, o.value use_create_by_derivation = False info = tx.output_info.get(address) diff --git a/electrum/plugins/ledger/ledger.py b/electrum/plugins/ledger/ledger.py index 73604a9a..939a7a5a 100644 --- a/electrum/plugins/ledger/ledger.py +++ b/electrum/plugins/ledger/ledger.py @@ -12,7 +12,7 @@ from electrum.transaction import Transaction from electrum.wallet import Standard_Wallet from ..hw_wallet import HW_PluginBase from ..hw_wallet.plugin import is_any_tx_output_on_change_branch -from electrum.util import print_error, is_verbose, bfh, bh2u, versiontuple +from electrum.util import print_error, bfh, bh2u, versiontuple from electrum.base_wizard import ScriptTypeNotSupported try: @@ -24,7 +24,7 @@ try: from btchip.btchipFirmwareWizard import checkFirmware, updateFirmware from btchip.btchipException import BTChipException BTCHIP = True - BTCHIP_DEBUG = is_verbose + BTCHIP_DEBUG = False except ImportError: BTCHIP = False @@ -394,9 +394,9 @@ class Ledger_KeyStore(Hardware_KeyStore): self.give_error("Transaction with more than 2 outputs not supported") has_change = False any_output_on_change_branch = is_any_tx_output_on_change_branch(tx) - for _type, address, amount in tx.outputs(): - assert _type == TYPE_ADDRESS - info = tx.output_info.get(address) + for o in tx.outputs(): + assert o.type == TYPE_ADDRESS + info = tx.output_info.get(o.address) if (info is not None) and len(tx.outputs()) > 1 \ and not has_change: index, xpubs, m = info @@ -407,9 +407,9 @@ class Ledger_KeyStore(Hardware_KeyStore): changePath = self.get_derivation()[2:] + "/%d/%d"%index has_change = True else: - output = address + output = o.address else: - output = address + output = o.address self.handler.show_message(_("Confirm Transaction on your Ledger device...")) try: diff --git a/electrum/plugins/revealer/qt.py b/electrum/plugins/revealer/qt.py index 0ce60562..198d1cc4 100644 --- a/electrum/plugins/revealer/qt.py +++ b/electrum/plugins/revealer/qt.py @@ -39,9 +39,9 @@ class Plugin(BasePlugin): BasePlugin.__init__(self, parent, config, name) self.base_dir = config.electrum_path()+'/revealer/' - if self.config.get('calibration_h') == None: + if self.config.get('calibration_h') is None: self.config.set_key('calibration_h', 0) - if self.config.get('calibration_v') == None: + if self.config.get('calibration_v') is None: self.config.set_key('calibration_v', 0) self.calibration_h = self.config.get('calibration_h') @@ -57,8 +57,9 @@ class Plugin(BasePlugin): make_dir(self.base_dir) @hook - def set_seed(self, seed, parent): + def set_seed(self, seed, has_extension, parent): self.cseed = seed.upper() + self.has_extension = has_extension parent.addButton(':icons/revealer.png', partial(self.setup_dialog, parent), "Revealer"+_(" secret backup utility")) def requires_settings(self): @@ -168,6 +169,10 @@ class Plugin(BasePlugin): "
","", self.base_dir+ self.filename+self.version+"_"+self.code_id,""])) dialog.close() + def ext_warning(self, dialog): + dialog.show_message(''.join(["",_("Warning: "), "", _("your seed extension will not be included in the encrypted backup.")])) + dialog.close() + def bdone(self, dialog): dialog.show_message(''.join([_("Digital Revealer ({}_{}) saved as PNG and PDF at:").format(self.version, self.code_id), "
","", self.base_dir + 'revealer_' +self.version + '_'+ self.code_id, ''])) @@ -263,7 +268,7 @@ class Plugin(BasePlugin): max_letters = 17 max_lines = 6 max_words = 3 - if len(txt) > 102: + else: fontsize = 9 linespace = 10 max_letters = 24 @@ -360,6 +365,9 @@ class Plugin(BasePlugin): self.filename = self.wallet_name+'_'+ _('seed')+'_' self.was = self.wallet_name +' ' + _('seed') + if self.has_extension: + self.ext_warning(self.c_dialog) + if not calibration: self.toPdf(QImage(cypherseed)) QDesktopServices.openUrl (QUrl.fromLocalFile(os.path.abspath(self.base_dir+self.filename+self.version+'_'+self.code_id+'.pdf'))) @@ -588,8 +596,8 @@ class Plugin(BasePlugin): qr_qt = self.paintQR(self.hex_noise.upper() +self.code_id) target = QRectF(base_img.width()-65-qr_size, base_img.height()-65-qr_size, - qr_size, qr_size ); - painter.drawImage(target, qr_qt); + qr_size, qr_size ) + painter.drawImage(target, qr_qt) painter.setPen(QPen(Qt.black, 4)) painter.drawLine(base_img.width()-65-qr_size, base_img.height()-65-qr_size, diff --git a/electrum/plugins/safe_t/__init__.py b/electrum/plugins/safe_t/__init__.py new file mode 100644 index 00000000..9bfb2d9b --- /dev/null +++ b/electrum/plugins/safe_t/__init__.py @@ -0,0 +1,8 @@ +from electrum.i18n import _ + +fullname = 'Safe-T mini Wallet' +description = _('Provides support for Safe-T mini hardware wallet') +requires = [('safetlib','github.com/archos-safe-t/python-safet')] +registers_keystore = ('hardware', 'safe_t', _("Safe-T mini wallet")) +available_for = ['qt', 'cmdline'] + diff --git a/electrum/plugins/safe_t/client.py b/electrum/plugins/safe_t/client.py new file mode 100644 index 00000000..568f753b --- /dev/null +++ b/electrum/plugins/safe_t/client.py @@ -0,0 +1,11 @@ +from safetlib.client import proto, BaseClient, ProtocolMixin +from .clientbase import SafeTClientBase + +class SafeTClient(SafeTClientBase, ProtocolMixin, BaseClient): + def __init__(self, transport, handler, plugin): + BaseClient.__init__(self, transport=transport) + ProtocolMixin.__init__(self, transport=transport) + SafeTClientBase.__init__(self, handler, plugin, proto) + + +SafeTClientBase.wrap_methods(SafeTClient) diff --git a/electrum/plugins/safe_t/clientbase.py b/electrum/plugins/safe_t/clientbase.py new file mode 100644 index 00000000..68a4544c --- /dev/null +++ b/electrum/plugins/safe_t/clientbase.py @@ -0,0 +1,252 @@ +import time +from struct import pack + +from electrum.i18n import _ +from electrum.util import PrintError, UserCancelled +from electrum.keystore import bip39_normalize_passphrase +from electrum.bitcoin import serialize_xpub + + +class GuiMixin(object): + # Requires: self.proto, self.device + + # ref: https://github.com/trezor/trezor-common/blob/44dfb07cfaafffada4b2ce0d15ba1d90d17cf35e/protob/types.proto#L89 + messages = { + 3: _("Confirm the transaction output on your {} device"), + 4: _("Confirm internal entropy on your {} device to begin"), + 5: _("Write down the seed word shown on your {}"), + 6: _("Confirm on your {} that you want to wipe it clean"), + 7: _("Confirm on your {} device the message to sign"), + 8: _("Confirm the total amount spent and the transaction fee on your " + "{} device"), + 10: _("Confirm wallet address on your {} device"), + 14: _("Choose on your {} device where to enter your passphrase"), + 'default': _("Check your {} device to continue"), + } + + def callback_Failure(self, msg): + # BaseClient's unfortunate call() implementation forces us to + # raise exceptions on failure in order to unwind the stack. + # However, making the user acknowledge they cancelled + # gets old very quickly, so we suppress those. The NotInitialized + # one is misnamed and indicates a passphrase request was cancelled. + if msg.code in (self.types.FailureType.PinCancelled, + self.types.FailureType.ActionCancelled, + self.types.FailureType.NotInitialized): + raise UserCancelled() + raise RuntimeError(msg.message) + + def callback_ButtonRequest(self, msg): + message = self.msg + if not message: + message = self.messages.get(msg.code, self.messages['default']) + self.handler.show_message(message.format(self.device), self.cancel) + return self.proto.ButtonAck() + + def callback_PinMatrixRequest(self, msg): + if msg.type == 2: + msg = _("Enter a new PIN for your {}:") + elif msg.type == 3: + msg = (_("Re-enter the new PIN for your {}.\n\n" + "NOTE: the positions of the numbers have changed!")) + else: + msg = _("Enter your current {} PIN:") + pin = self.handler.get_pin(msg.format(self.device)) + if len(pin) > 9: + self.handler.show_error(_('The PIN cannot be longer than 9 characters.')) + pin = '' # to cancel below + if not pin: + return self.proto.Cancel() + return self.proto.PinMatrixAck(pin=pin) + + def callback_PassphraseRequest(self, req): + if req and hasattr(req, 'on_device') and req.on_device is True: + return self.proto.PassphraseAck() + + if self.creating_wallet: + msg = _("Enter a passphrase to generate this wallet. Each time " + "you use this wallet your {} will prompt you for the " + "passphrase. If you forget the passphrase you cannot " + "access the bitcoins in the wallet.").format(self.device) + else: + msg = _("Enter the passphrase to unlock this wallet:") + passphrase = self.handler.get_passphrase(msg, self.creating_wallet) + if passphrase is None: + return self.proto.Cancel() + passphrase = bip39_normalize_passphrase(passphrase) + + ack = self.proto.PassphraseAck(passphrase=passphrase) + length = len(ack.passphrase) + if length > 50: + self.handler.show_error(_("Too long passphrase ({} > 50 chars).").format(length)) + return self.proto.Cancel() + return ack + + def callback_PassphraseStateRequest(self, msg): + return self.proto.PassphraseStateAck() + + def callback_WordRequest(self, msg): + self.step += 1 + msg = _("Step {}/24. Enter seed word as explained on " + "your {}:").format(self.step, self.device) + word = self.handler.get_word(msg) + # Unfortunately the device can't handle self.proto.Cancel() + return self.proto.WordAck(word=word) + + +class SafeTClientBase(GuiMixin, PrintError): + + def __init__(self, handler, plugin, proto): + assert hasattr(self, 'tx_api') # ProtocolMixin already constructed? + self.proto = proto + self.device = plugin.device + self.handler = handler + self.tx_api = plugin + self.types = plugin.types + self.msg = None + self.creating_wallet = False + self.used() + + def __str__(self): + return "%s/%s" % (self.label(), self.features.device_id) + + def label(self): + '''The name given by the user to the device.''' + return self.features.label + + def is_initialized(self): + '''True if initialized, False if wiped.''' + return self.features.initialized + + def is_pairable(self): + return not self.features.bootloader_mode + + def has_usable_connection_with_device(self): + try: + res = self.ping("electrum pinging device") + assert res == "electrum pinging device" + except BaseException: + return False + return True + + def used(self): + self.last_operation = time.time() + + def prevent_timeouts(self): + self.last_operation = float('inf') + + def timeout(self, cutoff): + '''Time out the client if the last operation was before cutoff.''' + if self.last_operation < cutoff: + self.print_error("timed out") + self.clear_session() + + @staticmethod + def expand_path(n): + '''Convert bip32 path to list of uint32 integers with prime flags + 0/-1/1' -> [0, 0x80000001, 0x80000001]''' + # This code is similar to code in safetlib where it unfortunately + # is not declared as a staticmethod. Our n has an extra element. + PRIME_DERIVATION_FLAG = 0x80000000 + path = [] + for x in n.split('/')[1:]: + prime = 0 + if x.endswith("'"): + x = x.replace('\'', '') + prime = PRIME_DERIVATION_FLAG + if x.startswith('-'): + prime = PRIME_DERIVATION_FLAG + path.append(abs(int(x)) | prime) + return path + + def cancel(self): + '''Provided here as in keepkeylib but not safetlib.''' + self.transport.write(self.proto.Cancel()) + + def i4b(self, x): + return pack('>I', x) + + def get_xpub(self, bip32_path, xtype): + address_n = self.expand_path(bip32_path) + creating = False + node = self.get_public_node(address_n, creating).node + return serialize_xpub(xtype, node.chain_code, node.public_key, node.depth, self.i4b(node.fingerprint), self.i4b(node.child_num)) + + def toggle_passphrase(self): + if self.features.passphrase_protection: + self.msg = _("Confirm on your {} device to disable passphrases") + else: + self.msg = _("Confirm on your {} device to enable passphrases") + enabled = not self.features.passphrase_protection + self.apply_settings(use_passphrase=enabled) + + def change_label(self, label): + self.msg = _("Confirm the new label on your {} device") + self.apply_settings(label=label) + + def change_homescreen(self, homescreen): + self.msg = _("Confirm on your {} device to change your home screen") + self.apply_settings(homescreen=homescreen) + + def set_pin(self, remove): + if remove: + self.msg = _("Confirm on your {} device to disable PIN protection") + elif self.features.pin_protection: + self.msg = _("Confirm on your {} device to change your PIN") + else: + self.msg = _("Confirm on your {} device to set a PIN") + self.change_pin(remove) + + def clear_session(self): + '''Clear the session to force pin (and passphrase if enabled) + re-entry. Does not leak exceptions.''' + self.print_error("clear session:", self) + self.prevent_timeouts() + try: + super(SafeTClientBase, self).clear_session() + except BaseException as e: + # If the device was removed it has the same effect... + self.print_error("clear_session: ignoring error", str(e)) + + def get_public_node(self, address_n, creating): + self.creating_wallet = creating + return super(SafeTClientBase, self).get_public_node(address_n) + + def close(self): + '''Called when Our wallet was closed or the device removed.''' + self.print_error("closing client") + self.clear_session() + # Release the device + self.transport.close() + + def firmware_version(self): + f = self.features + return (f.major_version, f.minor_version, f.patch_version) + + def atleast_version(self, major, minor=0, patch=0): + return self.firmware_version() >= (major, minor, patch) + + @staticmethod + def wrapper(func): + '''Wrap methods to clear any message box they opened.''' + + def wrapped(self, *args, **kwargs): + try: + self.prevent_timeouts() + return func(self, *args, **kwargs) + finally: + self.used() + self.handler.finished() + self.creating_wallet = False + self.msg = None + + return wrapped + + @staticmethod + def wrap_methods(cls): + for method in ['apply_settings', 'change_pin', + 'get_address', 'get_public_node', + 'load_device_by_mnemonic', 'load_device_by_xprv', + 'recovery_device', 'reset_device', 'sign_message', + 'sign_tx', 'wipe_device']: + setattr(cls, method, cls.wrapper(getattr(cls, method))) diff --git a/electrum/plugins/safe_t/cmdline.py b/electrum/plugins/safe_t/cmdline.py new file mode 100644 index 00000000..9c6346d3 --- /dev/null +++ b/electrum/plugins/safe_t/cmdline.py @@ -0,0 +1,14 @@ +from electrum.plugin import hook +from .safe_t import SafeTPlugin +from ..hw_wallet import CmdLineHandler + +class Plugin(SafeTPlugin): + handler = CmdLineHandler() + @hook + def init_keystore(self, keystore): + if not isinstance(keystore, self.keystore_class): + return + keystore.handler = self.handler + + def create_handler(self, window): + return self.handler diff --git a/electrum/plugins/safe_t/qt.py b/electrum/plugins/safe_t/qt.py new file mode 100644 index 00000000..b9580aa1 --- /dev/null +++ b/electrum/plugins/safe_t/qt.py @@ -0,0 +1,492 @@ +from functools import partial +import threading + +from PyQt5.Qt import Qt +from PyQt5.Qt import QGridLayout, QInputDialog, QPushButton +from PyQt5.Qt import QVBoxLayout, QLabel + +from electrum.gui.qt.util import * +from electrum.i18n import _ +from electrum.plugin import hook, DeviceMgr +from electrum.util import PrintError, UserCancelled, bh2u +from electrum.wallet import Wallet, Standard_Wallet + +from ..hw_wallet.qt import QtHandlerBase, QtPluginBase +from .safe_t import SafeTPlugin, TIM_NEW, TIM_RECOVER, TIM_MNEMONIC + + +PASSPHRASE_HELP_SHORT =_( + "Passphrases allow you to access new wallets, each " + "hidden behind a particular case-sensitive passphrase.") +PASSPHRASE_HELP = PASSPHRASE_HELP_SHORT + " " + _( + "You need to create a separate Electrum wallet for each passphrase " + "you use as they each generate different addresses. Changing " + "your passphrase does not lose other wallets, each is still " + "accessible behind its own passphrase.") +RECOMMEND_PIN = _( + "You should enable PIN protection. Your PIN is the only protection " + "for your bitcoins if your device is lost or stolen.") +PASSPHRASE_NOT_PIN = _( + "If you forget a passphrase you will be unable to access any " + "bitcoins in the wallet behind it. A passphrase is not a PIN. " + "Only change this if you are sure you understand it.") + + +class QtHandler(QtHandlerBase): + + pin_signal = pyqtSignal(object) + + def __init__(self, win, pin_matrix_widget_class, device): + super(QtHandler, self).__init__(win, device) + self.pin_signal.connect(self.pin_dialog) + self.pin_matrix_widget_class = pin_matrix_widget_class + + def get_pin(self, msg): + self.done.clear() + self.pin_signal.emit(msg) + self.done.wait() + return self.response + + def pin_dialog(self, msg): + # Needed e.g. when resetting a device + self.clear_dialog() + dialog = WindowModalDialog(self.top_level_window(), _("Enter PIN")) + matrix = self.pin_matrix_widget_class() + vbox = QVBoxLayout() + vbox.addWidget(QLabel(msg)) + vbox.addWidget(matrix) + vbox.addLayout(Buttons(CancelButton(dialog), OkButton(dialog))) + dialog.setLayout(vbox) + dialog.exec_() + self.response = str(matrix.get_value()) + self.done.set() + + +class QtPlugin(QtPluginBase): + # Derived classes must provide the following class-static variables: + # icon_file + # pin_matrix_widget_class + + def create_handler(self, window): + return QtHandler(window, self.pin_matrix_widget_class(), self.device) + + @hook + def receive_menu(self, menu, addrs, wallet): + if len(addrs) != 1: + return + for keystore in wallet.get_keystores(): + if type(keystore) == self.keystore_class: + def show_address(): + keystore.thread.add(partial(self.show_address, wallet, addrs[0], keystore)) + menu.addAction(_("Show on {}").format(self.device), show_address) + break + + def show_settings_dialog(self, window, keystore): + device_id = self.choose_device(window, keystore) + if device_id: + SettingsDialog(window, self, keystore, device_id).exec_() + + def request_safe_t_init_settings(self, wizard, method, device): + vbox = QVBoxLayout() + next_enabled = True + label = QLabel(_("Enter a label to name your device:")) + name = QLineEdit() + hl = QHBoxLayout() + hl.addWidget(label) + hl.addWidget(name) + hl.addStretch(1) + vbox.addLayout(hl) + + def clean_text(widget): + text = widget.toPlainText().strip() + return ' '.join(text.split()) + + if method in [TIM_NEW, TIM_RECOVER]: + gb = QGroupBox() + hbox1 = QHBoxLayout() + gb.setLayout(hbox1) + vbox.addWidget(gb) + gb.setTitle(_("Select your seed length:")) + bg = QButtonGroup() + for i, count in enumerate([12, 18, 24]): + rb = QRadioButton(gb) + rb.setText(_("%d words") % count) + bg.addButton(rb) + bg.setId(rb, i) + hbox1.addWidget(rb) + rb.setChecked(True) + cb_pin = QCheckBox(_('Enable PIN protection')) + cb_pin.setChecked(True) + else: + text = QTextEdit() + text.setMaximumHeight(60) + if method == TIM_MNEMONIC: + msg = _("Enter your BIP39 mnemonic:") + else: + msg = _("Enter the master private key beginning with xprv:") + def set_enabled(): + from electrum.keystore import is_xprv + wizard.next_button.setEnabled(is_xprv(clean_text(text))) + text.textChanged.connect(set_enabled) + next_enabled = False + + vbox.addWidget(QLabel(msg)) + vbox.addWidget(text) + pin = QLineEdit() + pin.setValidator(QRegExpValidator(QRegExp('[1-9]{0,9}'))) + pin.setMaximumWidth(100) + hbox_pin = QHBoxLayout() + hbox_pin.addWidget(QLabel(_("Enter your PIN (digits 1-9):"))) + hbox_pin.addWidget(pin) + hbox_pin.addStretch(1) + + if method in [TIM_NEW, TIM_RECOVER]: + vbox.addWidget(WWLabel(RECOMMEND_PIN)) + vbox.addWidget(cb_pin) + else: + vbox.addLayout(hbox_pin) + + passphrase_msg = WWLabel(PASSPHRASE_HELP_SHORT) + passphrase_warning = WWLabel(PASSPHRASE_NOT_PIN) + passphrase_warning.setStyleSheet("color: red") + cb_phrase = QCheckBox(_('Enable passphrases')) + cb_phrase.setChecked(False) + vbox.addWidget(passphrase_msg) + vbox.addWidget(passphrase_warning) + vbox.addWidget(cb_phrase) + + wizard.exec_layout(vbox, next_enabled=next_enabled) + + if method in [TIM_NEW, TIM_RECOVER]: + item = bg.checkedId() + pin = cb_pin.isChecked() + else: + item = ' '.join(str(clean_text(text)).split()) + pin = str(pin.text()) + + return (item, name.text(), pin, cb_phrase.isChecked()) + + +class Plugin(SafeTPlugin, QtPlugin): + icon_unpaired = ":icons/safe-t_unpaired.png" + icon_paired = ":icons/safe-t.png" + + @classmethod + def pin_matrix_widget_class(self): + from safetlib.qt.pinmatrix import PinMatrixWidget + return PinMatrixWidget + + +class SettingsDialog(WindowModalDialog): + '''This dialog doesn't require a device be paired with a wallet. + We want users to be able to wipe a device even if they've forgotten + their PIN.''' + + def __init__(self, window, plugin, keystore, device_id): + title = _("{} Settings").format(plugin.device) + super(SettingsDialog, self).__init__(window, title) + self.setMaximumWidth(540) + + devmgr = plugin.device_manager() + config = devmgr.config + handler = keystore.handler + thread = keystore.thread + hs_rows, hs_cols = (64, 128) + + def invoke_client(method, *args, **kw_args): + unpair_after = kw_args.pop('unpair_after', False) + + def task(): + client = devmgr.client_by_id(device_id) + if not client: + raise RuntimeError("Device not connected") + if method: + getattr(client, method)(*args, **kw_args) + if unpair_after: + devmgr.unpair_id(device_id) + return client.features + + thread.add(task, on_success=update) + + def update(features): + self.features = features + set_label_enabled() + if features.bootloader_hash: + bl_hash = bh2u(features.bootloader_hash) + bl_hash = "\n".join([bl_hash[:32], bl_hash[32:]]) + else: + bl_hash = "N/A" + noyes = [_("No"), _("Yes")] + endis = [_("Enable Passphrases"), _("Disable Passphrases")] + disen = [_("Disabled"), _("Enabled")] + setchange = [_("Set a PIN"), _("Change PIN")] + + version = "%d.%d.%d" % (features.major_version, + features.minor_version, + features.patch_version) + + device_label.setText(features.label) + pin_set_label.setText(noyes[features.pin_protection]) + passphrases_label.setText(disen[features.passphrase_protection]) + bl_hash_label.setText(bl_hash) + label_edit.setText(features.label) + device_id_label.setText(features.device_id) + initialized_label.setText(noyes[features.initialized]) + version_label.setText(version) + clear_pin_button.setVisible(features.pin_protection) + clear_pin_warning.setVisible(features.pin_protection) + pin_button.setText(setchange[features.pin_protection]) + pin_msg.setVisible(not features.pin_protection) + passphrase_button.setText(endis[features.passphrase_protection]) + language_label.setText(features.language) + + def set_label_enabled(): + label_apply.setEnabled(label_edit.text() != self.features.label) + + def rename(): + invoke_client('change_label', label_edit.text()) + + def toggle_passphrase(): + title = _("Confirm Toggle Passphrase Protection") + currently_enabled = self.features.passphrase_protection + if currently_enabled: + msg = _("After disabling passphrases, you can only pair this " + "Electrum wallet if it had an empty passphrase. " + "If its passphrase was not empty, you will need to " + "create a new wallet with the install wizard. You " + "can use this wallet again at any time by re-enabling " + "passphrases and entering its passphrase.") + else: + msg = _("Your current Electrum wallet can only be used with " + "an empty passphrase. You must create a separate " + "wallet with the install wizard for other passphrases " + "as each one generates a new set of addresses.") + msg += "\n\n" + _("Are you sure you want to proceed?") + if not self.question(msg, title=title): + return + invoke_client('toggle_passphrase', unpair_after=currently_enabled) + + def change_homescreen(): + dialog = QFileDialog(self, _("Choose Homescreen")) + filename, __ = dialog.getOpenFileName() + if not filename: + return # user cancelled + + if filename.endswith('.toif'): + img = open(filename, 'rb').read() + if img[:8] != b'TOIf\x90\x00\x90\x00': + handler.show_error('File is not a TOIF file with size of 144x144') + return + else: + from PIL import Image # FIXME + im = Image.open(filename) + if im.size != (128, 64): + handler.show_error('Image must be 128 x 64 pixels') + return + im = im.convert('1') + pix = im.load() + img = bytearray(1024) + for j in range(64): + for i in range(128): + if pix[i, j]: + o = (i + j * 128) + img[o // 8] |= (1 << (7 - o % 8)) + img = bytes(img) + invoke_client('change_homescreen', img) + + def clear_homescreen(): + invoke_client('change_homescreen', b'\x00') + + def set_pin(): + invoke_client('set_pin', remove=False) + + def clear_pin(): + invoke_client('set_pin', remove=True) + + def wipe_device(): + wallet = window.wallet + if wallet and sum(wallet.get_balance()): + title = _("Confirm Device Wipe") + msg = _("Are you SURE you want to wipe the device?\n" + "Your wallet still has bitcoins in it!") + if not self.question(msg, title=title, + icon=QMessageBox.Critical): + return + invoke_client('wipe_device', unpair_after=True) + + def slider_moved(): + mins = timeout_slider.sliderPosition() + timeout_minutes.setText(_("%2d minutes") % mins) + + def slider_released(): + config.set_session_timeout(timeout_slider.sliderPosition() * 60) + + # Information tab + info_tab = QWidget() + info_layout = QVBoxLayout(info_tab) + info_glayout = QGridLayout() + info_glayout.setColumnStretch(2, 1) + device_label = QLabel() + pin_set_label = QLabel() + passphrases_label = QLabel() + version_label = QLabel() + device_id_label = QLabel() + bl_hash_label = QLabel() + bl_hash_label.setWordWrap(True) + language_label = QLabel() + initialized_label = QLabel() + rows = [ + (_("Device Label"), device_label), + (_("PIN set"), pin_set_label), + (_("Passphrases"), passphrases_label), + (_("Firmware Version"), version_label), + (_("Device ID"), device_id_label), + (_("Bootloader Hash"), bl_hash_label), + (_("Language"), language_label), + (_("Initialized"), initialized_label), + ] + for row_num, (label, widget) in enumerate(rows): + info_glayout.addWidget(QLabel(label), row_num, 0) + info_glayout.addWidget(widget, row_num, 1) + info_layout.addLayout(info_glayout) + + # Settings tab + settings_tab = QWidget() + settings_layout = QVBoxLayout(settings_tab) + settings_glayout = QGridLayout() + + # Settings tab - Label + label_msg = QLabel(_("Name this {}. If you have multiple devices " + "their labels help distinguish them.") + .format(plugin.device)) + label_msg.setWordWrap(True) + label_label = QLabel(_("Device Label")) + label_edit = QLineEdit() + label_edit.setMinimumWidth(150) + label_edit.setMaxLength(plugin.MAX_LABEL_LEN) + label_apply = QPushButton(_("Apply")) + label_apply.clicked.connect(rename) + label_edit.textChanged.connect(set_label_enabled) + settings_glayout.addWidget(label_label, 0, 0) + settings_glayout.addWidget(label_edit, 0, 1, 1, 2) + settings_glayout.addWidget(label_apply, 0, 3) + settings_glayout.addWidget(label_msg, 1, 1, 1, -1) + + # Settings tab - PIN + pin_label = QLabel(_("PIN Protection")) + pin_button = QPushButton() + pin_button.clicked.connect(set_pin) + settings_glayout.addWidget(pin_label, 2, 0) + settings_glayout.addWidget(pin_button, 2, 1) + pin_msg = QLabel(_("PIN protection is strongly recommended. " + "A PIN is your only protection against someone " + "stealing your bitcoins if they obtain physical " + "access to your {}.").format(plugin.device)) + pin_msg.setWordWrap(True) + pin_msg.setStyleSheet("color: red") + settings_glayout.addWidget(pin_msg, 3, 1, 1, -1) + + # Settings tab - Homescreen + homescreen_label = QLabel(_("Homescreen")) + homescreen_change_button = QPushButton(_("Change...")) + homescreen_clear_button = QPushButton(_("Reset")) + homescreen_change_button.clicked.connect(change_homescreen) + try: + import PIL + except ImportError: + homescreen_change_button.setDisabled(True) + homescreen_change_button.setToolTip( + _("Required package 'PIL' is not available - Please install it.") + ) + homescreen_clear_button.clicked.connect(clear_homescreen) + homescreen_msg = QLabel(_("You can set the homescreen on your " + "device to personalize it. You must " + "choose a {} x {} monochrome black and " + "white image.").format(hs_rows, hs_cols)) + homescreen_msg.setWordWrap(True) + settings_glayout.addWidget(homescreen_label, 4, 0) + settings_glayout.addWidget(homescreen_change_button, 4, 1) + settings_glayout.addWidget(homescreen_clear_button, 4, 2) + settings_glayout.addWidget(homescreen_msg, 5, 1, 1, -1) + + # Settings tab - Session Timeout + timeout_label = QLabel(_("Session Timeout")) + timeout_minutes = QLabel() + timeout_slider = QSlider(Qt.Horizontal) + timeout_slider.setRange(1, 60) + timeout_slider.setSingleStep(1) + timeout_slider.setTickInterval(5) + timeout_slider.setTickPosition(QSlider.TicksBelow) + timeout_slider.setTracking(True) + timeout_msg = QLabel( + _("Clear the session after the specified period " + "of inactivity. Once a session has timed out, " + "your PIN and passphrase (if enabled) must be " + "re-entered to use the device.")) + timeout_msg.setWordWrap(True) + timeout_slider.setSliderPosition(config.get_session_timeout() // 60) + slider_moved() + timeout_slider.valueChanged.connect(slider_moved) + timeout_slider.sliderReleased.connect(slider_released) + settings_glayout.addWidget(timeout_label, 6, 0) + settings_glayout.addWidget(timeout_slider, 6, 1, 1, 3) + settings_glayout.addWidget(timeout_minutes, 6, 4) + settings_glayout.addWidget(timeout_msg, 7, 1, 1, -1) + settings_layout.addLayout(settings_glayout) + settings_layout.addStretch(1) + + # Advanced tab + advanced_tab = QWidget() + advanced_layout = QVBoxLayout(advanced_tab) + advanced_glayout = QGridLayout() + + # Advanced tab - clear PIN + clear_pin_button = QPushButton(_("Disable PIN")) + clear_pin_button.clicked.connect(clear_pin) + clear_pin_warning = QLabel( + _("If you disable your PIN, anyone with physical access to your " + "{} device can spend your bitcoins.").format(plugin.device)) + clear_pin_warning.setWordWrap(True) + clear_pin_warning.setStyleSheet("color: red") + advanced_glayout.addWidget(clear_pin_button, 0, 2) + advanced_glayout.addWidget(clear_pin_warning, 1, 0, 1, 5) + + # Advanced tab - toggle passphrase protection + passphrase_button = QPushButton() + passphrase_button.clicked.connect(toggle_passphrase) + passphrase_msg = WWLabel(PASSPHRASE_HELP) + passphrase_warning = WWLabel(PASSPHRASE_NOT_PIN) + passphrase_warning.setStyleSheet("color: red") + advanced_glayout.addWidget(passphrase_button, 3, 2) + advanced_glayout.addWidget(passphrase_msg, 4, 0, 1, 5) + advanced_glayout.addWidget(passphrase_warning, 5, 0, 1, 5) + + # Advanced tab - wipe device + wipe_device_button = QPushButton(_("Wipe Device")) + wipe_device_button.clicked.connect(wipe_device) + wipe_device_msg = QLabel( + _("Wipe the device, removing all data from it. The firmware " + "is left unchanged.")) + wipe_device_msg.setWordWrap(True) + wipe_device_warning = QLabel( + _("Only wipe a device if you have the recovery seed written down " + "and the device wallet(s) are empty, otherwise the bitcoins " + "will be lost forever.")) + wipe_device_warning.setWordWrap(True) + wipe_device_warning.setStyleSheet("color: red") + advanced_glayout.addWidget(wipe_device_button, 6, 2) + advanced_glayout.addWidget(wipe_device_msg, 7, 0, 1, 5) + advanced_glayout.addWidget(wipe_device_warning, 8, 0, 1, 5) + advanced_layout.addLayout(advanced_glayout) + advanced_layout.addStretch(1) + + tabs = QTabWidget(self) + tabs.addTab(info_tab, _("Information")) + tabs.addTab(settings_tab, _("Settings")) + tabs.addTab(advanced_tab, _("Advanced")) + dialog_vbox = QVBoxLayout(self) + dialog_vbox.addWidget(tabs) + dialog_vbox.addLayout(Buttons(CloseButton(self))) + + # Update information + invoke_client(None) diff --git a/electrum/plugins/safe_t/safe_t.py b/electrum/plugins/safe_t/safe_t.py new file mode 100644 index 00000000..22008977 --- /dev/null +++ b/electrum/plugins/safe_t/safe_t.py @@ -0,0 +1,509 @@ +from binascii import hexlify, unhexlify +import traceback +import sys + +from electrum.util import bfh, bh2u, versiontuple, UserCancelled +from electrum.bitcoin import (b58_address_to_hash160, xpub_from_pubkey, deserialize_xpub, + TYPE_ADDRESS, TYPE_SCRIPT, is_address) +from electrum import constants +from electrum.i18n import _ +from electrum.plugin import BasePlugin, Device +from electrum.transaction import deserialize, Transaction +from electrum.keystore import Hardware_KeyStore, is_xpubkey, parse_xpubkey, xtype_from_derivation +from electrum.base_wizard import ScriptTypeNotSupported + +from ..hw_wallet import HW_PluginBase +from ..hw_wallet.plugin import is_any_tx_output_on_change_branch, trezor_validate_op_return_output_and_get_data + + +# Safe-T mini initialization methods +TIM_NEW, TIM_RECOVER, TIM_MNEMONIC, TIM_PRIVKEY = range(0, 4) + +# script "generation" +SCRIPT_GEN_LEGACY, SCRIPT_GEN_P2SH_SEGWIT, SCRIPT_GEN_NATIVE_SEGWIT = range(0, 3) + + +class SafeTKeyStore(Hardware_KeyStore): + hw_type = 'safe_t' + device = 'Safe-T mini' + + def get_derivation(self): + return self.derivation + + def get_script_gen(self): + xtype = xtype_from_derivation(self.derivation) + if xtype in ('p2wpkh', 'p2wsh'): + return SCRIPT_GEN_NATIVE_SEGWIT + elif xtype in ('p2wpkh-p2sh', 'p2wsh-p2sh'): + return SCRIPT_GEN_P2SH_SEGWIT + else: + return SCRIPT_GEN_LEGACY + + def get_client(self, force_pair=True): + return self.plugin.get_client(self, force_pair) + + def decrypt_message(self, sequence, message, password): + raise RuntimeError(_('Encryption and decryption are not implemented by {}').format(self.device)) + + def sign_message(self, sequence, message, password): + client = self.get_client() + address_path = self.get_derivation() + "/%d/%d"%sequence + address_n = client.expand_path(address_path) + msg_sig = client.sign_message(self.plugin.get_coin_name(), address_n, message) + return msg_sig.signature + + def sign_transaction(self, tx, password): + if tx.is_complete(): + return + # previous transactions used as inputs + prev_tx = {} + # path of the xpubs that are involved + xpub_path = {} + for txin in tx.inputs(): + pubkeys, x_pubkeys = tx.get_sorted_pubkeys(txin) + tx_hash = txin['prevout_hash'] + if txin.get('prev_tx') is None and not Transaction.is_segwit_input(txin): + raise Exception(_('Offline signing with {} is not supported for legacy inputs.').format(self.device)) + prev_tx[tx_hash] = txin['prev_tx'] + for x_pubkey in x_pubkeys: + if not is_xpubkey(x_pubkey): + continue + xpub, s = parse_xpubkey(x_pubkey) + if xpub == self.get_master_public_key(): + xpub_path[xpub] = self.get_derivation() + + self.plugin.sign_transaction(self, tx, prev_tx, xpub_path) + + +class SafeTPlugin(HW_PluginBase): + # Derived classes provide: + # + # class-static variables: client_class, firmware_URL, handler_class, + # libraries_available, libraries_URL, minimum_firmware, + # wallet_class, types + + firmware_URL = 'https://safe-t.io' + libraries_URL = 'https://github.com/archos-safe-t/python-safet' + minimum_firmware = (1, 0, 5) + keystore_class = SafeTKeyStore + minimum_library = (0, 1, 0) + SUPPORTED_XTYPES = ('standard', 'p2wpkh-p2sh', 'p2wpkh', 'p2wsh-p2sh', 'p2wsh') + + MAX_LABEL_LEN = 32 + + def __init__(self, parent, config, name): + HW_PluginBase.__init__(self, parent, config, name) + + try: + # Minimal test if python-safet is installed + import safetlib + try: + library_version = safetlib.__version__ + except AttributeError: + # python-safet only introduced __version__ in 0.1.0 + library_version = 'unknown' + if library_version == 'unknown' or \ + versiontuple(library_version) < self.minimum_library: + self.libraries_available_message = ( + _("Library version for '{}' is too old.").format(name) + + '\nInstalled: {}, Needed: {}' + .format(library_version, self.minimum_library)) + self.print_stderr(self.libraries_available_message) + raise ImportError() + self.libraries_available = True + except ImportError: + self.libraries_available = False + return + + from . import client + from . import transport + import safetlib.messages + self.client_class = client.SafeTClient + self.types = safetlib.messages + self.DEVICE_IDS = ('Safe-T mini',) + + self.transport_handler = transport.SafeTTransport() + self.device_manager().register_enumerate_func(self.enumerate) + + def enumerate(self): + devices = self.transport_handler.enumerate_devices() + return [Device(d.get_path(), -1, d.get_path(), 'Safe-T mini', 0) for d in devices] + + def create_client(self, device, handler): + try: + self.print_error("connecting to device at", device.path) + transport = self.transport_handler.get_transport(device.path) + except BaseException as e: + self.print_error("cannot connect at", device.path, str(e)) + return None + + if not transport: + self.print_error("cannot connect at", device.path) + return + + self.print_error("connected to device at", device.path) + client = self.client_class(transport, handler, self) + + # Try a ping for device sanity + try: + client.ping('t') + except BaseException as e: + self.print_error("ping failed", str(e)) + return None + + if not client.atleast_version(*self.minimum_firmware): + msg = (_('Outdated {} firmware for device labelled {}. Please ' + 'download the updated firmware from {}') + .format(self.device, client.label(), self.firmware_URL)) + self.print_error(msg) + if handler: + handler.show_error(msg) + else: + raise Exception(msg) + return None + + return client + + def get_client(self, keystore, force_pair=True): + devmgr = self.device_manager() + handler = keystore.handler + with devmgr.hid_lock: + client = devmgr.client_for_keystore(self, handler, keystore, force_pair) + # returns the client for a given keystore. can use xpub + if client: + client.used() + return client + + def get_coin_name(self): + return "Testnet" if constants.net.TESTNET else "Bitcoin" + + def initialize_device(self, device_id, wizard, handler): + # Initialization method + msg = _("Choose how you want to initialize your {}.\n\n" + "The first two methods are secure as no secret information " + "is entered into your computer.\n\n" + "For the last two methods you input secrets on your keyboard " + "and upload them to your {}, and so you should " + "only do those on a computer you know to be trustworthy " + "and free of malware." + ).format(self.device, self.device) + choices = [ + # Must be short as QT doesn't word-wrap radio button text + (TIM_NEW, _("Let the device generate a completely new seed randomly")), + (TIM_RECOVER, _("Recover from a seed you have previously written down")), + (TIM_MNEMONIC, _("Upload a BIP39 mnemonic to generate the seed")), + (TIM_PRIVKEY, _("Upload a master private key")) + ] + def f(method): + import threading + settings = self.request_safe_t_init_settings(wizard, method, self.device) + t = threading.Thread(target=self._initialize_device_safe, args=(settings, method, device_id, wizard, handler)) + t.setDaemon(True) + t.start() + exit_code = wizard.loop.exec_() + if exit_code != 0: + # this method (initialize_device) was called with the expectation + # of leaving the device in an initialized state when finishing. + # signal that this is not the case: + raise UserCancelled() + wizard.choice_dialog(title=_('Initialize Device'), message=msg, choices=choices, run_next=f) + + def _initialize_device_safe(self, settings, method, device_id, wizard, handler): + exit_code = 0 + try: + self._initialize_device(settings, method, device_id, wizard, handler) + except UserCancelled: + exit_code = 1 + except BaseException as e: + traceback.print_exc(file=sys.stderr) + handler.show_error(str(e)) + exit_code = 1 + finally: + wizard.loop.exit(exit_code) + + def _initialize_device(self, settings, method, device_id, wizard, handler): + item, label, pin_protection, passphrase_protection = settings + + if method == TIM_RECOVER: + handler.show_error(_( + "You will be asked to enter 24 words regardless of your " + "seed's actual length. If you enter a word incorrectly or " + "misspell it, you cannot change it or go back - you will need " + "to start again from the beginning.\n\nSo please enter " + "the words carefully!"), + blocking=True) + + language = 'english' + devmgr = self.device_manager() + client = devmgr.client_by_id(device_id) + + if method == TIM_NEW: + strength = 64 * (item + 2) # 128, 192 or 256 + u2f_counter = 0 + skip_backup = False + client.reset_device(True, strength, passphrase_protection, + pin_protection, label, language, + u2f_counter, skip_backup) + elif method == TIM_RECOVER: + word_count = 6 * (item + 2) # 12, 18 or 24 + client.step = 0 + client.recovery_device(word_count, passphrase_protection, + pin_protection, label, language) + elif method == TIM_MNEMONIC: + pin = pin_protection # It's the pin, not a boolean + client.load_device_by_mnemonic(str(item), pin, + passphrase_protection, + label, language) + else: + pin = pin_protection # It's the pin, not a boolean + client.load_device_by_xprv(item, pin, passphrase_protection, + label, language) + + def _make_node_path(self, xpub, address_n): + _, depth, fingerprint, child_num, chain_code, key = deserialize_xpub(xpub) + node = self.types.HDNodeType( + depth=depth, + fingerprint=int.from_bytes(fingerprint, 'big'), + child_num=int.from_bytes(child_num, 'big'), + chain_code=chain_code, + public_key=key, + ) + return self.types.HDNodePathType(node=node, address_n=address_n) + + def setup_device(self, device_info, wizard, purpose): + devmgr = self.device_manager() + device_id = device_info.device.id_ + client = devmgr.client_by_id(device_id) + if client is None: + raise Exception(_('Failed to create a client for this device.') + '\n' + + _('Make sure it is in the correct state.')) + # fixme: we should use: client.handler = wizard + client.handler = self.create_handler(wizard) + if not device_info.initialized: + self.initialize_device(device_id, wizard, client.handler) + client.get_xpub('m', 'standard') + client.used() + + def get_xpub(self, device_id, derivation, xtype, wizard): + if xtype not in self.SUPPORTED_XTYPES: + raise ScriptTypeNotSupported(_('This type of script is not supported with {}.').format(self.device)) + devmgr = self.device_manager() + client = devmgr.client_by_id(device_id) + client.handler = wizard + xpub = client.get_xpub(derivation, xtype) + client.used() + return xpub + + def get_safet_input_script_type(self, script_gen, is_multisig): + if script_gen == SCRIPT_GEN_NATIVE_SEGWIT: + return self.types.InputScriptType.SPENDWITNESS + elif script_gen == SCRIPT_GEN_P2SH_SEGWIT: + return self.types.InputScriptType.SPENDP2SHWITNESS + else: + if is_multisig: + return self.types.InputScriptType.SPENDMULTISIG + else: + return self.types.InputScriptType.SPENDADDRESS + + def sign_transaction(self, keystore, tx, prev_tx, xpub_path): + self.prev_tx = prev_tx + self.xpub_path = xpub_path + client = self.get_client(keystore) + inputs = self.tx_inputs(tx, True, keystore.get_script_gen()) + outputs = self.tx_outputs(keystore.get_derivation(), tx, keystore.get_script_gen()) + signatures = client.sign_tx(self.get_coin_name(), inputs, outputs, lock_time=tx.locktime)[0] + signatures = [(bh2u(x) + '01') for x in signatures] + tx.update_signatures(signatures) + + def show_address(self, wallet, address, keystore=None): + if keystore is None: + keystore = wallet.get_keystore() + if not self.show_address_helper(wallet, address, keystore): + return + client = self.get_client(keystore) + if not client.atleast_version(1, 0): + keystore.handler.show_error(_("Your device firmware is too old")) + return + change, index = wallet.get_address_index(address) + derivation = keystore.derivation + address_path = "%s/%d/%d"%(derivation, change, index) + address_n = client.expand_path(address_path) + xpubs = wallet.get_master_public_keys() + if len(xpubs) == 1: + script_gen = keystore.get_script_gen() + script_type = self.get_safet_input_script_type(script_gen, is_multisig=False) + client.get_address(self.get_coin_name(), address_n, True, script_type=script_type) + else: + def f(xpub): + return self._make_node_path(xpub, [change, index]) + pubkeys = wallet.get_public_keys(address) + # sort xpubs using the order of pubkeys + sorted_pubkeys, sorted_xpubs = zip(*sorted(zip(pubkeys, xpubs))) + pubkeys = list(map(f, sorted_xpubs)) + multisig = self.types.MultisigRedeemScriptType( + pubkeys=pubkeys, + signatures=[b''] * wallet.n, + m=wallet.m, + ) + script_gen = keystore.get_script_gen() + script_type = self.get_safet_input_script_type(script_gen, is_multisig=True) + client.get_address(self.get_coin_name(), address_n, True, multisig=multisig, script_type=script_type) + + def tx_inputs(self, tx, for_sig=False, script_gen=SCRIPT_GEN_LEGACY): + inputs = [] + for txin in tx.inputs(): + txinputtype = self.types.TxInputType() + if txin['type'] == 'coinbase': + prev_hash = b"\x00"*32 + prev_index = 0xffffffff # signed int -1 + else: + if for_sig: + x_pubkeys = txin['x_pubkeys'] + if len(x_pubkeys) == 1: + x_pubkey = x_pubkeys[0] + xpub, s = parse_xpubkey(x_pubkey) + xpub_n = self.client_class.expand_path(self.xpub_path[xpub]) + txinputtype._extend_address_n(xpub_n + s) + txinputtype.script_type = self.get_safet_input_script_type(script_gen, is_multisig=False) + else: + def f(x_pubkey): + if is_xpubkey(x_pubkey): + xpub, s = parse_xpubkey(x_pubkey) + else: + xpub = xpub_from_pubkey(0, bfh(x_pubkey)) + s = [] + return self._make_node_path(xpub, s) + pubkeys = list(map(f, x_pubkeys)) + multisig = self.types.MultisigRedeemScriptType( + pubkeys=pubkeys, + signatures=list(map(lambda x: bfh(x)[:-1] if x else b'', txin.get('signatures'))), + m=txin.get('num_sig'), + ) + script_type = self.get_safet_input_script_type(script_gen, is_multisig=True) + txinputtype = self.types.TxInputType( + script_type=script_type, + multisig=multisig + ) + # find which key is mine + for x_pubkey in x_pubkeys: + if is_xpubkey(x_pubkey): + xpub, s = parse_xpubkey(x_pubkey) + if xpub in self.xpub_path: + xpub_n = self.client_class.expand_path(self.xpub_path[xpub]) + txinputtype._extend_address_n(xpub_n + s) + break + + prev_hash = unhexlify(txin['prevout_hash']) + prev_index = txin['prevout_n'] + + if 'value' in txin: + txinputtype.amount = txin['value'] + txinputtype.prev_hash = prev_hash + txinputtype.prev_index = prev_index + + if txin.get('scriptSig') is not None: + script_sig = bfh(txin['scriptSig']) + txinputtype.script_sig = script_sig + + txinputtype.sequence = txin.get('sequence', 0xffffffff - 1) + + inputs.append(txinputtype) + + return inputs + + def tx_outputs(self, derivation, tx, script_gen=SCRIPT_GEN_LEGACY): + + def create_output_by_derivation(info): + index, xpubs, m = info + if len(xpubs) == 1: + if script_gen == SCRIPT_GEN_NATIVE_SEGWIT: + script_type = self.types.OutputScriptType.PAYTOWITNESS + elif script_gen == SCRIPT_GEN_P2SH_SEGWIT: + script_type = self.types.OutputScriptType.PAYTOP2SHWITNESS + else: + script_type = self.types.OutputScriptType.PAYTOADDRESS + address_n = self.client_class.expand_path(derivation + "/%d/%d" % index) + txoutputtype = self.types.TxOutputType( + amount=amount, + script_type=script_type, + address_n=address_n, + ) + else: + if script_gen == SCRIPT_GEN_NATIVE_SEGWIT: + script_type = self.types.OutputScriptType.PAYTOWITNESS + elif script_gen == SCRIPT_GEN_P2SH_SEGWIT: + script_type = self.types.OutputScriptType.PAYTOP2SHWITNESS + else: + script_type = self.types.OutputScriptType.PAYTOMULTISIG + address_n = self.client_class.expand_path("/%d/%d" % index) + pubkeys = [self._make_node_path(xpub, address_n) for xpub in xpubs] + multisig = self.types.MultisigRedeemScriptType( + pubkeys=pubkeys, + signatures=[b''] * len(pubkeys), + m=m) + txoutputtype = self.types.TxOutputType( + multisig=multisig, + amount=amount, + address_n=self.client_class.expand_path(derivation + "/%d/%d" % index), + script_type=script_type) + return txoutputtype + + def create_output_by_address(): + txoutputtype = self.types.TxOutputType() + txoutputtype.amount = amount + if _type == TYPE_SCRIPT: + txoutputtype.script_type = self.types.OutputScriptType.PAYTOOPRETURN + txoutputtype.op_return_data = trezor_validate_op_return_output_and_get_data(o) + elif _type == TYPE_ADDRESS: + txoutputtype.script_type = self.types.OutputScriptType.PAYTOADDRESS + txoutputtype.address = address + return txoutputtype + + outputs = [] + has_change = False + any_output_on_change_branch = is_any_tx_output_on_change_branch(tx) + + for o in tx.outputs(): + _type, address, amount = o.type, o.address, o.value + use_create_by_derivation = False + + info = tx.output_info.get(address) + if info is not None and not has_change: + index, xpubs, m = info + on_change_branch = index[0] == 1 + # prioritise hiding outputs on the 'change' branch from user + # because no more than one change address allowed + # note: ^ restriction can be removed once we require fw + # that has https://github.com/trezor/trezor-mcu/pull/306 + if on_change_branch == any_output_on_change_branch: + use_create_by_derivation = True + has_change = True + + if use_create_by_derivation: + txoutputtype = create_output_by_derivation(info) + else: + txoutputtype = create_output_by_address() + outputs.append(txoutputtype) + + return outputs + + def electrum_tx_to_txtype(self, tx): + t = self.types.TransactionType() + if tx is None: + # probably for segwit input and we don't need this prev txn + return t + d = deserialize(tx.raw) + t.version = d['version'] + t.lock_time = d['lockTime'] + inputs = self.tx_inputs(tx) + t._extend_inputs(inputs) + for vout in d['outputs']: + o = t._add_bin_outputs() + o.amount = vout['value'] + o.script_pubkey = bfh(vout['scriptPubKey']) + return t + + # This function is called from the TREZOR libraries (via tx_api) + def get_tx(self, tx_hash): + tx = self.prev_tx[tx_hash] + return self.electrum_tx_to_txtype(tx) diff --git a/electrum/plugins/safe_t/transport.py b/electrum/plugins/safe_t/transport.py new file mode 100644 index 00000000..59dc915f --- /dev/null +++ b/electrum/plugins/safe_t/transport.py @@ -0,0 +1,95 @@ +from electrum.util import PrintError + + +class SafeTTransport(PrintError): + + @staticmethod + def all_transports(): + """Reimplemented safetlib.transport.all_transports so that we can + enable/disable specific transports. + """ + try: + # only to detect safetlib version + from safetlib.transport import all_transports + except ImportError: + # old safetlib. compat for safetlib < 0.9.2 + transports = [] + #try: + # from safetlib.transport_bridge import BridgeTransport + # transports.append(BridgeTransport) + #except BaseException: + # pass + try: + from safetlib.transport_hid import HidTransport + transports.append(HidTransport) + except BaseException: + pass + try: + from safetlib.transport_udp import UdpTransport + transports.append(UdpTransport) + except BaseException: + pass + try: + from safetlib.transport_webusb import WebUsbTransport + transports.append(WebUsbTransport) + except BaseException: + pass + else: + # new safetlib. + transports = [] + #try: + # from safetlib.transport.bridge import BridgeTransport + # transports.append(BridgeTransport) + #except BaseException: + # pass + try: + from safetlib.transport.hid import HidTransport + transports.append(HidTransport) + except BaseException: + pass + try: + from safetlib.transport.udp import UdpTransport + transports.append(UdpTransport) + except BaseException: + pass + try: + from safetlib.transport.webusb import WebUsbTransport + transports.append(WebUsbTransport) + except BaseException: + pass + return transports + return transports + + def enumerate_devices(self): + """Just like safetlib.transport.enumerate_devices, + but with exception catching, so that transports can fail separately. + """ + devices = [] + for transport in self.all_transports(): + try: + new_devices = transport.enumerate() + except BaseException as e: + self.print_error('enumerate failed for {}. error {}' + .format(transport.__name__, str(e))) + else: + devices.extend(new_devices) + return devices + + def get_transport(self, path=None): + """Reimplemented safetlib.transport.get_transport, + (1) for old safetlib + (2) to be able to disable specific transports + (3) to call our own enumerate_devices that catches exceptions + """ + if path is None: + try: + return self.enumerate_devices()[0] + except IndexError: + raise Exception("No Safe-T mini found") from None + + def match_prefix(a, b): + return a.startswith(b) or b.startswith(a) + transports = [t for t in self.all_transports() if match_prefix(path, t.PATH_PREFIX)] + if transports: + return transports[0].find_by_path(path) + raise Exception("Unknown path prefix '%s'" % path) diff --git a/electrum/plugins/trezor/trezor.py b/electrum/plugins/trezor/trezor.py index 94b0754f..e9893824 100644 --- a/electrum/plugins/trezor/trezor.py +++ b/electrum/plugins/trezor/trezor.py @@ -13,7 +13,7 @@ from electrum.keystore import Hardware_KeyStore, is_xpubkey, parse_xpubkey, xtyp from electrum.base_wizard import ScriptTypeNotSupported from ..hw_wallet import HW_PluginBase -from ..hw_wallet.plugin import is_any_tx_output_on_change_branch +from ..hw_wallet.plugin import is_any_tx_output_on_change_branch, trezor_validate_op_return_output_and_get_data # TREZOR initialization methods @@ -157,7 +157,10 @@ class TrezorPlugin(HW_PluginBase): 'download the updated firmware from {}') .format(self.device, client.label(), self.firmware_URL)) self.print_error(msg) - handler.show_error(msg) + if handler: + handler.show_error(msg) + else: + raise Exception(msg) return None return client @@ -362,7 +365,7 @@ class TrezorPlugin(HW_PluginBase): for txin in tx.inputs(): txinputtype = self.types.TxInputType() if txin['type'] == 'coinbase': - prev_hash = "\0"*32 + prev_hash = b"\x00"*32 prev_index = 0xffffffff # signed int -1 else: if for_sig: @@ -461,7 +464,7 @@ class TrezorPlugin(HW_PluginBase): txoutputtype.amount = amount if _type == TYPE_SCRIPT: txoutputtype.script_type = self.types.OutputScriptType.PAYTOOPRETURN - txoutputtype.op_return_data = address[2:] + txoutputtype.op_return_data = trezor_validate_op_return_output_and_get_data(o) elif _type == TYPE_ADDRESS: txoutputtype.script_type = self.types.OutputScriptType.PAYTOADDRESS txoutputtype.address = address @@ -471,7 +474,8 @@ class TrezorPlugin(HW_PluginBase): has_change = False any_output_on_change_branch = is_any_tx_output_on_change_branch(tx) - for _type, address, amount in tx.outputs(): + for o in tx.outputs(): + _type, address, amount = o.type, o.address, o.value use_create_by_derivation = False info = tx.output_info.get(address) diff --git a/electrum/plugins/trustedcoin/trustedcoin.py b/electrum/plugins/trustedcoin/trustedcoin.py index 9877da97..ccd24e33 100644 --- a/electrum/plugins/trustedcoin/trustedcoin.py +++ b/electrum/plugins/trustedcoin/trustedcoin.py @@ -33,6 +33,7 @@ from urllib.parse import quote from electrum import bitcoin, ecc, constants, keystore, version from electrum.bitcoin import * +from electrum.transaction import TxOutput from electrum.mnemonic import Mnemonic from electrum.wallet import Multisig_Wallet, Deterministic_Wallet from electrum.i18n import _ @@ -273,7 +274,7 @@ class Wallet_2fa(Multisig_Wallet): fee = self.extra_fee(config) if not is_sweep else 0 if fee: address = self.billing_info['billing_address'] - fee_output = (TYPE_ADDRESS, address, fee) + fee_output = TxOutput(TYPE_ADDRESS, address, fee) try: tx = mk_tx(outputs + [fee_output]) except NotEnoughFunds: @@ -395,9 +396,9 @@ class TrustedCoinPlugin(BasePlugin): def get_tx_extra_fee(self, wallet, tx): if type(wallet) != Wallet_2fa: return - for _type, addr, amount in tx.outputs(): - if _type == TYPE_ADDRESS and wallet.is_billing_address(addr): - return addr, amount + for o in tx.outputs(): + if o.type == TYPE_ADDRESS and wallet.is_billing_address(o.address): + return o.address, o.value def finish_requesting(func): def f(self, *args, **kwargs): diff --git a/electrum/simple_config.py b/electrum/simple_config.py index 03558cd7..25efbb8d 100644 --- a/electrum/simple_config.py +++ b/electrum/simple_config.py @@ -4,7 +4,8 @@ import time import os import stat from decimal import Decimal -from typing import Union +from typing import Union, Optional +from numbers import Real from copy import deepcopy @@ -295,35 +296,51 @@ class SimpleConfig(PrintError): return fee return get_fee_within_limits - @impose_hard_limits_on_fee - def eta_to_fee(self, slider_pos) -> Union[int, None]: + def eta_to_fee(self, slider_pos) -> Optional[int]: """Returns fee in sat/kbyte.""" slider_pos = max(slider_pos, 0) slider_pos = min(slider_pos, len(FEE_ETA_TARGETS)) if slider_pos < len(FEE_ETA_TARGETS): - target_blocks = FEE_ETA_TARGETS[slider_pos] - fee = self.fee_estimates.get(target_blocks) + num_blocks = FEE_ETA_TARGETS[slider_pos] + fee = self.eta_target_to_fee(num_blocks) else: - fee = self.fee_estimates.get(2) - if fee is not None: - fee += fee/2 - fee = int(fee) + fee = self.eta_target_to_fee(1) return fee - def fee_to_depth(self, target_fee): + @impose_hard_limits_on_fee + def eta_target_to_fee(self, num_blocks: int) -> Optional[int]: + """Returns fee in sat/kbyte.""" + if num_blocks == 1: + fee = self.fee_estimates.get(2) + if fee is not None: + fee += fee / 2 + fee = int(fee) + else: + fee = self.fee_estimates.get(num_blocks) + return fee + + def fee_to_depth(self, target_fee: Real) -> int: + """For a given sat/vbyte fee, returns an estimate of how deep + it would be in the current mempool in vbytes. + Pessimistic == overestimates the depth. + """ depth = 0 for fee, s in self.mempool_fees: depth += s if fee <= target_fee: break - else: - return 0 return depth - @impose_hard_limits_on_fee def depth_to_fee(self, slider_pos) -> int: """Returns fee in sat/kbyte.""" target = self.depth_target(slider_pos) + return self.depth_target_to_fee(target) + + @impose_hard_limits_on_fee + def depth_target_to_fee(self, target: int) -> int: + """Returns fee in sat/kbyte. + target: desired mempool depth in vbytes + """ depth = 0 for fee, s in self.mempool_fees: depth += s @@ -331,6 +348,10 @@ class SimpleConfig(PrintError): break else: return 0 + # add one sat/byte as currently that is + # the max precision of the histogram + fee += 1 + # convert to sat/kbyte return fee * 1000 def depth_target(self, slider_pos): diff --git a/electrum/storage.py b/electrum/storage.py index 70fb19a5..156df968 100644 --- a/electrum/storage.py +++ b/electrum/storage.py @@ -29,7 +29,7 @@ import json import copy import re import stat -import pbkdf2, hmac, hashlib +import hmac, hashlib import base64 import zlib from collections import defaultdict @@ -44,7 +44,7 @@ from .keystore import bip44_derivation OLD_SEED_VERSION = 4 # electrum versions < 2.0 NEW_SEED_VERSION = 11 # electrum versions >= 2.0 -FINAL_SEED_VERSION = 17 # electrum >= 2.7 will set this to prevent +FINAL_SEED_VERSION = 18 # electrum >= 2.7 will set this to prevent # old versions from overwriting new format @@ -67,15 +67,84 @@ def get_derivation_used_for_hw_device_encryption(): # storage encryption version STO_EV_PLAINTEXT, STO_EV_USER_PW, STO_EV_XPUB_PW = range(0, 3) -class WalletStorage(PrintError): - def __init__(self, path, manual_upgrades=False): - self.print_error("wallet path", path) - self.manual_upgrades = manual_upgrades - self.lock = threading.RLock() +class JsonDB(PrintError): + + def __init__(self, path): + self.db_lock = threading.RLock() self.data = {} self.path = path self.modified = False + + def get(self, key, default=None): + with self.db_lock: + v = self.data.get(key) + if v is None: + v = default + else: + v = copy.deepcopy(v) + return v + + def put(self, key, value): + try: + json.dumps(key, cls=util.MyEncoder) + json.dumps(value, cls=util.MyEncoder) + except: + self.print_error("json error: cannot save", key) + return + with self.db_lock: + if value is not None: + if self.data.get(key) != value: + self.modified = True + self.data[key] = copy.deepcopy(value) + elif key in self.data: + self.modified = True + self.data.pop(key) + + @profiler + def write(self): + with self.db_lock: + self._write() + + def _write(self): + if threading.currentThread().isDaemon(): + self.print_error('warning: daemon thread cannot write db') + return + if not self.modified: + return + s = json.dumps(self.data, indent=4, sort_keys=True, cls=util.MyEncoder) + s = self.encrypt_before_writing(s) + + temp_path = "%s.tmp.%s" % (self.path, os.getpid()) + with open(temp_path, "w", encoding='utf-8') as f: + f.write(s) + f.flush() + os.fsync(f.fileno()) + + mode = os.stat(self.path).st_mode if os.path.exists(self.path) else stat.S_IREAD | stat.S_IWRITE + # perform atomic write on POSIX systems + try: + os.rename(temp_path, self.path) + except: + os.remove(self.path) + os.rename(temp_path, self.path) + os.chmod(self.path, mode) + self.print_error("saved", self.path) + self.modified = False + + def encrypt_before_writing(self, plaintext: str) -> str: + return plaintext + + def file_exists(self): + return self.path and os.path.exists(self.path) + + +class WalletStorage(JsonDB): + + def __init__(self, path, manual_upgrades=False): + self.print_error("wallet path", path) + JsonDB.__init__(self, path) + self.manual_upgrades = manual_upgrades self.pubkey = None if self.file_exists(): with open(self.path, "r", encoding='utf-8') as f: @@ -160,12 +229,9 @@ class WalletStorage(PrintError): except: return STO_EV_PLAINTEXT - def file_exists(self): - return self.path and os.path.exists(self.path) - @staticmethod def get_eckey_from_password(password): - secret = pbkdf2.PBKDF2(password, '', iterations=1024, macmodule=hmac, digestmodule=hashlib.sha512).read(64) + secret = hashlib.pbkdf2_hmac('sha512', password.encode('utf-8'), b'', iterations=1024) ec_key = ecc.ECPrivkey.from_arbitrary_size_secret(secret) return ec_key @@ -189,6 +255,17 @@ class WalletStorage(PrintError): s = s.decode('utf8') self.load_data(s) + def encrypt_before_writing(self, plaintext: str) -> str: + s = plaintext + if self.pubkey: + s = bytes(s, 'utf8') + c = zlib.compress(s) + enc_magic = self._get_encryption_magic() + public_key = ecc.ECPubkey(bfh(self.pubkey)) + s = public_key.encrypt_message(c, enc_magic) + s = s.decode('utf8') + return s + def check_password(self, password): """Raises an InvalidPassword exception on invalid password""" if not self.is_encrypted(): @@ -211,71 +288,9 @@ class WalletStorage(PrintError): self.pubkey = None self._encryption_version = STO_EV_PLAINTEXT # make sure next storage.write() saves changes - with self.lock: + with self.db_lock: self.modified = True - def get(self, key, default=None): - with self.lock: - v = self.data.get(key) - if v is None: - v = default - else: - v = copy.deepcopy(v) - return v - - def put(self, key, value): - try: - json.dumps(key, cls=util.MyEncoder) - json.dumps(value, cls=util.MyEncoder) - except: - self.print_error("json error: cannot save", key) - return - with self.lock: - if value is not None: - if self.data.get(key) != value: - self.modified = True - self.data[key] = copy.deepcopy(value) - elif key in self.data: - self.modified = True - self.data.pop(key) - - @profiler - def write(self): - with self.lock: - self._write() - - def _write(self): - if threading.currentThread().isDaemon(): - self.print_error('warning: daemon thread cannot write wallet') - return - if not self.modified: - return - s = json.dumps(self.data, indent=4, sort_keys=True, cls=util.MyEncoder) - if self.pubkey: - s = bytes(s, 'utf8') - c = zlib.compress(s) - enc_magic = self._get_encryption_magic() - public_key = ecc.ECPubkey(bfh(self.pubkey)) - s = public_key.encrypt_message(c, enc_magic) - s = s.decode('utf8') - - temp_path = "%s.tmp.%s" % (self.path, os.getpid()) - with open(temp_path, "w", encoding='utf-8') as f: - f.write(s) - f.flush() - os.fsync(f.fileno()) - - mode = os.stat(self.path).st_mode if os.path.exists(self.path) else stat.S_IREAD | stat.S_IWRITE - # perform atomic write on POSIX systems - try: - os.rename(temp_path, self.path) - except: - os.remove(self.path) - os.rename(temp_path, self.path) - os.chmod(self.path, mode) - self.print_error("saved", self.path) - self.modified = False - def requires_split(self): d = self.get('accounts', {}) return len(d) > 1 @@ -304,7 +319,7 @@ class WalletStorage(PrintError): storage2.upgrade() storage2.write() result = [storage1.path, storage2.path] - elif wallet_type in ['bip44', 'trezor', 'keepkey', 'ledger', 'btchip', 'digitalbitbox']: + elif wallet_type in ['bip44', 'trezor', 'keepkey', 'ledger', 'btchip', 'digitalbitbox', 'safe_t']: mpk = storage.get('master_public_keys') for k in d.keys(): i = int(k) @@ -341,6 +356,7 @@ class WalletStorage(PrintError): self.convert_version_15() self.convert_version_16() self.convert_version_17() + self.convert_version_18() self.put('seed_version', FINAL_SEED_VERSION) # just to be sure self.write() @@ -401,7 +417,7 @@ class WalletStorage(PrintError): self.put('wallet_type', 'standard') self.put('keystore', d) - elif wallet_type in ['trezor', 'keepkey', 'ledger', 'digitalbitbox']: + elif wallet_type in ['trezor', 'keepkey', 'ledger', 'digitalbitbox', 'safe_t']: xpub = xpubs["x/0'"] derivation = self.get('derivation', bip44_derivation(0)) d = { @@ -555,6 +571,15 @@ class WalletStorage(PrintError): self.put('seed_version', 17) + def convert_version_18(self): + # delete verified_tx3 as its structure changed + if not self._is_upgrade_method_needed(17, 17): + return + + self.put('verified_tx3', None) + + self.put('seed_version', 18) + def convert_imported(self): if not self._is_upgrade_method_needed(0, 13): return @@ -637,7 +662,7 @@ class WalletStorage(PrintError): # version 1.9.8 created v6 wallets when an incorrect seed was entered in the restore dialog msg += '\n\nThis file was created because of a bug in version 1.9.8.' if self.get('master_public_keys') is None and self.get('master_private_keys') is None and self.get('imported_keys') is None: - # pbkdf2 was not included with the binaries, and wallet creation aborted. + # pbkdf2 (at that time an additional dependency) was not included with the binaries, and wallet creation aborted. msg += "\nIt does not contain any keys, and can safely be removed." else: # creation was complete if electrum was run from source diff --git a/electrum/tests/test_simple_config.py b/electrum/tests/test_simple_config.py index a4c79312..6a3dbd02 100644 --- a/electrum/tests/test_simple_config.py +++ b/electrum/tests/test_simple_config.py @@ -110,6 +110,43 @@ class Test_SimpleConfig(SequentialTestCase): result.pop('config_version', None) self.assertEqual({"something": "a"}, result) + def test_depth_target_to_fee(self): + config = SimpleConfig(self.options) + config.mempool_fees = [[49, 100110], [10, 121301], [6, 153731], [5, 125872], [1, 36488810]] + self.assertEqual( 2 * 1000, config.depth_target_to_fee(1000000)) + self.assertEqual( 6 * 1000, config.depth_target_to_fee( 500000)) + self.assertEqual( 7 * 1000, config.depth_target_to_fee( 250000)) + self.assertEqual(11 * 1000, config.depth_target_to_fee( 200000)) + self.assertEqual(50 * 1000, config.depth_target_to_fee( 100000)) + config.mempool_fees = [] + self.assertEqual( 1 * 1000, config.depth_target_to_fee(10 ** 5)) + self.assertEqual( 1 * 1000, config.depth_target_to_fee(10 ** 6)) + self.assertEqual( 1 * 1000, config.depth_target_to_fee(10 ** 7)) + config.mempool_fees = [[1, 36488810]] + self.assertEqual( 2 * 1000, config.depth_target_to_fee(10 ** 5)) + self.assertEqual( 2 * 1000, config.depth_target_to_fee(10 ** 6)) + self.assertEqual( 2 * 1000, config.depth_target_to_fee(10 ** 7)) + self.assertEqual( 1 * 1000, config.depth_target_to_fee(10 ** 8)) + config.mempool_fees = [[5, 125872], [1, 36488810]] + self.assertEqual( 6 * 1000, config.depth_target_to_fee(10 ** 5)) + self.assertEqual( 2 * 1000, config.depth_target_to_fee(10 ** 6)) + self.assertEqual( 2 * 1000, config.depth_target_to_fee(10 ** 7)) + self.assertEqual( 1 * 1000, config.depth_target_to_fee(10 ** 8)) + + def test_fee_to_depth(self): + config = SimpleConfig(self.options) + config.mempool_fees = [[49, 100000], [10, 120000], [6, 150000], [5, 125000], [1, 36000000]] + self.assertEqual(100000, config.fee_to_depth(500)) + self.assertEqual(100000, config.fee_to_depth(50)) + self.assertEqual(100000, config.fee_to_depth(49)) + self.assertEqual(220000, config.fee_to_depth(48)) + self.assertEqual(220000, config.fee_to_depth(10)) + self.assertEqual(370000, config.fee_to_depth(9)) + self.assertEqual(370000, config.fee_to_depth(6.5)) + self.assertEqual(370000, config.fee_to_depth(6)) + self.assertEqual(495000, config.fee_to_depth(5.5)) + self.assertEqual(36495000, config.fee_to_depth(0.5)) + class TestUserConfig(SequentialTestCase): diff --git a/electrum/tests/test_wallet_vertical.py b/electrum/tests/test_wallet_vertical.py index 96d42dcc..1a38c922 100644 --- a/electrum/tests/test_wallet_vertical.py +++ b/electrum/tests/test_wallet_vertical.py @@ -7,8 +7,10 @@ from typing import Sequence from electrum import storage, bitcoin, keystore, constants from electrum import Transaction from electrum import SimpleConfig -from electrum.wallet import TX_HEIGHT_UNCONFIRMED, TX_HEIGHT_UNCONF_PARENT, sweep, Multisig_Wallet, Standard_Wallet, Imported_Wallet +from electrum.address_synchronizer import TX_HEIGHT_UNCONFIRMED, TX_HEIGHT_UNCONF_PARENT +from electrum.wallet import sweep, Multisig_Wallet, Standard_Wallet, Imported_Wallet from electrum.util import bfh, bh2u +from electrum.transaction import TxOutput from electrum.plugins.trustedcoin import trustedcoin @@ -17,6 +19,11 @@ from . import SequentialTestCase from .test_bitcoin import needs_test_with_all_ecc_implementations +_UNICODE_HORROR_HEX = 'e282bf20f09f988020f09f98882020202020e3818620e38191e3819fe381be20e3828fe3828b2077cda2cda2cd9d68cda16fcda2cda120ccb8cda26bccb5cd9f6eccb4cd98c7ab77ccb8cc9b73cd9820cc80cc8177cd98cda2e1b8a9ccb561d289cca1cda27420cca7cc9568cc816fccb572cd8fccb5726f7273cca120ccb6cda1cda06cc4afccb665cd9fcd9f20ccb6cd9d696ecda220cd8f74cc9568ccb7cca1cd9f6520cd9fcd9f64cc9b61cd9c72cc95cda16bcca2cca820cda168ccb465cd8f61ccb7cca2cca17274cc81cd8f20ccb4ccb7cda0c3b2ccb5ccb666ccb82075cca7cd986ec3adcc9bcd9c63cda2cd8f6fccb7cd8f64ccb8cda265cca1cd9d3fcd9e' +UNICODE_HORROR = bfh(_UNICODE_HORROR_HEX).decode('utf-8') +# '₿ 😀 😈 う けたま わる w͢͢͝h͡o͢͡ ̸͢k̵͟n̴͘ǫw̸̛s͘ ̀́w͘͢ḩ̵a҉̡͢t ̧̕h́o̵r͏̵rors̡ ̶͡͠lį̶e͟͟ ̶͝in͢ ͏t̕h̷̡͟e ͟͟d̛a͜r̕͡k̢̨ ͡h̴e͏a̷̢̡rt́͏ ̴̷͠ò̵̶f̸ u̧͘ní̛͜c͢͏o̷͏d̸͢e̡͝?͞' + + class WalletIntegrityHelper: gap_limit = 1 # make tests run faster @@ -67,7 +74,6 @@ class WalletIntegrityHelper: return w -# TODO passphrase/seed_extension class TestWalletKeystoreAddressIntegrityForMainnet(SequentialTestCase): @needs_test_with_all_ecc_implementations @@ -110,6 +116,26 @@ class TestWalletKeystoreAddressIntegrityForMainnet(SequentialTestCase): self.assertEqual(w.get_receiving_addresses()[0], 'bc1q3g5tmkmlvxryhh843v4dz026avatc0zzr6h3af') self.assertEqual(w.get_change_addresses()[0], 'bc1qdy94n2q5qcp0kg7v9yzwe6wvfkhnvyzje7nx2p') + @needs_test_with_all_ecc_implementations + @mock.patch.object(storage.WalletStorage, '_write') + def test_electrum_seed_segwit_passphrase(self, mock_write): + seed_words = 'bitter grass shiver impose acquire brush forget axis eager alone wine silver' + self.assertEqual(bitcoin.seed_type(seed_words), 'segwit') + + ks = keystore.from_seed(seed_words, UNICODE_HORROR, False) + + WalletIntegrityHelper.check_seeded_keystore_sanity(self, ks) + self.assertTrue(isinstance(ks, keystore.BIP32_KeyStore)) + + self.assertEqual(ks.xprv, 'zprvAZDmEQiCLUcZXPfrBXoksCD2R6RMAzAre7SUyBotibisy9c7vGhLYvHaP3d9rYU12DKAWdZfscPNA7qEPgTkCDqX5sE93ryAJAQvkDbfLxU') + self.assertEqual(ks.xpub, 'zpub6nD7dvF6ArArjskKHZLmEL9ky8FqaSti1LN5maDWGwFrqwwGTp1b6ic4EHwciFNaYDmCXcQYxXSiF9BjcLCMPcaYkVN2nQD6QjYQ8vpSR3Z') + + w = WalletIntegrityHelper.create_standard_wallet(ks) + self.assertEqual(w.txin_type, 'p2wpkh') + + self.assertEqual(w.get_receiving_addresses()[0], 'bc1qx94dutas7ysn2my645cyttujrms5d9p57f6aam') + self.assertEqual(w.get_change_addresses()[0], 'bc1qcywwsy87sdp8vz5rfjh3sxdv6rt95kujdqq38g') + @needs_test_with_all_ecc_implementations @mock.patch.object(storage.WalletStorage, '_write') def test_electrum_seed_old(self, mock_write): @@ -182,6 +208,25 @@ class TestWalletKeystoreAddressIntegrityForMainnet(SequentialTestCase): self.assertEqual(w.get_receiving_addresses()[0], '16j7Dqk3Z9DdTdBtHcCVLaNQy9MTgywUUo') self.assertEqual(w.get_change_addresses()[0], '1GG5bVeWgAp5XW7JLCphse14QaC4qiHyWn') + @needs_test_with_all_ecc_implementations + @mock.patch.object(storage.WalletStorage, '_write') + def test_bip39_seed_bip44_standard_passphrase(self, mock_write): + seed_words = 'treat dwarf wealth gasp brass outside high rent blood crowd make initial' + self.assertEqual(keystore.bip39_is_checksum_valid(seed_words), (True, True)) + + ks = keystore.from_bip39_seed(seed_words, UNICODE_HORROR, "m/44'/0'/0'") + + self.assertTrue(isinstance(ks, keystore.BIP32_KeyStore)) + + self.assertEqual(ks.xprv, 'xprv9z8izheguGnLopSqkY7GcGFrP2Gu6rzBvvHo6uB9B8DWJhsows6WDZAsbBTaP3ncP2AVbTQphyEQkahrB9s1L7ihZtfz5WGQPMbXwsUtSik') + self.assertEqual(ks.xpub, 'xpub6D85QDBajeLe2JXJrZeGyQCaw47PWKi3J9DPuHakjTkVBWCxVQQkmMVMSSfnw39tj9FntbozpRtb1AJ8ubjeVSBhyK4M5mzdvsXZzKPwodT') + + w = WalletIntegrityHelper.create_standard_wallet(ks) + self.assertEqual(w.txin_type, 'p2pkh') + + self.assertEqual(w.get_receiving_addresses()[0], '1F88g2naBMhDB7pYFttPWGQgryba3hPevM') + self.assertEqual(w.get_change_addresses()[0], '1H4QD1rg2zQJ4UjuAVJr5eW1fEM8WMqyxh') + @needs_test_with_all_ecc_implementations @mock.patch.object(storage.WalletStorage, '_write') def test_bip39_seed_bip49_p2sh_segwit(self, mock_write): @@ -488,7 +533,7 @@ class TestWalletSending(TestCaseForTestnet): wallet1.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED) # wallet1 -> wallet2 - outputs = [(bitcoin.TYPE_ADDRESS, wallet2.get_receiving_address(), 250000)] + outputs = [TxOutput(bitcoin.TYPE_ADDRESS, wallet2.get_receiving_address(), 250000)] tx = wallet1.mktx(outputs=outputs, password=None, config=self.config, fee=5000) self.assertTrue(tx.is_complete()) @@ -508,7 +553,7 @@ class TestWalletSending(TestCaseForTestnet): wallet2.receive_tx_callback(tx.txid(), tx, TX_HEIGHT_UNCONFIRMED) # wallet2 -> wallet1 - outputs = [(bitcoin.TYPE_ADDRESS, wallet1.get_receiving_address(), 100000)] + outputs = [TxOutput(bitcoin.TYPE_ADDRESS, wallet1.get_receiving_address(), 100000)] tx = wallet2.mktx(outputs=outputs, password=None, config=self.config, fee=5000) self.assertTrue(tx.is_complete()) @@ -561,7 +606,7 @@ class TestWalletSending(TestCaseForTestnet): wallet1a.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED) # wallet1 -> wallet2 - outputs = [(bitcoin.TYPE_ADDRESS, wallet2.get_receiving_address(), 370000)] + outputs = [TxOutput(bitcoin.TYPE_ADDRESS, wallet2.get_receiving_address(), 370000)] tx = wallet1a.mktx(outputs=outputs, password=None, config=self.config, fee=5000) tx = Transaction(tx.serialize()) # simulates moving partial txn between cosigners self.assertFalse(tx.is_complete()) @@ -584,7 +629,7 @@ class TestWalletSending(TestCaseForTestnet): wallet2.receive_tx_callback(tx.txid(), tx, TX_HEIGHT_UNCONFIRMED) # wallet2 -> wallet1 - outputs = [(bitcoin.TYPE_ADDRESS, wallet1a.get_receiving_address(), 100000)] + outputs = [TxOutput(bitcoin.TYPE_ADDRESS, wallet1a.get_receiving_address(), 100000)] tx = wallet2.mktx(outputs=outputs, password=None, config=self.config, fee=5000) self.assertTrue(tx.is_complete()) @@ -652,7 +697,7 @@ class TestWalletSending(TestCaseForTestnet): wallet1a.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED) # wallet1 -> wallet2 - outputs = [(bitcoin.TYPE_ADDRESS, wallet2a.get_receiving_address(), 165000)] + outputs = [TxOutput(bitcoin.TYPE_ADDRESS, wallet2a.get_receiving_address(), 165000)] tx = wallet1a.mktx(outputs=outputs, password=None, config=self.config, fee=5000) txid = tx.txid() tx = Transaction(tx.serialize()) # simulates moving partial txn between cosigners @@ -678,7 +723,7 @@ class TestWalletSending(TestCaseForTestnet): wallet2a.receive_tx_callback(tx.txid(), tx, TX_HEIGHT_UNCONFIRMED) # wallet2 -> wallet1 - outputs = [(bitcoin.TYPE_ADDRESS, wallet1a.get_receiving_address(), 100000)] + outputs = [TxOutput(bitcoin.TYPE_ADDRESS, wallet1a.get_receiving_address(), 100000)] tx = wallet2a.mktx(outputs=outputs, password=None, config=self.config, fee=5000) txid = tx.txid() tx = Transaction(tx.serialize()) # simulates moving partial txn between cosigners @@ -732,7 +777,7 @@ class TestWalletSending(TestCaseForTestnet): wallet1a.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED) # wallet1 -> wallet2 - outputs = [(bitcoin.TYPE_ADDRESS, wallet2.get_receiving_address(), 1000000)] + outputs = [TxOutput(bitcoin.TYPE_ADDRESS, wallet2.get_receiving_address(), 1000000)] tx = wallet1a.mktx(outputs=outputs, password=None, config=self.config, fee=5000) self.assertTrue(tx.is_complete()) @@ -752,7 +797,7 @@ class TestWalletSending(TestCaseForTestnet): wallet2.receive_tx_callback(tx.txid(), tx, TX_HEIGHT_UNCONFIRMED) # wallet2 -> wallet1 - outputs = [(bitcoin.TYPE_ADDRESS, wallet1a.get_receiving_address(), 300000)] + outputs = [TxOutput(bitcoin.TYPE_ADDRESS, wallet1a.get_receiving_address(), 300000)] tx = wallet2.mktx(outputs=outputs, password=None, config=self.config, fee=5000) self.assertTrue(tx.is_complete()) @@ -788,7 +833,7 @@ class TestWalletSending(TestCaseForTestnet): wallet.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED) # create tx - outputs = [(bitcoin.TYPE_ADDRESS, '2N1VTMMFb91SH9SNRAkT7z8otP5eZEct4KL', 2500000)] + outputs = [TxOutput(bitcoin.TYPE_ADDRESS, '2N1VTMMFb91SH9SNRAkT7z8otP5eZEct4KL', 2500000)] coins = wallet.get_spendable_coins(domain=None, config=self.config) tx = wallet.make_unsigned_transaction(coins, outputs, config=self.config, fixed_fee=5000) tx.set_rbf(True) @@ -874,7 +919,7 @@ class TestWalletSending(TestCaseForTestnet): wallet.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED) # create tx - outputs = [(bitcoin.TYPE_ADDRESS, '2N1VTMMFb91SH9SNRAkT7z8otP5eZEct4KL', 2500000)] + outputs = [TxOutput(bitcoin.TYPE_ADDRESS, '2N1VTMMFb91SH9SNRAkT7z8otP5eZEct4KL', 2500000)] coins = wallet.get_spendable_coins(domain=None, config=self.config) tx = wallet.make_unsigned_transaction(coins, outputs, config=self.config, fixed_fee=5000) tx.set_rbf(True) @@ -1004,7 +1049,7 @@ class TestWalletOfflineSigning(TestCaseForTestnet): wallet_online.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED) # create unsigned tx - outputs = [(bitcoin.TYPE_ADDRESS, 'tb1qp0mv2sxsyxxfj5gl0332f9uyez93su9cf26757', 2500000)] + outputs = [TxOutput(bitcoin.TYPE_ADDRESS, 'tb1qp0mv2sxsyxxfj5gl0332f9uyez93su9cf26757', 2500000)] tx = wallet_online.mktx(outputs=outputs, password=None, config=self.config, fee=5000) tx.set_rbf(True) tx.locktime = 1325340 @@ -1044,7 +1089,7 @@ class TestWalletOfflineSigning(TestCaseForTestnet): wallet_online.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED) # create unsigned tx - outputs = [(bitcoin.TYPE_ADDRESS, 'tb1qp0mv2sxsyxxfj5gl0332f9uyez93su9cf26757', 2500000)] + outputs = [TxOutput(bitcoin.TYPE_ADDRESS, 'tb1qp0mv2sxsyxxfj5gl0332f9uyez93su9cf26757', 2500000)] tx = wallet_online.mktx(outputs=outputs, password=None, config=self.config, fee=5000) tx.set_rbf(True) tx.locktime = 1325341 @@ -1085,7 +1130,7 @@ class TestWalletOfflineSigning(TestCaseForTestnet): wallet_online.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED) # create unsigned tx - outputs = [(bitcoin.TYPE_ADDRESS, 'tb1qp0mv2sxsyxxfj5gl0332f9uyez93su9cf26757', 2500000)] + outputs = [TxOutput(bitcoin.TYPE_ADDRESS, 'tb1qp0mv2sxsyxxfj5gl0332f9uyez93su9cf26757', 2500000)] tx = wallet_online.mktx(outputs=outputs, password=None, config=self.config, fee=5000) tx.set_rbf(True) tx.locktime = 1325341 @@ -1121,7 +1166,7 @@ class TestWalletOfflineSigning(TestCaseForTestnet): wallet_online.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED) # create unsigned tx - outputs = [(bitcoin.TYPE_ADDRESS, 'tb1quk7ahlhr3qmjndy0uvu9y9hxfesrtahtta9ghm', 2500000)] + outputs = [TxOutput(bitcoin.TYPE_ADDRESS, 'tb1quk7ahlhr3qmjndy0uvu9y9hxfesrtahtta9ghm', 2500000)] tx = wallet_online.mktx(outputs=outputs, password=None, config=self.config, fee=5000) tx.set_rbf(True) tx.locktime = 1325340 @@ -1155,7 +1200,7 @@ class TestWalletOfflineSigning(TestCaseForTestnet): wallet_online.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED) # create unsigned tx - outputs = [(bitcoin.TYPE_ADDRESS, 'tb1quk7ahlhr3qmjndy0uvu9y9hxfesrtahtta9ghm', 2500000)] + outputs = [TxOutput(bitcoin.TYPE_ADDRESS, 'tb1quk7ahlhr3qmjndy0uvu9y9hxfesrtahtta9ghm', 2500000)] tx = wallet_online.mktx(outputs=outputs, password=None, config=self.config, fee=5000) tx.set_rbf(True) tx.locktime = 1325340 @@ -1189,7 +1234,7 @@ class TestWalletOfflineSigning(TestCaseForTestnet): wallet_online.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED) # create unsigned tx - outputs = [(bitcoin.TYPE_ADDRESS, 'tb1quk7ahlhr3qmjndy0uvu9y9hxfesrtahtta9ghm', 2500000)] + outputs = [TxOutput(bitcoin.TYPE_ADDRESS, 'tb1quk7ahlhr3qmjndy0uvu9y9hxfesrtahtta9ghm', 2500000)] tx = wallet_online.mktx(outputs=outputs, password=None, config=self.config, fee=5000) tx.set_rbf(True) tx.locktime = 1325340 @@ -1226,7 +1271,7 @@ class TestWalletOfflineSigning(TestCaseForTestnet): wallet_online.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED) # create unsigned tx - outputs = [(bitcoin.TYPE_ADDRESS, 'tb1quk7ahlhr3qmjndy0uvu9y9hxfesrtahtta9ghm', 2500000)] + outputs = [TxOutput(bitcoin.TYPE_ADDRESS, 'tb1quk7ahlhr3qmjndy0uvu9y9hxfesrtahtta9ghm', 2500000)] tx = wallet_online.mktx(outputs=outputs, password=None, config=self.config, fee=5000) tx.set_rbf(True) tx.locktime = 1325340 @@ -1263,7 +1308,7 @@ class TestWalletOfflineSigning(TestCaseForTestnet): wallet_online.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED) # create unsigned tx - outputs = [(bitcoin.TYPE_ADDRESS, 'tb1quk7ahlhr3qmjndy0uvu9y9hxfesrtahtta9ghm', 2500000)] + outputs = [TxOutput(bitcoin.TYPE_ADDRESS, 'tb1quk7ahlhr3qmjndy0uvu9y9hxfesrtahtta9ghm', 2500000)] tx = wallet_online.mktx(outputs=outputs, password=None, config=self.config, fee=5000) tx.set_rbf(True) tx.locktime = 1325340 @@ -1300,7 +1345,7 @@ class TestWalletOfflineSigning(TestCaseForTestnet): wallet_online.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED) # create unsigned tx - outputs = [(bitcoin.TYPE_ADDRESS, 'tb1quk7ahlhr3qmjndy0uvu9y9hxfesrtahtta9ghm', 2500000)] + outputs = [TxOutput(bitcoin.TYPE_ADDRESS, 'tb1quk7ahlhr3qmjndy0uvu9y9hxfesrtahtta9ghm', 2500000)] tx = wallet_online.mktx(outputs=outputs, password=None, config=self.config, fee=5000) tx.set_rbf(True) tx.locktime = 1325340 @@ -1349,7 +1394,7 @@ class TestWalletOfflineSigning(TestCaseForTestnet): wallet_online.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED) # create unsigned tx - outputs = [(bitcoin.TYPE_ADDRESS, '2MuCQQHJNnrXzQzuqfUCfAwAjPqpyEHbgue', 2500000)] + outputs = [TxOutput(bitcoin.TYPE_ADDRESS, '2MuCQQHJNnrXzQzuqfUCfAwAjPqpyEHbgue', 2500000)] tx = wallet_online.mktx(outputs=outputs, password=None, config=self.config, fee=5000) tx.set_rbf(True) tx.locktime = 1325503 @@ -1406,7 +1451,7 @@ class TestWalletOfflineSigning(TestCaseForTestnet): wallet_online.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED) # create unsigned tx - outputs = [(bitcoin.TYPE_ADDRESS, '2N8CtJRwxb2GCaiWWdSHLZHHLoZy53CCyxf', 2500000)] + outputs = [TxOutput(bitcoin.TYPE_ADDRESS, '2N8CtJRwxb2GCaiWWdSHLZHHLoZy53CCyxf', 2500000)] tx = wallet_online.mktx(outputs=outputs, password=None, config=self.config, fee=5000) tx.set_rbf(True) tx.locktime = 1325504 @@ -1465,7 +1510,7 @@ class TestWalletOfflineSigning(TestCaseForTestnet): wallet_online.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED) # create unsigned tx - outputs = [(bitcoin.TYPE_ADDRESS, '2MyoZVy8T1t94yLmyKu8DP1SmbWvnxbkwRA', 2500000)] + outputs = [TxOutput(bitcoin.TYPE_ADDRESS, '2MyoZVy8T1t94yLmyKu8DP1SmbWvnxbkwRA', 2500000)] tx = wallet_online.mktx(outputs=outputs, password=None, config=self.config, fee=5000) tx.set_rbf(True) tx.locktime = 1325505 diff --git a/electrum/transaction.py b/electrum/transaction.py index 54ca5948..8fcce24c 100644 --- a/electrum/transaction.py +++ b/electrum/transaction.py @@ -27,7 +27,7 @@ # Note: The deserialization code originally comes from ABE. -from typing import Sequence, Union +from typing import Sequence, Union, NamedTuple from .util import print_error, profiler @@ -59,6 +59,10 @@ class NotRecognizedRedeemScript(Exception): pass +TxOutput = NamedTuple("TxOutput", [('type', int), ('address', str), ('value', Union[int, str])]) +# ^ value is str when the output is set to max: '!' + + class BCDataStream(object): def __init__(self): self.input = None @@ -672,7 +676,7 @@ class Transaction: `signatures` is expected to be a list of sigs with signatures[i] intended for self._inputs[i]. - This is used by the Trezor and KeepKey plugins. + This is used by the Trezor, KeepKey an Safe-T plugins. """ if self.is_complete(): return @@ -721,7 +725,7 @@ class Transaction: return d = deserialize(self.raw, force_full_parse) self._inputs = d['inputs'] - self._outputs = [(x['type'], x['address'], x['value']) for x in d['outputs']] + self._outputs = [TxOutput(x['type'], x['address'], x['value']) for x in d['outputs']] self.locktime = d['lockTime'] self.version = d['version'] self.is_partial_originally = d['partial'] @@ -1180,17 +1184,17 @@ class Transaction: def get_outputs(self): """convert pubkeys to addresses""" - o = [] - for type, x, v in self.outputs(): - if type == TYPE_ADDRESS: - addr = x - elif type == TYPE_PUBKEY: + outputs = [] + for o in self.outputs(): + if o.type == TYPE_ADDRESS: + addr = o.address + elif o.type == TYPE_PUBKEY: # TODO do we really want this conversion? it's not really that address after all - addr = bitcoin.public_key_to_p2pkh(bfh(x)) + addr = bitcoin.public_key_to_p2pkh(bfh(o.address)) else: - addr = 'SCRIPT ' + x - o.append((addr,v)) # consider using yield (addr, v) - return o + addr = 'SCRIPT ' + o.address + outputs.append((addr, o.value)) # consider using yield (addr, v) + return outputs def get_output_addresses(self): return [addr for addr, val in self.get_outputs()] diff --git a/electrum/util.py b/electrum/util.py index 0e9f1285..123dc212 100644 --- a/electrum/util.py +++ b/electrum/util.py @@ -23,6 +23,7 @@ import binascii import os, sys, re, json from collections import defaultdict +from typing import NamedTuple from datetime import datetime import decimal from decimal import Decimal @@ -31,6 +32,8 @@ import urllib import threading import hmac import stat +import inspect +from locale import localeconv from .i18n import _ @@ -118,6 +121,8 @@ class UserCancelled(Exception): pass class Satoshis(object): + __slots__ = ('value',) + def __new__(cls, value): self = super(Satoshis, cls).__new__(cls) self.value = value @@ -130,6 +135,8 @@ class Satoshis(object): return format_satoshis(self.value) + " BTC" class Fiat(object): + __slots__ = ('value', 'ccy') + def __new__(cls, value, ccy): self = super(Fiat, cls).__new__(cls) self.ccy = ccy @@ -164,12 +171,14 @@ class MyEncoder(json.JSONEncoder): class PrintError(object): '''A handy base class''' + verbosity_filter = '' + def diagnostic_name(self): return self.__class__.__name__ def print_error(self, *msg): - # only prints with --verbose flag - print_error("[%s]" % self.diagnostic_name(), *msg) + if self.verbosity_filter in verbosity or verbosity == '*': + print_error("[%s]" % self.diagnostic_name(), *msg) def print_stderr(self, *msg): print_stderr("[%s]" % self.diagnostic_name(), *msg) @@ -213,6 +222,7 @@ class DebugMem(ThreadJob): class DaemonThread(threading.Thread, PrintError): """ daemon thread that terminates cleanly """ + verbosity_filter = 'd' def __init__(self): threading.Thread.__init__(self) @@ -263,15 +273,14 @@ class DaemonThread(threading.Thread, PrintError): self.print_error("stopped") -# TODO: disable -is_verbose = True +verbosity = '*' def set_verbosity(b): - global is_verbose - is_verbose = b + global verbosity + verbosity = b def print_error(*args): - if not is_verbose: return + if not verbosity: return print_stderr(*args) def print_stderr(*args): @@ -307,14 +316,24 @@ def constant_time_compare(val1, val2): # decorator that prints execution time def profiler(func): - def do_profile(func, args, kw_args): - n = func.__name__ + def get_func_name(args): + arg_names_from_sig = inspect.getfullargspec(func).args + # prepend class name if there is one (and if we can find it) + if len(arg_names_from_sig) > 0 and len(args) > 0 \ + and arg_names_from_sig[0] in ('self', 'cls', 'klass'): + classname = args[0].__class__.__name__ + else: + classname = '' + name = '{}.{}'.format(classname, func.__name__) if classname else func.__name__ + return name + def do_profile(args, kw_args): + name = get_func_name(args) t0 = time.time() o = func(*args, **kw_args) t = time.time() - t0 - print_error("[profiler]", n, "%.4f"%t) + print_error("[profiler]", name, "%.4f"%t) return o - return lambda *args, **kw_args: do_profile(func, args, kw_args) + return lambda *args, **kw_args: do_profile(args, kw_args) def android_ext_dir(): @@ -463,8 +482,10 @@ def format_satoshis_plain(x, decimal_point = 8): return "{:.8f}".format(Decimal(x) / scale_factor).rstrip('0').rstrip('.') +DECIMAL_POINT = localeconv()['decimal_point'] + + def format_satoshis(x, num_zeros=0, decimal_point=8, precision=None, is_diff=False, whitespaces=False): - from locale import localeconv if x is None: return 'unknown' if precision is None: @@ -474,7 +495,7 @@ def format_satoshis(x, num_zeros=0, decimal_point=8, precision=None, is_diff=Fal decimal_format = '+' + decimal_format result = ("{:" + decimal_format + "f}").format(x / pow (10, decimal_point)).rstrip('0') integer_part, fract_part = result.split(".") - dp = localeconv()['decimal_point'] + dp = DECIMAL_POINT if len(fract_part) < num_zeros: fract_part += "0" * (num_zeros - len(fract_part)) result = integer_part + dp + fract_part @@ -901,3 +922,13 @@ def make_dir(path, allow_symlink=True): raise Exception('Dangling link: ' + path) os.mkdir(path) os.chmod(path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) + + +TxMinedStatus = NamedTuple("TxMinedStatus", [("height", int), + ("conf", int), + ("timestamp", int), + ("header_hash", str)]) +VerifiedTxInfo = NamedTuple("VerifiedTxInfo", [("height", int), + ("timestamp", int), + ("txpos", int), + ("header_hash", str)]) diff --git a/electrum/verifier.py b/electrum/verifier.py index 0a78ad05..4a0d82ec 100644 --- a/electrum/verifier.py +++ b/electrum/verifier.py @@ -20,12 +20,19 @@ # ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -from .util import ThreadJob, bh2u + +from typing import Sequence, Optional + +from .util import ThreadJob, bh2u, VerifiedTxInfo from .bitcoin import Hash, hash_decode, hash_encode from .transaction import Transaction +from .blockchain import hash_header -class InnerNodeOfSpvProofIsValidTx(Exception): pass +class MerkleVerificationFailure(Exception): pass +class MissingBlockHeader(MerkleVerificationFailure): pass +class MerkleRootMismatch(MerkleVerificationFailure): pass +class InnerNodeOfSpvProofIsValidTx(MerkleVerificationFailure): pass class SPV(ThreadJob): @@ -85,44 +92,41 @@ class SPV(ThreadJob): tx_hash = params[0] tx_height = merkle.get('block_height') pos = merkle.get('pos') - try: - merkle_root = self.hash_merkle_root(merkle['merkle'], tx_hash, pos) - except InnerNodeOfSpvProofIsValidTx: - self.print_error("merkle verification failed for {} (inner node looks like tx)" - .format(tx_hash)) - return + merkle_branch = merkle.get('merkle') header = self.network.blockchain().read_header(tx_height) - # FIXME: if verification fails below, - # we should make a fresh connection to a server to - # recover from this, as this TX will now never verify - if not header: - self.print_error( - "merkle verification failed for {} (missing header {})" - .format(tx_hash, tx_height)) - return - if header.get('merkle_root') != merkle_root: - self.print_error( - "merkle verification failed for {} (merkle root mismatch {} != {})" - .format(tx_hash, header.get('merkle_root'), merkle_root)) + try: + verify_tx_is_in_block(tx_hash, merkle_branch, pos, header, tx_height) + except MerkleVerificationFailure as e: + self.print_error(str(e)) + # FIXME: we should make a fresh connection to a server + # to recover from this, as this TX will now never verify return # we passed all the tests - self.merkle_roots[tx_hash] = merkle_root + self.merkle_roots[tx_hash] = header.get('merkle_root') try: # note: we could pop in the beginning, but then we would request # this proof again in case of verification failure from the same server self.requested_merkle.remove(tx_hash) except KeyError: pass self.print_error("verified %s" % tx_hash) - self.wallet.add_verified_tx(tx_hash, (tx_height, header.get('timestamp'), pos)) + header_hash = hash_header(header) + vtx_info = VerifiedTxInfo(tx_height, header.get('timestamp'), pos, header_hash) + self.wallet.add_verified_tx(tx_hash, vtx_info) if self.is_up_to_date() and self.wallet.is_up_to_date(): self.wallet.save_verified_tx(write=True) @classmethod - def hash_merkle_root(cls, merkle_s, target_hash, pos): - h = hash_decode(target_hash) - for i in range(len(merkle_s)): - item = merkle_s[i] - h = Hash(hash_decode(item) + h) if ((pos >> i) & 1) else Hash(h + hash_decode(item)) + def hash_merkle_root(cls, merkle_branch: Sequence[str], tx_hash: str, leaf_pos_in_tree: int): + """Return calculated merkle root.""" + try: + h = hash_decode(tx_hash) + merkle_branch_bytes = [hash_decode(item) for item in merkle_branch] + int(leaf_pos_in_tree) # raise if invalid + except Exception as e: + raise MerkleVerificationFailure(e) + + for i, item in enumerate(merkle_branch_bytes): + h = Hash(item + h) if ((leaf_pos_in_tree >> i) & 1) else Hash(h + item) cls._raise_if_valid_tx(bh2u(h)) return hash_encode(h) @@ -141,7 +145,7 @@ class SPV(ThreadJob): raise InnerNodeOfSpvProofIsValidTx() def undo_verifications(self): - height = self.blockchain.get_checkpoint() + height = self.blockchain.get_forkpoint() tx_hashes = self.wallet.undo_verifications(self.blockchain, height) for tx_hash in tx_hashes: self.print_error("redoing", tx_hash) @@ -156,3 +160,16 @@ class SPV(ThreadJob): def is_up_to_date(self): return not self.requested_merkle + + +def verify_tx_is_in_block(tx_hash: str, merkle_branch: Sequence[str], + leaf_pos_in_tree: int, block_header: Optional[dict], + block_height: int) -> None: + """Raise MerkleVerificationFailure if verification fails.""" + if not block_header: + raise MissingBlockHeader("merkle verification failed for {} (missing header {})" + .format(tx_hash, block_height)) + calc_merkle_root = SPV.hash_merkle_root(merkle_branch, tx_hash, leaf_pos_in_tree) + if block_header.get('merkle_root') != calc_merkle_root: + raise MerkleRootMismatch("merkle verification failed for {} ({} != {})".format( + tx_hash, block_header.get('merkle_root'), calc_merkle_root)) diff --git a/electrum/wallet.py b/electrum/wallet.py index 416deffd..c9fef88b 100644 --- a/electrum/wallet.py +++ b/electrum/wallet.py @@ -28,7 +28,7 @@ import os -import threading +import sys import random import time import json @@ -36,18 +36,14 @@ import copy import errno import traceback from functools import partial -from collections import defaultdict from numbers import Number from decimal import Decimal -import itertools - -import sys from .i18n import _ from .util import (NotEnoughFunds, PrintError, UserCancelled, profiler, format_satoshis, format_fee_satoshis, NoDynamicFeeEstimates, TimeoutException, WalletFileException, BitcoinException, - InvalidPassword) + InvalidPassword, format_time) from .bitcoin import * from .version import * @@ -55,10 +51,10 @@ from .keystore import load_keystore, Hardware_KeyStore from .storage import multisig_type, STO_EV_PLAINTEXT, STO_EV_USER_PW, STO_EV_XPUB_PW from . import transaction, bitcoin, coinchooser, paymentrequest, contacts -from .transaction import Transaction +from .transaction import Transaction, TxOutput from .plugin import run_hook -from .synchronizer import Synchronizer -from .verifier import SPV +from .address_synchronizer import (AddressSynchronizer, TX_HEIGHT_LOCAL, + TX_HEIGHT_UNCONF_PARENT, TX_HEIGHT_UNCONFIRMED) from .paymentrequest import PR_PAID, PR_UNPAID, PR_UNKNOWN, PR_EXPIRED from .paymentrequest import InvoiceStore @@ -71,9 +67,6 @@ TX_STATUS = [ _('Local'), ] -TX_HEIGHT_LOCAL = -2 -TX_HEIGHT_UNCONF_PARENT = -1 -TX_HEIGHT_UNCONFIRMED = 0 def relayfee(network): @@ -140,7 +133,7 @@ def sweep(privkeys, network, config, recipient, fee=None, imax=100): inputs, keypairs = sweep_preparations(privkeys, network, imax) total = sum(i.get('value') for i in inputs) if fee is None: - outputs = [(TYPE_ADDRESS, recipient, total)] + outputs = [TxOutput(TYPE_ADDRESS, recipient, total)] tx = Transaction.from_io(inputs, outputs) fee = config.estimate_fee(tx.estimated_size()) if total - fee < 0: @@ -148,7 +141,7 @@ def sweep(privkeys, network, config, recipient, fee=None, imax=100): if total - fee < dust_threshold(network): raise Exception(_('Not enough funds on address.') + '\nTotal: %d satoshis\nFee: %d\nDust Threshold: %d'%(total, fee, dust_threshold(network))) - outputs = [(TYPE_ADDRESS, recipient, total - fee)] + outputs = [TxOutput(TYPE_ADDRESS, recipient, total - fee)] locktime = network.get_local_height() tx = Transaction.from_io(inputs, outputs, locktime=locktime) @@ -158,66 +151,34 @@ def sweep(privkeys, network, config, recipient, fee=None, imax=100): return tx -class AddTransactionException(Exception): - pass - - -class UnrelatedTransactionException(AddTransactionException): - def __str__(self): - return _("Transaction is unrelated to this wallet.") - class CannotBumpFee(Exception): pass -class Abstract_Wallet(PrintError): + + +class Abstract_Wallet(AddressSynchronizer): """ Wallet classes are created to handle various address generation methods. Completion states (watching-only, single account, no seed, etc) are handled inside classes. """ max_change_outputs = 3 + gap_limit_for_change = 6 + verbosity_filter = 'w' def __init__(self, storage): + AddressSynchronizer.__init__(self, storage) + self.electrum_version = ELECTRUM_VERSION - self.storage = storage - self.network = None - # verifier (SPV) and synchronizer are started in start_threads - self.synchronizer = None - self.verifier = None - - self.gap_limit_for_change = 6 # constant - - # locks: if you need to take multiple ones, acquire them in the order they are defined here! - self.lock = threading.RLock() - self.transaction_lock = threading.RLock() - # saved fields self.use_change = storage.get('use_change', True) self.multiple_change = storage.get('multiple_change', False) self.labels = storage.get('labels', {}) self.frozen_addresses = set(storage.get('frozen_addresses',[])) - self.history = storage.get('addr_history',{}) # address -> list(txid, height) self.fiat_value = storage.get('fiat_value', {}) self.receive_requests = storage.get('payment_requests', {}) - # Verified transactions. txid -> (height, timestamp, block_pos). Access with self.lock. - self.verified_tx = storage.get('verified_tx3', {}) - # Transactions pending verification. txid -> tx_height. Access with self.lock. - self.unverified_tx = defaultdict(int) - - self.load_keystore() - self.load_addresses() - self.test_addresses_sanity() - self.load_transactions() - self.load_local_history() - self.check_history() - self.load_unverified_transactions() - self.remove_local_transactions_we_dont_have() - - # wallet.up_to_date is true when the wallet is synchronized - self.up_to_date = False - # save wallet type the first time if self.storage.get('wallet_type') is None: self.storage.put('wallet_type', self.wallet_type) @@ -228,6 +189,11 @@ class Abstract_Wallet(PrintError): self.coin_price_cache = {} + def load_and_cleanup(self): + self.load_keystore() + self.load_addresses() + self.test_addresses_sanity() + super().load_and_cleanup() def diagnostic_name(self): return self.basename() @@ -238,102 +204,6 @@ class Abstract_Wallet(PrintError): def get_master_public_key(self): return None - @profiler - def load_transactions(self): - # load txi, txo, tx_fees - self.txi = self.storage.get('txi', {}) - for txid, d in list(self.txi.items()): - for addr, lst in d.items(): - self.txi[txid][addr] = set([tuple(x) for x in lst]) - self.txo = self.storage.get('txo', {}) - self.tx_fees = self.storage.get('tx_fees', {}) - tx_list = self.storage.get('transactions', {}) - # load transactions - self.transactions = {} - for tx_hash, raw in tx_list.items(): - tx = Transaction(raw) - self.transactions[tx_hash] = tx - if self.txi.get(tx_hash) is None and self.txo.get(tx_hash) is None: - self.print_error("removing unreferenced tx", tx_hash) - self.transactions.pop(tx_hash) - # load spent_outpoints - _spent_outpoints = self.storage.get('spent_outpoints', {}) - self.spent_outpoints = defaultdict(dict) - for prevout_hash, d in _spent_outpoints.items(): - for prevout_n_str, spending_txid in d.items(): - prevout_n = int(prevout_n_str) - self.spent_outpoints[prevout_hash][prevout_n] = spending_txid - - @profiler - def load_local_history(self): - self._history_local = {} # address -> set(txid) - for txid in itertools.chain(self.txi, self.txo): - self._add_tx_to_local_history(txid) - - def remove_local_transactions_we_dont_have(self): - txid_set = set(self.txi) | set(self.txo) - for txid in txid_set: - tx_height = self.get_tx_height(txid)[0] - if tx_height == TX_HEIGHT_LOCAL and txid not in self.transactions: - self.remove_transaction(txid) - - @profiler - def save_transactions(self, write=False): - with self.transaction_lock: - tx = {} - for k,v in self.transactions.items(): - tx[k] = str(v) - self.storage.put('transactions', tx) - self.storage.put('txi', self.txi) - self.storage.put('txo', self.txo) - self.storage.put('tx_fees', self.tx_fees) - self.storage.put('addr_history', self.history) - self.storage.put('spent_outpoints', self.spent_outpoints) - if write: - self.storage.write() - - def save_verified_tx(self, write=False): - with self.lock: - self.storage.put('verified_tx3', self.verified_tx) - if write: - self.storage.write() - - def clear_history(self): - with self.lock: - with self.transaction_lock: - self.txi = {} - self.txo = {} - self.tx_fees = {} - self.spent_outpoints = defaultdict(dict) - self.history = {} - self.verified_tx = {} - self.transactions = {} - self.save_transactions() - - @profiler - def check_history(self): - save = False - - hist_addrs_mine = list(filter(lambda k: self.is_mine(k), self.history.keys())) - hist_addrs_not_mine = list(filter(lambda k: not self.is_mine(k), self.history.keys())) - - for addr in hist_addrs_not_mine: - self.history.pop(addr) - save = True - - for addr in hist_addrs_mine: - hist = self.history[addr] - - for tx_hash, tx_height in hist: - if self.txi.get(tx_hash) or self.txo.get(tx_hash): - continue - tx = self.transactions.get(tx_hash) - if tx is not None: - self.add_transaction(tx_hash, tx, allow_unrelated=True) - save = True - if save: - self.save_transactions() - def basename(self): return os.path.basename(self.storage.path) @@ -358,19 +228,6 @@ class Abstract_Wallet(PrintError): def is_deterministic(self): return self.keystore.is_deterministic() - def set_up_to_date(self, up_to_date): - with self.lock: - self.up_to_date = up_to_date - if up_to_date: - self.save_transactions(write=True) - # if the verifier is also up to date, persist that too; - # otherwise it will persist its results when it finishes - if self.verifier and self.verifier.is_up_to_date(): - self.save_verified_tx(write=True) - - def is_up_to_date(self): - with self.lock: return self.up_to_date - def set_label(self, name, text = None): changed = False old_text = self.labels.get(name) @@ -415,7 +272,13 @@ class Abstract_Wallet(PrintError): return def is_mine(self, address): - return address in self.get_addresses() + if not super().is_mine(address): + return False + try: + self.get_address_index(address) + except KeyError: + return False + return True def is_change(self, address): if not self.is_mine(address): @@ -441,159 +304,9 @@ class Abstract_Wallet(PrintError): def get_public_keys(self, address): return [self.get_public_key(address)] - def add_unverified_tx(self, tx_hash, tx_height): - if tx_height in (TX_HEIGHT_UNCONFIRMED, TX_HEIGHT_UNCONF_PARENT) \ - and tx_hash in self.verified_tx: - with self.lock: - self.verified_tx.pop(tx_hash) - if self.verifier: - self.verifier.remove_spv_proof_for_tx(tx_hash) - - # tx will be verified only if height > 0 - if tx_hash not in self.verified_tx: - with self.lock: - self.unverified_tx[tx_hash] = tx_height - - def add_verified_tx(self, tx_hash, info): - # Remove from the unverified map and add to the verified map - with self.lock: - self.unverified_tx.pop(tx_hash, None) - self.verified_tx[tx_hash] = info # (tx_height, timestamp, pos) - height, conf, timestamp = self.get_tx_height(tx_hash) - self.network.trigger_callback('verified', tx_hash, height, conf, timestamp) - - def get_unverified_txs(self): - '''Returns a map from tx hash to transaction height''' - with self.lock: - return dict(self.unverified_tx) # copy - - def undo_verifications(self, blockchain, height): - '''Used by the verifier when a reorg has happened''' - txs = set() - with self.lock: - for tx_hash, item in list(self.verified_tx.items()): - tx_height, timestamp, pos = item - if tx_height >= height: - header = blockchain.read_header(tx_height) - # fixme: use block hash, not timestamp - if not header or header.get('timestamp') != timestamp: - self.verified_tx.pop(tx_hash, None) - txs.add(tx_hash) - return txs - - def get_local_height(self): - """ return last known height if we are offline """ - return self.network.get_local_height() if self.network else self.storage.get('stored_height', 0) - - def get_tx_height(self, tx_hash): - """ Given a transaction, returns (height, conf, timestamp) """ - with self.lock: - if tx_hash in self.verified_tx: - height, timestamp, pos = self.verified_tx[tx_hash] - conf = max(self.get_local_height() - height + 1, 0) - return height, conf, timestamp - elif tx_hash in self.unverified_tx: - height = self.unverified_tx[tx_hash] - return height, 0, None - else: - # local transaction - return TX_HEIGHT_LOCAL, 0, None - - def get_txpos(self, tx_hash): - "return position, even if the tx is unverified" - with self.lock: - if tx_hash in self.verified_tx: - height, timestamp, pos = self.verified_tx[tx_hash] - return height, pos - elif tx_hash in self.unverified_tx: - height = self.unverified_tx[tx_hash] - return (height, 0) if height > 0 else ((1e9 - height), 0) - else: - return (1e9+1, 0) - def is_found(self): return self.history.values() != [[]] * len(self.history) - def get_num_tx(self, address): - """ return number of transactions where address is involved """ - return len(self.history.get(address, [])) - - def get_tx_delta(self, tx_hash, address): - "effect of tx on address" - delta = 0 - # substract the value of coins sent from address - d = self.txi.get(tx_hash, {}).get(address, []) - for n, v in d: - delta -= v - # add the value of the coins received at address - d = self.txo.get(tx_hash, {}).get(address, []) - for n, v, cb in d: - delta += v - return delta - - def get_tx_value(self, txid): - " effect of tx on the entire domain" - delta = 0 - for addr, d in self.txi.get(txid, {}).items(): - for n, v in d: - delta -= v - for addr, d in self.txo.get(txid, {}).items(): - for n, v, cb in d: - delta += v - return delta - - def get_wallet_delta(self, tx): - """ effect of tx on wallet """ - is_relevant = False # "related to wallet?" - is_mine = False - is_pruned = False - is_partial = False - v_in = v_out = v_out_mine = 0 - for txin in tx.inputs(): - addr = self.get_txin_address(txin) - if self.is_mine(addr): - is_mine = True - is_relevant = True - d = self.txo.get(txin['prevout_hash'], {}).get(addr, []) - for n, v, cb in d: - if n == txin['prevout_n']: - value = v - break - else: - value = None - if value is None: - is_pruned = True - else: - v_in += value - else: - is_partial = True - if not is_mine: - is_partial = False - for addr, value in tx.get_outputs(): - v_out += value - if self.is_mine(addr): - v_out_mine += value - is_relevant = True - if is_pruned: - # some inputs are mine: - fee = None - if is_mine: - v = v_out_mine - v_out - else: - # no input is mine - v = v_out_mine - else: - v = v_out_mine - v_in - if is_partial: - # some inputs are mine, but not all - fee = None - else: - # all inputs are mine - fee = v_in - v_out - if not is_mine: - fee = None - return is_relevant, is_mine, v, fee - def get_tx_info(self, tx): is_relevant, is_mine, v, fee = self.get_wallet_delta(tx) exp_n = None @@ -605,7 +318,8 @@ class Abstract_Wallet(PrintError): if tx.is_complete(): if tx_hash in self.transactions.keys(): label = self.get_label(tx_hash) - height, conf, timestamp = self.get_tx_height(tx_hash) + tx_mined_status = self.get_tx_height(tx_hash) + height, conf = tx_mined_status.height, tx_mined_status.conf if height > 0: if conf: status = _("{} confirmations").format(conf) @@ -643,394 +357,21 @@ class Abstract_Wallet(PrintError): return tx_hash, status, label, can_broadcast, can_bump, amount, fee, height, conf, timestamp, exp_n - def get_addr_io(self, address): - h = self.get_address_history(address) - received = {} - sent = {} - for tx_hash, height in h: - l = self.txo.get(tx_hash, {}).get(address, []) - for n, v, is_cb in l: - received[tx_hash + ':%d'%n] = (height, v, is_cb) - for tx_hash, height in h: - l = self.txi.get(tx_hash, {}).get(address, []) - for txi, v in l: - sent[txi] = height - return received, sent - - def get_addr_utxo(self, address): - coins, spent = self.get_addr_io(address) - for txi in spent: - coins.pop(txi) - out = {} - for txo, v in coins.items(): - tx_height, value, is_cb = v - prevout_hash, prevout_n = txo.split(':') - x = { - 'address':address, - 'value':value, - 'prevout_n':int(prevout_n), - 'prevout_hash':prevout_hash, - 'height':tx_height, - 'coinbase':is_cb - } - out[txo] = x - return out - - # return the total amount ever received by an address - def get_addr_received(self, address): - received, sent = self.get_addr_io(address) - return sum([v for height, v, is_cb in received.values()]) - - # return the balance of a bitcoin address: confirmed and matured, unconfirmed, unmatured - def get_addr_balance(self, address): - received, sent = self.get_addr_io(address) - c = u = x = 0 - local_height = self.get_local_height() - for txo, (tx_height, v, is_cb) in received.items(): - if is_cb and tx_height + COINBASE_MATURITY > local_height: - x += v - elif tx_height > 0: - c += v - else: - u += v - if txo in sent: - if sent[txo] > 0: - c -= v - else: - u -= v - return c, u, x - def get_spendable_coins(self, domain, config): confirmed_only = config.get('confirmed_only', False) - return self.get_utxos(domain, exclude_frozen=True, mature=True, confirmed_only=confirmed_only) - - def get_utxos(self, domain = None, exclude_frozen = False, mature = False, confirmed_only = False): - coins = [] - if domain is None: - domain = self.get_addresses() - domain = set(domain) - if exclude_frozen: - domain = set(domain) - self.frozen_addresses - for addr in domain: - utxos = self.get_addr_utxo(addr) - for x in utxos.values(): - if confirmed_only and x['height'] <= 0: - continue - if mature and x['coinbase'] and x['height'] + COINBASE_MATURITY > self.get_local_height(): - continue - coins.append(x) - continue - return coins + return self.get_utxos(domain, excluded=self.frozen_addresses, mature=True, confirmed_only=confirmed_only) def dummy_address(self): return self.get_receiving_addresses()[0] - def get_addresses(self): - out = [] - out += self.get_receiving_addresses() - out += self.get_change_addresses() - return out - def get_frozen_balance(self): return self.get_balance(self.frozen_addresses) - def get_balance(self, domain=None): - if domain is None: - domain = self.get_addresses() - domain = set(domain) - cc = uu = xx = 0 - for addr in domain: - c, u, x = self.get_addr_balance(addr) - cc += c - uu += u - xx += x - return cc, uu, xx - - def get_address_history(self, addr): - h = [] - # we need self.transaction_lock but get_tx_height will take self.lock - # so we need to take that too here, to enforce order of locks - with self.lock, self.transaction_lock: - related_txns = self._history_local.get(addr, set()) - for tx_hash in related_txns: - tx_height = self.get_tx_height(tx_hash)[0] - h.append((tx_hash, tx_height)) - return h - - def _add_tx_to_local_history(self, txid): - with self.transaction_lock: - for addr in itertools.chain(self.txi.get(txid, []), self.txo.get(txid, [])): - cur_hist = self._history_local.get(addr, set()) - cur_hist.add(txid) - self._history_local[addr] = cur_hist - - def _remove_tx_from_local_history(self, txid): - with self.transaction_lock: - for addr in itertools.chain(self.txi.get(txid, []), self.txo.get(txid, [])): - cur_hist = self._history_local.get(addr, set()) - try: - cur_hist.remove(txid) - except KeyError: - pass - else: - self._history_local[addr] = cur_hist - - def get_txin_address(self, txi): - addr = txi.get('address') - if addr and addr != "(pubkey)": - return addr - prevout_hash = txi.get('prevout_hash') - prevout_n = txi.get('prevout_n') - dd = self.txo.get(prevout_hash, {}) - for addr, l in dd.items(): - for n, v, is_cb in l: - if n == prevout_n: - return addr - return None - - def get_txout_address(self, txo): - _type, x, v = txo - if _type == TYPE_ADDRESS: - addr = x - elif _type == TYPE_PUBKEY: - addr = bitcoin.public_key_to_p2pkh(bfh(x)) - else: - addr = None - return addr - - def get_conflicting_transactions(self, tx): - """Returns a set of transaction hashes from the wallet history that are - directly conflicting with tx, i.e. they have common outpoints being - spent with tx. If the tx is already in wallet history, that will not be - reported as a conflict. - """ - conflicting_txns = set() - with self.transaction_lock: - for txin in tx.inputs(): - if txin['type'] == 'coinbase': - continue - prevout_hash = txin['prevout_hash'] - prevout_n = txin['prevout_n'] - spending_tx_hash = self.spent_outpoints[prevout_hash].get(prevout_n) - if spending_tx_hash is None: - continue - # this outpoint has already been spent, by spending_tx - assert spending_tx_hash in self.transactions - conflicting_txns |= {spending_tx_hash} - txid = tx.txid() - if txid in conflicting_txns: - # this tx is already in history, so it conflicts with itself - if len(conflicting_txns) > 1: - raise Exception('Found conflicting transactions already in wallet history.') - conflicting_txns -= {txid} - return conflicting_txns - - def add_transaction(self, tx_hash, tx, allow_unrelated=False): - assert tx_hash, tx_hash - assert tx, tx - assert tx.is_complete() - # we need self.transaction_lock but get_tx_height will take self.lock - # so we need to take that too here, to enforce order of locks - with self.lock, self.transaction_lock: - # NOTE: returning if tx in self.transactions might seem like a good idea - # BUT we track is_mine inputs in a txn, and during subsequent calls - # of add_transaction tx, we might learn of more-and-more inputs of - # being is_mine, as we roll the gap_limit forward - is_coinbase = tx.inputs()[0]['type'] == 'coinbase' - tx_height = self.get_tx_height(tx_hash)[0] - if not allow_unrelated: - # note that during sync, if the transactions are not properly sorted, - # it could happen that we think tx is unrelated but actually one of the inputs is is_mine. - # this is the main motivation for allow_unrelated - is_mine = any([self.is_mine(self.get_txin_address(txin)) for txin in tx.inputs()]) - is_for_me = any([self.is_mine(self.get_txout_address(txo)) for txo in tx.outputs()]) - if not is_mine and not is_for_me: - raise UnrelatedTransactionException() - # Find all conflicting transactions. - # In case of a conflict, - # 1. confirmed > mempool > local - # 2. this new txn has priority over existing ones - # When this method exits, there must NOT be any conflict, so - # either keep this txn and remove all conflicting (along with dependencies) - # or drop this txn - conflicting_txns = self.get_conflicting_transactions(tx) - if conflicting_txns: - existing_mempool_txn = any( - self.get_tx_height(tx_hash2)[0] in (TX_HEIGHT_UNCONFIRMED, TX_HEIGHT_UNCONF_PARENT) - for tx_hash2 in conflicting_txns) - existing_confirmed_txn = any( - self.get_tx_height(tx_hash2)[0] > 0 - for tx_hash2 in conflicting_txns) - if existing_confirmed_txn and tx_height <= 0: - # this is a non-confirmed tx that conflicts with confirmed txns; drop. - return False - if existing_mempool_txn and tx_height == TX_HEIGHT_LOCAL: - # this is a local tx that conflicts with non-local txns; drop. - return False - # keep this txn and remove all conflicting - to_remove = set() - to_remove |= conflicting_txns - for conflicting_tx_hash in conflicting_txns: - to_remove |= self.get_depending_transactions(conflicting_tx_hash) - for tx_hash2 in to_remove: - self.remove_transaction(tx_hash2) - # add inputs - def add_value_from_prev_output(): - dd = self.txo.get(prevout_hash, {}) - # note: this nested loop takes linear time in num is_mine outputs of prev_tx - for addr, outputs in dd.items(): - # note: instead of [(n, v, is_cb), ...]; we could store: {n -> (v, is_cb)} - for n, v, is_cb in outputs: - if n == prevout_n: - if addr and self.is_mine(addr): - if d.get(addr) is None: - d[addr] = set() - d[addr].add((ser, v)) - return - self.txi[tx_hash] = d = {} - for txi in tx.inputs(): - if txi['type'] == 'coinbase': - continue - prevout_hash = txi['prevout_hash'] - prevout_n = txi['prevout_n'] - ser = prevout_hash + ':%d' % prevout_n - self.spent_outpoints[prevout_hash][prevout_n] = tx_hash - add_value_from_prev_output() - # add outputs - self.txo[tx_hash] = d = {} - for n, txo in enumerate(tx.outputs()): - v = txo[2] - ser = tx_hash + ':%d'%n - addr = self.get_txout_address(txo) - if addr and self.is_mine(addr): - if d.get(addr) is None: - d[addr] = [] - d[addr].append((n, v, is_coinbase)) - # give v to txi that spends me - next_tx = self.spent_outpoints[tx_hash].get(n) - if next_tx is not None: - dd = self.txi.get(next_tx, {}) - if dd.get(addr) is None: - dd[addr] = set() - if (ser, v) not in dd[addr]: - dd[addr].add((ser, v)) - self._add_tx_to_local_history(next_tx) - # add to local history - self._add_tx_to_local_history(tx_hash) - # save - self.transactions[tx_hash] = tx - return True - - def remove_transaction(self, tx_hash): - def remove_from_spent_outpoints(): - # undo spends in spent_outpoints - if tx is not None: # if we have the tx, this branch is faster - for txin in tx.inputs(): - if txin['type'] == 'coinbase': - continue - prevout_hash = txin['prevout_hash'] - prevout_n = txin['prevout_n'] - self.spent_outpoints[prevout_hash].pop(prevout_n, None) - if not self.spent_outpoints[prevout_hash]: - self.spent_outpoints.pop(prevout_hash) - else: # expensive but always works - for prevout_hash, d in list(self.spent_outpoints.items()): - for prevout_n, spending_txid in d.items(): - if spending_txid == tx_hash: - self.spent_outpoints[prevout_hash].pop(prevout_n, None) - if not self.spent_outpoints[prevout_hash]: - self.spent_outpoints.pop(prevout_hash) - # Remove this tx itself; if nothing spends from it. - # It is not so clear what to do if other txns spend from it, but it will be - # removed when those other txns are removed. - if not self.spent_outpoints[tx_hash]: - self.spent_outpoints.pop(tx_hash) - - with self.transaction_lock: - self.print_error("removing tx from history", tx_hash) - tx = self.transactions.pop(tx_hash, None) - remove_from_spent_outpoints() - self._remove_tx_from_local_history(tx_hash) - self.txi.pop(tx_hash, None) - self.txo.pop(tx_hash, None) - - def receive_tx_callback(self, tx_hash, tx, tx_height): - self.add_unverified_tx(tx_hash, tx_height) - self.add_transaction(tx_hash, tx, allow_unrelated=True) - - def receive_history_callback(self, addr, hist, tx_fees): - with self.lock: - old_hist = self.get_address_history(addr) - for tx_hash, height in old_hist: - if (tx_hash, height) not in hist: - # make tx local - self.unverified_tx.pop(tx_hash, None) - self.verified_tx.pop(tx_hash, None) - if self.verifier: - self.verifier.remove_spv_proof_for_tx(tx_hash) - self.history[addr] = hist - - for tx_hash, tx_height in hist: - # add it in case it was previously unconfirmed - self.add_unverified_tx(tx_hash, tx_height) - # if addr is new, we have to recompute txi and txo - tx = self.transactions.get(tx_hash) - if tx is None: - continue - self.add_transaction(tx_hash, tx, allow_unrelated=True) - - # Store fees - self.tx_fees.update(tx_fees) - - def get_history(self, domain=None): - # get domain - if domain is None: - domain = self.get_addresses() - domain = set(domain) - # 1. Get the history of each address in the domain, maintain the - # delta of a tx as the sum of its deltas on domain addresses - tx_deltas = defaultdict(int) - for addr in domain: - h = self.get_address_history(addr) - for tx_hash, height in h: - delta = self.get_tx_delta(tx_hash, addr) - if delta is None or tx_deltas[tx_hash] is None: - tx_deltas[tx_hash] = None - else: - tx_deltas[tx_hash] += delta - - # 2. create sorted history - history = [] - for tx_hash in tx_deltas: - delta = tx_deltas[tx_hash] - height, conf, timestamp = self.get_tx_height(tx_hash) - history.append((tx_hash, height, conf, timestamp, delta)) - history.sort(key = lambda x: self.get_txpos(x[0])) - history.reverse() - - # 3. add balance - c, u, x = self.get_balance(domain) - balance = c + u + x - h2 = [] - for tx_hash, height, conf, timestamp, delta in history: - h2.append((tx_hash, height, conf, timestamp, delta, balance)) - if balance is None or delta is None: - balance = None - else: - balance -= delta - h2.reverse() - - # fixme: this may happen if history is incomplete - if balance not in [None, 0]: - self.print_error("Error: history not synchronized") - return [] - - return h2 - def balance_at_timestamp(self, domain, target_timestamp): h = self.get_history(domain) - for tx_hash, height, conf, timestamp, value, balance in h: - if timestamp > target_timestamp: + balance = 0 + for tx_hash, tx_mined_status, value, balance in h: + if tx_mined_status.timestamp > target_timestamp: return balance - value # return last balance return balance @@ -1045,21 +386,23 @@ class Abstract_Wallet(PrintError): fiat_income = Decimal(0) fiat_expenditures = Decimal(0) h = self.get_history(domain) - for tx_hash, height, conf, timestamp, value, balance in h: - if from_timestamp and (timestamp or time.time()) < from_timestamp: + now = time.time() + for tx_hash, tx_mined_status, value, balance in h: + timestamp = tx_mined_status.timestamp + if from_timestamp and (timestamp or now) < from_timestamp: continue - if to_timestamp and (timestamp or time.time()) >= to_timestamp: + if to_timestamp and (timestamp or now) >= to_timestamp: continue item = { - 'txid':tx_hash, - 'height':height, - 'confirmations':conf, - 'timestamp':timestamp, + 'txid': tx_hash, + 'height': tx_mined_status.height, + 'confirmations': tx_mined_status.conf, + 'timestamp': timestamp, 'value': Satoshis(value), - 'balance': Satoshis(balance) + 'balance': Satoshis(balance), + 'date': timestamp_to_datetime(timestamp), + 'label': self.get_label(tx_hash), } - item['date'] = timestamp_to_datetime(timestamp) - item['label'] = self.get_label(tx_hash) if show_addresses: tx = self.transactions.get(tx_hash) item['inputs'] = list(map(lambda x: dict((k, x[k]) for k in ('prevout_hash', 'prevout_n')), tx.inputs())) @@ -1074,10 +417,9 @@ class Abstract_Wallet(PrintError): income += value # fiat computations if fx and fx.is_enabled(): - date = timestamp_to_datetime(timestamp) fiat_value = self.get_fiat_value(tx_hash, fx.ccy) fiat_default = fiat_value is None - fiat_value = fiat_value if fiat_value is not None else value / Decimal(COIN) * self.price_at_timestamp(tx_hash, fx.timestamp_rate) + fiat_value = fiat_value if fiat_value is not None else value / Decimal(COIN) * self.price_at_timestamp(tx_hash, fx.timestamp_rate) # item['fiat_value'] = Fiat(fiat_value, fx.ccy) item['fiat_default'] = fiat_default if value < 0: @@ -1144,9 +486,11 @@ class Abstract_Wallet(PrintError): return ', '.join(labels) return '' - def get_tx_status(self, tx_hash, height, conf, timestamp): - from .util import format_time + def get_tx_status(self, tx_hash, tx_mined_status): extra = [] + height = tx_mined_status.height + conf = tx_mined_status.conf + timestamp = tx_mined_status.timestamp if conf == 0: tx = self.transactions.get(tx_hash) if not tx: @@ -1193,11 +537,10 @@ class Abstract_Wallet(PrintError): # check outputs i_max = None for i, o in enumerate(outputs): - _type, data, value = o - if _type == TYPE_ADDRESS: - if not is_address(data): - raise Exception("Invalid bitcoin address: {}".format(data)) - if value == '!': + if o.type == TYPE_ADDRESS: + if not is_address(o.address): + raise Exception("Invalid bitcoin address: {}".format(o.address)) + if o.value == '!': if i_max is not None: raise Exception("More than one output set to spend max") i_max = i @@ -1248,14 +591,13 @@ class Abstract_Wallet(PrintError): else: # FIXME?? this might spend inputs with negative effective value... sendable = sum(map(lambda x:x['value'], inputs)) - _type, data, value = outputs[i_max] - outputs[i_max] = (_type, data, 0) + outputs[i_max] = outputs[i_max]._replace(value=0) tx = Transaction.from_io(inputs, outputs[:]) fee = fee_estimator(tx.estimated_size()) amount = sendable - tx.output_value() - fee if amount < 0: raise NotEnoughFunds() - outputs[i_max] = (_type, data, amount) + outputs[i_max] = outputs[i_max]._replace(value=amount) tx = Transaction.from_io(inputs, outputs[:]) # Sort the inputs and outputs deterministically @@ -1285,36 +627,6 @@ class Abstract_Wallet(PrintError): return True return False - def load_unverified_transactions(self): - # review transactions that are in the history - for addr, hist in self.history.items(): - for tx_hash, tx_height in hist: - # add it in case it was previously unconfirmed - self.add_unverified_tx(tx_hash, tx_height) - - def start_threads(self, network): - self.network = network - if self.network is not None: - self.verifier = SPV(self.network, self) - self.synchronizer = Synchronizer(self, network) - network.add_jobs([self.verifier, self.synchronizer]) - else: - self.verifier = None - self.synchronizer = None - - def stop_threads(self): - if self.network: - self.network.remove_jobs([self.synchronizer, self.verifier]) - self.synchronizer.release() - self.synchronizer = None - self.verifier = None - # Now no references to the synchronizer or verifier - # remain so they will be GC-ed - self.storage.put('stored_height', self.get_local_height()) - self.save_transactions() - self.save_verified_tx() - self.storage.write() - def wait_until_synchronized(self, callback=None): def wait_for_wallet(): self.set_up_to_date(False) @@ -1343,17 +655,6 @@ class Abstract_Wallet(PrintError): def can_export(self): return not self.is_watching_only() and hasattr(self.keystore, 'get_private_key') - def is_used(self, address): - h = self.history.get(address,[]) - if len(h) == 0: - return False - c, u, x = self.get_addr_balance(address) - return c + u + x == 0 - - def is_empty(self, address): - c, u, x = self.get_addr_balance(address) - return c+u+x == 0 - def address_is_old(self, address, age_limit=2): age = -1 h = self.history.get(address, []) @@ -1390,14 +691,13 @@ class Abstract_Wallet(PrintError): s = sorted(s, key=lambda x: x[2]) for o in s: i = outputs.index(o) - otype, address, value = o - if value - delta >= self.dust_threshold(): - outputs[i] = otype, address, value - delta + if o.value - delta >= self.dust_threshold(): + outputs[i] = o._replace(value=o.value-delta) delta = 0 break else: del outputs[i] - delta -= value + delta -= o.value if delta > 0: continue if delta > 0: @@ -1410,8 +710,8 @@ class Abstract_Wallet(PrintError): def cpfp(self, tx, fee): txid = tx.txid() for i, o in enumerate(tx.outputs()): - otype, address, value = o - if otype == TYPE_ADDRESS and self.is_mine(address): + address, value = o.address, o.value + if o.type == TYPE_ADDRESS and self.is_mine(address): break else: return @@ -1421,7 +721,7 @@ class Abstract_Wallet(PrintError): return self.add_input_info(item) inputs = [item] - outputs = [(TYPE_ADDRESS, address, value - fee)] + outputs = [TxOutput(TYPE_ADDRESS, address, value - fee)] locktime = self.get_local_height() # note: no need to call tx.BIP_LI01_sort() here - single input/output return Transaction.from_io(inputs, outputs, locktime=locktime) @@ -1541,8 +841,7 @@ class Abstract_Wallet(PrintError): txid, n = txo.split(':') info = self.verified_tx.get(txid) if info: - tx_height, timestamp, pos = info - conf = local_height - tx_height + conf = local_height - info.height else: conf = 0 l.append((conf, v)) @@ -1605,7 +904,7 @@ class Abstract_Wallet(PrintError): expiration = 0 conf = None if amount: - if self.up_to_date: + if self.is_up_to_date(): paid, conf = self.get_payment_status(address, amount) status = PR_PAID if paid else PR_UNPAID if status == PR_UNPAID and expiration is not None and time.time() > timestamp + expiration: @@ -1700,12 +999,6 @@ class Abstract_Wallet(PrintError): def can_delete_address(self): return False - def add_address(self, address): - if address not in self.history: - self.history[address] = [] - if self.synchronizer: - self.synchronizer.add(address) - def has_password(self): return self.has_keystore_encryption() or self.has_storage_encryption() @@ -1799,7 +1092,7 @@ class Abstract_Wallet(PrintError): def price_at_timestamp(self, txid, price_func): """Returns fiat price of bitcoin at the time tx got confirmed.""" - height, conf, timestamp = self.get_tx_height(txid) + timestamp = self.get_tx_height(txid).timestamp return price_func(timestamp if timestamp else time.time()) def unrealized_gains(self, domain, price_func, ccy): @@ -1923,13 +1216,11 @@ class Imported_Wallet(Simple_Wallet): def is_beyond_limit(self, address): return False - def is_mine(self, address): - return address in self.addresses - def get_fingerprint(self): return '' - def get_addresses(self, include_change=False): + def get_addresses(self): + # note: overridden so that the history can be cleared return sorted(self.addresses.keys()) def get_receiving_addresses(self): @@ -1944,9 +1235,9 @@ class Imported_Wallet(Simple_Wallet): if address in self.addresses: return '' self.addresses[address] = {} - self.storage.put('addresses', self.addresses) - self.storage.write() self.add_address(address) + self.save_addresses() + self.save_transactions(write=True) return address def delete_address(self, address): @@ -1972,7 +1263,7 @@ class Imported_Wallet(Simple_Wallet): self.verified_tx.pop(tx_hash, None) self.unverified_tx.pop(tx_hash, None) self.transactions.pop(tx_hash, None) - self.storage.put('verified_tx3', self.verified_tx) + self.save_verified_tx() self.save_transactions() self.set_label(address, None) @@ -1994,7 +1285,7 @@ class Imported_Wallet(Simple_Wallet): else: self.keystore.delete_imported_key(pubkey) self.save_keystore() - self.storage.put('addresses', self.addresses) + self.save_addresses() self.storage.write() @@ -2022,9 +1313,9 @@ class Imported_Wallet(Simple_Wallet): raise NotImplementedError(txin_type) self.addresses[addr] = {'type':txin_type, 'pubkey':pubkey, 'redeem_script':redeem_script} self.save_keystore() - self.save_addresses() - self.storage.write() self.add_address(addr) + self.save_addresses() + self.save_transactions(write=True) return addr def get_redeem_script(self, address): @@ -2063,6 +1354,14 @@ class Deterministic_Wallet(Abstract_Wallet): def has_seed(self): return self.keystore.has_seed() + def get_addresses(self): + # note: overridden so that the history can be cleared. + # addresses are ordered based on derivation + out = [] + out += self.get_receiving_addresses() + out += self.get_change_addresses() + return out + def get_receiving_addresses(self): return self.receiving_addresses @@ -2164,9 +1463,6 @@ class Deterministic_Wallet(Abstract_Wallet): return False return True - def is_mine(self, address): - return address in self._addr_to_addr_index - def get_address_index(self, address): return self._addr_to_addr_index[address] @@ -2371,4 +1667,4 @@ class Wallet(object): return Multisig_Wallet if wallet_type in wallet_constructors: return wallet_constructors[wallet_type] - raise RuntimeError("Unknown wallet type: " + str(wallet_type)) + raise WalletFileException("Unknown wallet type: " + str(wallet_type)) diff --git a/icons.qrc b/icons.qrc index 82cfba32..feb07a02 100644 --- a/icons.qrc +++ b/icons.qrc @@ -31,6 +31,8 @@ icons/qrcode.png icons/qrcode_white.png icons/preferences.png + icons/safe-t_unpaired.png + icons/safe-t.png icons/seed.png icons/status_connected.png icons/status_connected_proxy.png diff --git a/icons/safe-t.png b/icons/safe-t.png new file mode 100644 index 00000000..7a574dec Binary files /dev/null and b/icons/safe-t.png differ diff --git a/icons/safe-t_unpaired.png b/icons/safe-t_unpaired.png new file mode 100644 index 00000000..c67a344f Binary files /dev/null and b/icons/safe-t_unpaired.png differ diff --git a/run_electrum b/run_electrum index b5d5af00..46654ec1 100755 --- a/run_electrum +++ b/run_electrum @@ -46,7 +46,6 @@ def check_imports(): import ecdsa import requests import qrcode - import pbkdf2 import google.protobuf import jsonrpclib except ImportError as e: @@ -356,7 +355,7 @@ if __name__ == '__main__': # config is an object passed to the various constructors (wallet, interface, gui) if is_android: config_options = { - 'verbose': True, + 'verbosity': '', 'cmd': 'gui', 'gui': 'kivy', } @@ -377,7 +376,7 @@ if __name__ == '__main__': config_options['electrum_path'] = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'electrum_data') # kivy sometimes freezes when we write to sys.stderr - set_verbosity(config_options.get('verbose') and config_options.get('gui')!='kivy') + set_verbosity(config_options.get('verbosity') if config_options.get('gui') != 'kivy' else '') # check uri uri = config_options.get('url') diff --git a/setup.py b/setup.py index 937375fb..18c5cb88 100755 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ # python setup.py sdist --format=zip,gztar -from setuptools import setup +from setuptools import setup, find_packages import os import sys import platform @@ -43,7 +43,6 @@ if platform.system() in ['Linux', 'FreeBSD', 'DragonFly']: extras_require = { 'hardware': requirements_hw, 'fast': ['pycryptodomex'], - ':python_version < "3.5"': ['typing>=3.0.0'], } extras_require['full'] = extras_require['hardware'] + extras_require['fast'] @@ -58,28 +57,15 @@ setup( 'electrum.gui', 'electrum.gui.qt', 'electrum.plugins', - 'electrum.plugins.audio_modem', - 'electrum.plugins.cosigner_pool', - 'electrum.plugins.email_requests', - 'electrum.plugins.greenaddress_instant', - 'electrum.plugins.hw_wallet', - 'electrum.plugins.keepkey', - 'electrum.plugins.labels', - 'electrum.plugins.ledger', - 'electrum.plugins.revealer', - 'electrum.plugins.trezor', - 'electrum.plugins.digitalbitbox', - 'electrum.plugins.trustedcoin', - 'electrum.plugins.virtualkeyboard', - ], + ] + [('electrum.plugins.'+pkg) for pkg in find_packages('electrum/plugins')], package_dir={ 'electrum': 'electrum' }, package_data={ '': ['*.txt', '*.json', '*.ttf', '*.otf'], 'electrum': [ - 'electrum/wordlist/*.txt', - 'electrum/locale/*/LC_MESSAGES/electrum.mo', + 'wordlist/*.txt', + 'locale/*/LC_MESSAGES/electrum.mo', ], }, scripts=['electrum/electrum'], diff --git a/tox.ini b/tox.ini index 9a245bbc..b00d2923 100644 --- a/tox.ini +++ b/tox.ini @@ -6,7 +6,7 @@ deps= pytest coverage commands= - coverage run --source=electrum -m py.test -v + coverage run --source=electrum '--omit=electrum/gui/*,electrum/plugins/*,electrum/scripts/*,electrum/tests/*' -m py.test -v coverage report extras= fast