From 73fee2fefaec23ef0b529479f8b69b8fb1dcaaae Mon Sep 17 00:00:00 2001 From: Filip Gospodinov Date: Tue, 10 Jul 2018 13:33:46 +0200 Subject: [PATCH 01/56] build-wine: allow local testing Before, it was only possible to test commits that are on Github (pull request or merged). Now, changes can be tested locally too. This introduces the risk that a release could be built containing uncommitted changes which by definition breaks deterministic builds. Fortunately, this will always be detected because the version string is created using `git describe --tags --dirty`. Also, retire $TARGET variable because it decouples the build scripts from the commit revision to be built. This is a problem for deterministic builds. --- .travis.yml | 4 ++-- contrib/build-wine/build-electrum-git.sh | 26 ++++-------------------- contrib/build-wine/build.sh | 6 +----- contrib/build-wine/docker/README.md | 8 ++++---- contrib/build-wine/electrum.nsi | 4 ++-- contrib/build-wine/prepare-wine.sh | 5 ----- 6 files changed, 13 insertions(+), 40 deletions(-) diff --git a/.travis.yml b/.travis.yml index a68aaed8..63dde547 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,7 +33,7 @@ jobs: install: - sudo docker build --no-cache -t electrum-wine-builder-img ./contrib/build-wine/docker/ script: - - sudo docker run --name electrum-wine-builder-cont -v $PWD:/opt/electrum --rm --workdir /opt/electrum/contrib/build-wine electrum-wine-builder-img ./build.sh $TRAVIS_COMMIT + - sudo docker run --name electrum-wine-builder-cont -v $PWD:/opt/wine64/drive_c/electrum --rm --workdir /opt/wine64/drive_c/electrum/contrib/build-wine electrum-wine-builder-img ./build.sh after_success: true - os: osx language: c @@ -53,4 +53,4 @@ jobs: script: - ./contrib/deterministic-build/check_submodules.sh after_success: true - if: tag IS present \ No newline at end of file + if: tag IS present diff --git a/contrib/build-wine/build-electrum-git.sh b/contrib/build-wine/build-electrum-git.sh index 86897069..2a103eba 100755 --- a/contrib/build-wine/build-electrum-git.sh +++ b/contrib/build-wine/build-electrum-git.sh @@ -19,23 +19,7 @@ set -e mkdir -p tmp cd tmp -if [ -d ./electrum ]; then - rm ./electrum -rf -fi - -git clone https://github.com/spesmilo/electrum -b master - -pushd electrum -if [ ! -z "$1" ]; then - # a commit/tag/branch was specified - if ! git cat-file -e "$1" 2> /dev/null - then # can't find target - # try pull requests - git config --local --add remote.origin.fetch '+refs/pull/*/merge:refs/remotes/origin/pr/*' - git fetch --all - fi - git checkout $1 -fi +pushd $WINEPREFIX/drive_c/electrum # Load electrum-icons and electrum-locale for this release git submodule init @@ -59,11 +43,9 @@ popd find -exec touch -d '2000-11-11T11:11:11+00:00' {} + popd -rm -rf $WINEPREFIX/drive_c/electrum -cp -r electrum $WINEPREFIX/drive_c/electrum -cp electrum/LICENCE . -cp -r ./electrum/contrib/deterministic-build/electrum-locale/locale $WINEPREFIX/drive_c/electrum/lib/ -cp ./electrum/contrib/deterministic-build/electrum-icons/icons_rc.py $WINEPREFIX/drive_c/electrum/gui/qt/ +cp $WINEPREFIX/drive_c/electrum/LICENCE . +cp -r $WINEPREFIX/drive_c/electrum/contrib/deterministic-build/electrum-locale/locale $WINEPREFIX/drive_c/electrum/lib/ +cp $WINEPREFIX/drive_c/electrum/contrib/deterministic-build/electrum-icons/icons_rc.py $WINEPREFIX/drive_c/electrum/gui/qt/ # Install frozen dependencies $PYTHON -m pip install -r ../../deterministic-build/requirements.txt diff --git a/contrib/build-wine/build.sh b/contrib/build-wine/build.sh index ef4012b4..01ca071f 100755 --- a/contrib/build-wine/build.sh +++ b/contrib/build-wine/build.sh @@ -2,10 +2,6 @@ # Lucky number export PYTHONHASHSEED=22 -if [ ! -z "$1" ]; then - to_build="$1" -fi - here=$(dirname "$0") test -n "$here" -a -d "$here" || exit @@ -28,5 +24,5 @@ find -exec touch -d '2000-11-11T11:11:11+00:00' {} + popd ls -l /opt/wine64/drive_c/python* -$here/build-electrum-git.sh $to_build && \ +$here/build-electrum-git.sh && \ echo "Done." diff --git a/contrib/build-wine/docker/README.md b/contrib/build-wine/docker/README.md index ae7e0365..7cab0f58 100644 --- a/contrib/build-wine/docker/README.md +++ b/contrib/build-wine/docker/README.md @@ -25,14 +25,14 @@ folder. 3. Build Windows binaries ``` - $ TARGET=master + $ git checkout $REV $ sudo docker run \ --name electrum-wine-builder-cont \ - -v .:/opt/electrum \ + -v $PWD:/opt/wine64/drive_c/electrum \ --rm \ - --workdir /opt/electrum/contrib/build-wine \ + --workdir /opt/wine64/drive_c/electrum/contrib/build-wine \ electrum-wine-builder-img \ - ./build.sh $TARGET + ./build.sh ``` 4. The generated binaries are in `./contrib/build-wine/dist`. diff --git a/contrib/build-wine/electrum.nsi b/contrib/build-wine/electrum.nsi index 59e90d75..8d2482e2 100644 --- a/contrib/build-wine/electrum.nsi +++ b/contrib/build-wine/electrum.nsi @@ -72,7 +72,7 @@ !define MUI_ABORTWARNING !define MUI_ABORTWARNING_TEXT "Are you sure you wish to abort the installation of ${PRODUCT_NAME}?" - !define MUI_ICON "tmp\electrum\icons\electrum.ico" + !define MUI_ICON "c:\electrum\icons\electrum.ico" ;-------------------------------- ;Pages @@ -111,7 +111,7 @@ Section ;Files to pack into the installer File /r "dist\electrum\*.*" - File "..\..\icons\electrum.ico" + File "c:\electrum\icons\electrum.ico" ;Store installation folder WriteRegStr HKCU "Software\${PRODUCT_NAME}" "" $INSTDIR diff --git a/contrib/build-wine/prepare-wine.sh b/contrib/build-wine/prepare-wine.sh index 4af3c9f5..2fe1d1d0 100755 --- a/contrib/build-wine/prepare-wine.sh +++ b/contrib/build-wine/prepare-wine.sh @@ -79,11 +79,6 @@ retry() { here=$(dirname $(readlink -e $0)) set -e -# Clean up Wine environment -echo "Cleaning $WINEPREFIX" -rm -rf $WINEPREFIX -echo "done" - wine 'wineboot' cd /tmp/electrum-build From b8ab36546dad3b0774222b1f2e0bf61628c0d490 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Sat, 14 Jul 2018 18:45:02 +0200 Subject: [PATCH 02/56] mempool fees: increase estimate by max precision of histogram related: #4551 --- electrum/simple_config.py | 19 +++++++++++++-- electrum/tests/test_simple_config.py | 36 ++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/electrum/simple_config.py b/electrum/simple_config.py index 03558cd7..9063f1a1 100644 --- a/electrum/simple_config.py +++ b/electrum/simple_config.py @@ -5,6 +5,7 @@ import os import stat from decimal import Decimal from typing import Union +from numbers import Real from copy import deepcopy @@ -310,7 +311,11 @@ class SimpleConfig(PrintError): fee = int(fee) return fee - def fee_to_depth(self, target_fee): + def fee_to_depth(self, target_fee: Real) -> int: + """For a given sat/vbyte fee, returns an estimate of how deep + it would be in the current mempool in vbytes. + Pessimistic == overestimates the depth. + """ depth = 0 for fee, s in self.mempool_fees: depth += s @@ -320,10 +325,16 @@ class SimpleConfig(PrintError): return 0 return depth - @impose_hard_limits_on_fee def depth_to_fee(self, slider_pos) -> int: """Returns fee in sat/kbyte.""" target = self.depth_target(slider_pos) + return self.depth_target_to_fee(target) + + @impose_hard_limits_on_fee + def depth_target_to_fee(self, target: int) -> int: + """Returns fee in sat/kbyte. + target: desired mempool depth in sat/vbyte + """ depth = 0 for fee, s in self.mempool_fees: depth += s @@ -331,6 +342,10 @@ class SimpleConfig(PrintError): break else: return 0 + # add one sat/byte as currently that is + # the max precision of the histogram + fee += 1 + # convert to sat/kbyte return fee * 1000 def depth_target(self, slider_pos): diff --git a/electrum/tests/test_simple_config.py b/electrum/tests/test_simple_config.py index a4c79312..14ea7baf 100644 --- a/electrum/tests/test_simple_config.py +++ b/electrum/tests/test_simple_config.py @@ -110,6 +110,42 @@ class Test_SimpleConfig(SequentialTestCase): result.pop('config_version', None) self.assertEqual({"something": "a"}, result) + def test_depth_target_to_fee(self): + config = SimpleConfig({}) + config.mempool_fees = [[49, 100110], [10, 121301], [6, 153731], [5, 125872], [1, 36488810]] + self.assertEqual( 2 * 1000, config.depth_target_to_fee(1000000)) + self.assertEqual( 6 * 1000, config.depth_target_to_fee( 500000)) + self.assertEqual( 7 * 1000, config.depth_target_to_fee( 250000)) + self.assertEqual(11 * 1000, config.depth_target_to_fee( 200000)) + self.assertEqual(50 * 1000, config.depth_target_to_fee( 100000)) + config.mempool_fees = [] + self.assertEqual( 1 * 1000, config.depth_target_to_fee(10 ** 5)) + self.assertEqual( 1 * 1000, config.depth_target_to_fee(10 ** 6)) + self.assertEqual( 1 * 1000, config.depth_target_to_fee(10 ** 7)) + config.mempool_fees = [[1, 36488810]] + self.assertEqual( 2 * 1000, config.depth_target_to_fee(10 ** 5)) + self.assertEqual( 2 * 1000, config.depth_target_to_fee(10 ** 6)) + self.assertEqual( 2 * 1000, config.depth_target_to_fee(10 ** 7)) + self.assertEqual( 1 * 1000, config.depth_target_to_fee(10 ** 8)) + config.mempool_fees = [[5, 125872], [1, 36488810]] + self.assertEqual( 6 * 1000, config.depth_target_to_fee(10 ** 5)) + self.assertEqual( 2 * 1000, config.depth_target_to_fee(10 ** 6)) + self.assertEqual( 2 * 1000, config.depth_target_to_fee(10 ** 7)) + self.assertEqual( 1 * 1000, config.depth_target_to_fee(10 ** 8)) + + def test_fee_to_depth(self): + config = SimpleConfig({}) + config.mempool_fees = [[49, 100000], [10, 120000], [6, 150000], [5, 125000], [1, 36000000]] + self.assertEqual(100000, config.fee_to_depth(500)) + self.assertEqual(100000, config.fee_to_depth(50)) + self.assertEqual(100000, config.fee_to_depth(49)) + self.assertEqual(220000, config.fee_to_depth(48)) + self.assertEqual(220000, config.fee_to_depth(10)) + self.assertEqual(370000, config.fee_to_depth(9)) + self.assertEqual(370000, config.fee_to_depth(6.5)) + self.assertEqual(370000, config.fee_to_depth(6)) + self.assertEqual(495000, config.fee_to_depth(5.5)) + class TestUserConfig(SequentialTestCase): From 8bb59fcc3c235d971df9669afc1d7ac5a226e6ba Mon Sep 17 00:00:00 2001 From: SomberNight Date: Sat, 14 Jul 2018 18:54:27 +0200 Subject: [PATCH 03/56] follow-up prev: fix bug in fee_to_depth, and typo and tests --- electrum/simple_config.py | 4 +--- electrum/tests/test_simple_config.py | 5 +++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/electrum/simple_config.py b/electrum/simple_config.py index 9063f1a1..e4d75dae 100644 --- a/electrum/simple_config.py +++ b/electrum/simple_config.py @@ -321,8 +321,6 @@ class SimpleConfig(PrintError): depth += s if fee <= target_fee: break - else: - return 0 return depth def depth_to_fee(self, slider_pos) -> int: @@ -333,7 +331,7 @@ class SimpleConfig(PrintError): @impose_hard_limits_on_fee def depth_target_to_fee(self, target: int) -> int: """Returns fee in sat/kbyte. - target: desired mempool depth in sat/vbyte + target: desired mempool depth in vbytes """ depth = 0 for fee, s in self.mempool_fees: diff --git a/electrum/tests/test_simple_config.py b/electrum/tests/test_simple_config.py index 14ea7baf..6a3dbd02 100644 --- a/electrum/tests/test_simple_config.py +++ b/electrum/tests/test_simple_config.py @@ -111,7 +111,7 @@ class Test_SimpleConfig(SequentialTestCase): self.assertEqual({"something": "a"}, result) def test_depth_target_to_fee(self): - config = SimpleConfig({}) + config = SimpleConfig(self.options) config.mempool_fees = [[49, 100110], [10, 121301], [6, 153731], [5, 125872], [1, 36488810]] self.assertEqual( 2 * 1000, config.depth_target_to_fee(1000000)) self.assertEqual( 6 * 1000, config.depth_target_to_fee( 500000)) @@ -134,7 +134,7 @@ class Test_SimpleConfig(SequentialTestCase): self.assertEqual( 1 * 1000, config.depth_target_to_fee(10 ** 8)) def test_fee_to_depth(self): - config = SimpleConfig({}) + config = SimpleConfig(self.options) config.mempool_fees = [[49, 100000], [10, 120000], [6, 150000], [5, 125000], [1, 36000000]] self.assertEqual(100000, config.fee_to_depth(500)) self.assertEqual(100000, config.fee_to_depth(50)) @@ -145,6 +145,7 @@ class Test_SimpleConfig(SequentialTestCase): self.assertEqual(370000, config.fee_to_depth(6.5)) self.assertEqual(370000, config.fee_to_depth(6)) self.assertEqual(495000, config.fee_to_depth(5.5)) + self.assertEqual(36495000, config.fee_to_depth(0.5)) class TestUserConfig(SequentialTestCase): From cf7caa7ef9bf10ea8a50ad2146dfa94d605b5394 Mon Sep 17 00:00:00 2001 From: Johann Bauer Date: Sun, 15 Jul 2018 19:25:28 +0200 Subject: [PATCH 04/56] Don't measure coverage for files in gui or plugins --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 9a245bbc..87053dcb 100644 --- a/tox.ini +++ b/tox.ini @@ -6,7 +6,7 @@ deps= pytest coverage commands= - coverage run --source=electrum -m py.test -v + coverage run --source=electrum '--omit=electrum/gui/*,electrum/plugins/*,electrum/scripts/*' -m py.test -v coverage report extras= fast From c856633b9c9c9eaca00125ca32a0bb498b04e26f Mon Sep 17 00:00:00 2001 From: Marcel O'Neil Date: Mon, 16 Jul 2018 00:58:41 -0400 Subject: [PATCH 05/56] remove test files from coverage --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 87053dcb..b00d2923 100644 --- a/tox.ini +++ b/tox.ini @@ -6,7 +6,7 @@ deps= pytest coverage commands= - coverage run --source=electrum '--omit=electrum/gui/*,electrum/plugins/*,electrum/scripts/*' -m py.test -v + coverage run --source=electrum '--omit=electrum/gui/*,electrum/plugins/*,electrum/scripts/*,electrum/tests/*' -m py.test -v coverage report extras= fast From 21204fc5528f496997ec3fd70222ac994a878d44 Mon Sep 17 00:00:00 2001 From: Marcel O'Neil Date: Tue, 17 Jul 2018 22:09:04 -0400 Subject: [PATCH 06/56] localization: fix download link + badge --- README.rst | 4 ++-- contrib/make_locale | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 4a201e81..c7665604 100644 --- a/README.rst +++ b/README.rst @@ -15,9 +15,9 @@ Electrum - Lightweight Bitcoin client .. image:: https://coveralls.io/repos/github/spesmilo/electrum/badge.svg?branch=master :target: https://coveralls.io/github/spesmilo/electrum?branch=master :alt: Test coverage statistics -.. image:: https://img.shields.io/badge/help-translating-blue.svg +.. image:: https://d322cqt584bo4o.cloudfront.net/electrum/localized.svg :target: https://crowdin.com/project/electrum - :alt: Help translating Electrum online + :alt: Help translate Electrum online diff --git a/contrib/make_locale b/contrib/make_locale index 052f5d87..3c28d570 100755 --- a/contrib/make_locale +++ b/contrib/make_locale @@ -54,7 +54,7 @@ if crowdin_api_key: # Download & unzip print('Download translations') -s = requests.request('GET', 'https://crowdin.com/download/project/' + crowdin_identifier + '.zip').content +s = requests.request('GET', 'https://crowdin.com/backend/download/project/' + crowdin_identifier + '.zip').content zfobj = zipfile.ZipFile(io.BytesIO(s)) print('Unzip translations') From e3888752d6508ebbb9b4e9a8a09555914cd68fd2 Mon Sep 17 00:00:00 2001 From: ThomasV Date: Wed, 18 Jul 2018 11:18:57 +0200 Subject: [PATCH 07/56] separate address synchronizer from wallet --- electrum/__init__.py | 2 +- electrum/address_synchronizer.py | 494 +++++++++++++++++++++++++ electrum/gui/qt/history_list.py | 2 +- electrum/gui/qt/main_window.py | 3 +- electrum/gui/qt/transaction_dialog.py | 1 - electrum/tests/test_wallet_vertical.py | 3 +- electrum/wallet.py | 485 +----------------------- 7 files changed, 509 insertions(+), 481 deletions(-) create mode 100644 electrum/address_synchronizer.py diff --git a/electrum/__init__.py b/electrum/__init__.py index d8df161f..48a60c15 100644 --- a/electrum/__init__.py +++ b/electrum/__init__.py @@ -1,6 +1,6 @@ from .version import ELECTRUM_VERSION from .util import format_satoshis, print_msg, print_error, set_verbosity -from .wallet import Synchronizer, Wallet +from .wallet import Wallet from .storage import WalletStorage from .coinchooser import COIN_CHOOSERS from .network import Network, pick_random_server diff --git a/electrum/address_synchronizer.py b/electrum/address_synchronizer.py new file mode 100644 index 00000000..bbc8b0b2 --- /dev/null +++ b/electrum/address_synchronizer.py @@ -0,0 +1,494 @@ +# Electrum - lightweight Bitcoin client +# Copyright (C) 2018 The Electrum Developers +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation files +# (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, +# publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS +# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import threading +import itertools +from collections import defaultdict + +from .util import PrintError, profiler +from .transaction import Transaction +from .synchronizer import Synchronizer +from .verifier import SPV + +TX_HEIGHT_LOCAL = -2 +TX_HEIGHT_UNCONF_PARENT = -1 +TX_HEIGHT_UNCONFIRMED = 0 + +class AddTransactionException(Exception): + pass + + +class UnrelatedTransactionException(AddTransactionException): + def __str__(self): + return _("Transaction is unrelated to this wallet.") + +class AddressSynchronizer(PrintError): + """ + inherited by wallet + """ + + def __init__(self, storage): + self.storage = storage + self.network = None + # verifier (SPV) and synchronizer are started in start_threads + self.synchronizer = None + self.verifier = None + # locks: if you need to take multiple ones, acquire them in the order they are defined here! + self.lock = threading.RLock() + self.transaction_lock = threading.RLock() + # address -> list(txid, height) + self.history = storage.get('addr_history',{}) + # Verified transactions. txid -> (height, timestamp, block_pos). Access with self.lock. + self.verified_tx = storage.get('verified_tx3', {}) + # Transactions pending verification. txid -> tx_height. Access with self.lock. + self.unverified_tx = defaultdict(int) + # true when synchronized + self.up_to_date = False + self.load_transactions() + self.load_local_history() + self.load_unverified_transactions() + self.remove_local_transactions_we_dont_have() + + def load_unverified_transactions(self): + # review transactions that are in the history + for addr, hist in self.history.items(): + for tx_hash, tx_height in hist: + # add it in case it was previously unconfirmed + self.add_unverified_tx(tx_hash, tx_height) + + def start_threads(self, network): + self.network = network + if self.network is not None: + self.verifier = SPV(self.network, self) + self.synchronizer = Synchronizer(self, network) + network.add_jobs([self.verifier, self.synchronizer]) + else: + self.verifier = None + self.synchronizer = None + + def stop_threads(self): + if self.network: + self.network.remove_jobs([self.synchronizer, self.verifier]) + self.synchronizer.release() + self.synchronizer = None + self.verifier = None + # Now no references to the synchronizer or verifier + # remain so they will be GC-ed + self.storage.put('stored_height', self.get_local_height()) + self.save_transactions() + self.save_verified_tx() + self.storage.write() + + def add_address(self, address): + if address not in self.history: + self.history[address] = [] + self.set_up_to_date(False) + if self.synchronizer: + self.synchronizer.add(address) + + def get_conflicting_transactions(self, tx): + """Returns a set of transaction hashes from the wallet history that are + directly conflicting with tx, i.e. they have common outpoints being + spent with tx. If the tx is already in wallet history, that will not be + reported as a conflict. + """ + conflicting_txns = set() + with self.transaction_lock: + for txin in tx.inputs(): + if txin['type'] == 'coinbase': + continue + prevout_hash = txin['prevout_hash'] + prevout_n = txin['prevout_n'] + spending_tx_hash = self.spent_outpoints[prevout_hash].get(prevout_n) + if spending_tx_hash is None: + continue + # this outpoint has already been spent, by spending_tx + assert spending_tx_hash in self.transactions + conflicting_txns |= {spending_tx_hash} + txid = tx.txid() + if txid in conflicting_txns: + # this tx is already in history, so it conflicts with itself + if len(conflicting_txns) > 1: + raise Exception('Found conflicting transactions already in wallet history.') + conflicting_txns -= {txid} + return conflicting_txns + + def add_transaction(self, tx_hash, tx, allow_unrelated=False): + assert tx_hash, tx_hash + assert tx, tx + assert tx.is_complete() + # we need self.transaction_lock but get_tx_height will take self.lock + # so we need to take that too here, to enforce order of locks + with self.lock, self.transaction_lock: + # NOTE: returning if tx in self.transactions might seem like a good idea + # BUT we track is_mine inputs in a txn, and during subsequent calls + # of add_transaction tx, we might learn of more-and-more inputs of + # being is_mine, as we roll the gap_limit forward + is_coinbase = tx.inputs()[0]['type'] == 'coinbase' + tx_height = self.get_tx_height(tx_hash)[0] + if not allow_unrelated: + # note that during sync, if the transactions are not properly sorted, + # it could happen that we think tx is unrelated but actually one of the inputs is is_mine. + # this is the main motivation for allow_unrelated + is_mine = any([self.is_mine(self.get_txin_address(txin)) for txin in tx.inputs()]) + is_for_me = any([self.is_mine(self.get_txout_address(txo)) for txo in tx.outputs()]) + if not is_mine and not is_for_me: + raise UnrelatedTransactionException() + # Find all conflicting transactions. + # In case of a conflict, + # 1. confirmed > mempool > local + # 2. this new txn has priority over existing ones + # When this method exits, there must NOT be any conflict, so + # either keep this txn and remove all conflicting (along with dependencies) + # or drop this txn + conflicting_txns = self.get_conflicting_transactions(tx) + if conflicting_txns: + existing_mempool_txn = any( + self.get_tx_height(tx_hash2)[0] in (TX_HEIGHT_UNCONFIRMED, TX_HEIGHT_UNCONF_PARENT) + for tx_hash2 in conflicting_txns) + existing_confirmed_txn = any( + self.get_tx_height(tx_hash2)[0] > 0 + for tx_hash2 in conflicting_txns) + if existing_confirmed_txn and tx_height <= 0: + # this is a non-confirmed tx that conflicts with confirmed txns; drop. + return False + if existing_mempool_txn and tx_height == TX_HEIGHT_LOCAL: + # this is a local tx that conflicts with non-local txns; drop. + return False + # keep this txn and remove all conflicting + to_remove = set() + to_remove |= conflicting_txns + for conflicting_tx_hash in conflicting_txns: + to_remove |= self.get_depending_transactions(conflicting_tx_hash) + for tx_hash2 in to_remove: + self.remove_transaction(tx_hash2) + # add inputs + def add_value_from_prev_output(): + dd = self.txo.get(prevout_hash, {}) + # note: this nested loop takes linear time in num is_mine outputs of prev_tx + for addr, outputs in dd.items(): + # note: instead of [(n, v, is_cb), ...]; we could store: {n -> (v, is_cb)} + for n, v, is_cb in outputs: + if n == prevout_n: + if addr and self.is_mine(addr): + if d.get(addr) is None: + d[addr] = set() + d[addr].add((ser, v)) + return + self.txi[tx_hash] = d = {} + for txi in tx.inputs(): + if txi['type'] == 'coinbase': + continue + prevout_hash = txi['prevout_hash'] + prevout_n = txi['prevout_n'] + ser = prevout_hash + ':%d' % prevout_n + self.spent_outpoints[prevout_hash][prevout_n] = tx_hash + add_value_from_prev_output() + # add outputs + self.txo[tx_hash] = d = {} + for n, txo in enumerate(tx.outputs()): + v = txo[2] + ser = tx_hash + ':%d'%n + addr = self.get_txout_address(txo) + if addr and self.is_mine(addr): + if d.get(addr) is None: + d[addr] = [] + d[addr].append((n, v, is_coinbase)) + # give v to txi that spends me + next_tx = self.spent_outpoints[tx_hash].get(n) + if next_tx is not None: + dd = self.txi.get(next_tx, {}) + if dd.get(addr) is None: + dd[addr] = set() + if (ser, v) not in dd[addr]: + dd[addr].add((ser, v)) + self._add_tx_to_local_history(next_tx) + # add to local history + self._add_tx_to_local_history(tx_hash) + # save + self.transactions[tx_hash] = tx + return True + + def remove_transaction(self, tx_hash): + def remove_from_spent_outpoints(): + # undo spends in spent_outpoints + if tx is not None: # if we have the tx, this branch is faster + for txin in tx.inputs(): + if txin['type'] == 'coinbase': + continue + prevout_hash = txin['prevout_hash'] + prevout_n = txin['prevout_n'] + self.spent_outpoints[prevout_hash].pop(prevout_n, None) + if not self.spent_outpoints[prevout_hash]: + self.spent_outpoints.pop(prevout_hash) + else: # expensive but always works + for prevout_hash, d in list(self.spent_outpoints.items()): + for prevout_n, spending_txid in d.items(): + if spending_txid == tx_hash: + self.spent_outpoints[prevout_hash].pop(prevout_n, None) + if not self.spent_outpoints[prevout_hash]: + self.spent_outpoints.pop(prevout_hash) + # Remove this tx itself; if nothing spends from it. + # It is not so clear what to do if other txns spend from it, but it will be + # removed when those other txns are removed. + if not self.spent_outpoints[tx_hash]: + self.spent_outpoints.pop(tx_hash) + + with self.transaction_lock: + self.print_error("removing tx from history", tx_hash) + tx = self.transactions.pop(tx_hash, None) + remove_from_spent_outpoints() + self._remove_tx_from_local_history(tx_hash) + self.txi.pop(tx_hash, None) + self.txo.pop(tx_hash, None) + + def receive_tx_callback(self, tx_hash, tx, tx_height): + self.add_unverified_tx(tx_hash, tx_height) + self.add_transaction(tx_hash, tx, allow_unrelated=True) + + def receive_history_callback(self, addr, hist, tx_fees): + with self.lock: + old_hist = self.get_address_history(addr) + for tx_hash, height in old_hist: + if (tx_hash, height) not in hist: + # make tx local + self.unverified_tx.pop(tx_hash, None) + self.verified_tx.pop(tx_hash, None) + if self.verifier: + self.verifier.remove_spv_proof_for_tx(tx_hash) + self.history[addr] = hist + + for tx_hash, tx_height in hist: + # add it in case it was previously unconfirmed + self.add_unverified_tx(tx_hash, tx_height) + # if addr is new, we have to recompute txi and txo + tx = self.transactions.get(tx_hash) + if tx is None: + continue + self.add_transaction(tx_hash, tx, allow_unrelated=True) + + # Store fees + self.tx_fees.update(tx_fees) + + @profiler + def load_transactions(self): + # load txi, txo, tx_fees + self.txi = self.storage.get('txi', {}) + for txid, d in list(self.txi.items()): + for addr, lst in d.items(): + self.txi[txid][addr] = set([tuple(x) for x in lst]) + self.txo = self.storage.get('txo', {}) + self.tx_fees = self.storage.get('tx_fees', {}) + tx_list = self.storage.get('transactions', {}) + # load transactions + self.transactions = {} + for tx_hash, raw in tx_list.items(): + tx = Transaction(raw) + self.transactions[tx_hash] = tx + if self.txi.get(tx_hash) is None and self.txo.get(tx_hash) is None: + self.print_error("removing unreferenced tx", tx_hash) + self.transactions.pop(tx_hash) + # load spent_outpoints + _spent_outpoints = self.storage.get('spent_outpoints', {}) + self.spent_outpoints = defaultdict(dict) + for prevout_hash, d in _spent_outpoints.items(): + for prevout_n_str, spending_txid in d.items(): + prevout_n = int(prevout_n_str) + self.spent_outpoints[prevout_hash][prevout_n] = spending_txid + + @profiler + def load_local_history(self): + self._history_local = {} # address -> set(txid) + for txid in itertools.chain(self.txi, self.txo): + self._add_tx_to_local_history(txid) + + def remove_local_transactions_we_dont_have(self): + txid_set = set(self.txi) | set(self.txo) + for txid in txid_set: + tx_height = self.get_tx_height(txid)[0] + if tx_height == TX_HEIGHT_LOCAL and txid not in self.transactions: + self.remove_transaction(txid) + + @profiler + def save_transactions(self, write=False): + with self.transaction_lock: + tx = {} + for k,v in self.transactions.items(): + tx[k] = str(v) + self.storage.put('transactions', tx) + self.storage.put('txi', self.txi) + self.storage.put('txo', self.txo) + self.storage.put('tx_fees', self.tx_fees) + self.storage.put('addr_history', self.history) + self.storage.put('spent_outpoints', self.spent_outpoints) + if write: + self.storage.write() + + def save_verified_tx(self, write=False): + with self.lock: + self.storage.put('verified_tx3', self.verified_tx) + if write: + self.storage.write() + + def clear_history(self): + with self.lock: + with self.transaction_lock: + self.txi = {} + self.txo = {} + self.tx_fees = {} + self.spent_outpoints = defaultdict(dict) + self.history = {} + self.verified_tx = {} + self.transactions = {} + self.save_transactions() + + def get_history(self, domain=None): + # get domain + if domain is None: + domain = self.get_addresses() + domain = set(domain) + # 1. Get the history of each address in the domain, maintain the + # delta of a tx as the sum of its deltas on domain addresses + tx_deltas = defaultdict(int) + for addr in domain: + h = self.get_address_history(addr) + for tx_hash, height in h: + delta = self.get_tx_delta(tx_hash, addr) + if delta is None or tx_deltas[tx_hash] is None: + tx_deltas[tx_hash] = None + else: + tx_deltas[tx_hash] += delta + # 2. create sorted history + history = [] + for tx_hash in tx_deltas: + delta = tx_deltas[tx_hash] + height, conf, timestamp = self.get_tx_height(tx_hash) + history.append((tx_hash, height, conf, timestamp, delta)) + history.sort(key = lambda x: self.get_txpos(x[0])) + history.reverse() + # 3. add balance + c, u, x = self.get_balance(domain) + balance = c + u + x + h2 = [] + for tx_hash, height, conf, timestamp, delta in history: + h2.append((tx_hash, height, conf, timestamp, delta, balance)) + if balance is None or delta is None: + balance = None + else: + balance -= delta + h2.reverse() + # fixme: this may happen if history is incomplete + if balance not in [None, 0]: + self.print_error("Error: history not synchronized") + return [] + + return h2 + + def _add_tx_to_local_history(self, txid): + with self.transaction_lock: + for addr in itertools.chain(self.txi.get(txid, []), self.txo.get(txid, [])): + cur_hist = self._history_local.get(addr, set()) + cur_hist.add(txid) + self._history_local[addr] = cur_hist + + def _remove_tx_from_local_history(self, txid): + with self.transaction_lock: + for addr in itertools.chain(self.txi.get(txid, []), self.txo.get(txid, [])): + cur_hist = self._history_local.get(addr, set()) + try: + cur_hist.remove(txid) + except KeyError: + pass + else: + self._history_local[addr] = cur_hist + + def add_unverified_tx(self, tx_hash, tx_height): + if tx_height in (TX_HEIGHT_UNCONFIRMED, TX_HEIGHT_UNCONF_PARENT) \ + and tx_hash in self.verified_tx: + with self.lock: + self.verified_tx.pop(tx_hash) + if self.verifier: + self.verifier.remove_spv_proof_for_tx(tx_hash) + + # tx will be verified only if height > 0 + if tx_hash not in self.verified_tx: + with self.lock: + self.unverified_tx[tx_hash] = tx_height + + def add_verified_tx(self, tx_hash, info): + # Remove from the unverified map and add to the verified map + with self.lock: + self.unverified_tx.pop(tx_hash, None) + self.verified_tx[tx_hash] = info # (tx_height, timestamp, pos) + height, conf, timestamp = self.get_tx_height(tx_hash) + self.network.trigger_callback('verified', tx_hash, height, conf, timestamp) + + def get_unverified_txs(self): + '''Returns a map from tx hash to transaction height''' + with self.lock: + return dict(self.unverified_tx) # copy + + def undo_verifications(self, blockchain, height): + '''Used by the verifier when a reorg has happened''' + txs = set() + with self.lock: + for tx_hash, item in list(self.verified_tx.items()): + tx_height, timestamp, pos = item + if tx_height >= height: + header = blockchain.read_header(tx_height) + # fixme: use block hash, not timestamp + if not header or header.get('timestamp') != timestamp: + self.verified_tx.pop(tx_hash, None) + txs.add(tx_hash) + return txs + + def get_local_height(self): + """ return last known height if we are offline """ + return self.network.get_local_height() if self.network else self.storage.get('stored_height', 0) + + def get_tx_height(self, tx_hash): + """ Given a transaction, returns (height, conf, timestamp) """ + with self.lock: + if tx_hash in self.verified_tx: + height, timestamp, pos = self.verified_tx[tx_hash] + conf = max(self.get_local_height() - height + 1, 0) + return height, conf, timestamp + elif tx_hash in self.unverified_tx: + height = self.unverified_tx[tx_hash] + return height, 0, None + else: + # local transaction + return TX_HEIGHT_LOCAL, 0, None + + def set_up_to_date(self, up_to_date): + with self.lock: + self.up_to_date = up_to_date + if up_to_date: + self.save_transactions(write=True) + # if the verifier is also up to date, persist that too; + # otherwise it will persist its results when it finishes + if self.verifier and self.verifier.is_up_to_date(): + self.save_verified_tx(write=True) + + def is_up_to_date(self): + with self.lock: return self.up_to_date diff --git a/electrum/gui/qt/history_list.py b/electrum/gui/qt/history_list.py index 83ab3fdb..8f1a5ab3 100644 --- a/electrum/gui/qt/history_list.py +++ b/electrum/gui/qt/history_list.py @@ -26,7 +26,7 @@ import webbrowser import datetime -from electrum.wallet import AddTransactionException, TX_HEIGHT_LOCAL +from electrum.address_synchronizer import TX_HEIGHT_LOCAL from .util import * from electrum.i18n import _ from electrum.util import block_explorer_URL, profiler, print_error diff --git a/electrum/gui/qt/main_window.py b/electrum/gui/qt/main_window.py index ac2c5282..1cdae70d 100644 --- a/electrum/gui/qt/main_window.py +++ b/electrum/gui/qt/main_window.py @@ -51,7 +51,8 @@ from electrum.util import (format_time, format_satoshis, format_fee_satoshis, base_units, base_units_list, base_unit_name_to_decimal_point, decimal_point_to_base_unit_name, quantize_feerate) from electrum.transaction import Transaction -from electrum.wallet import Multisig_Wallet, AddTransactionException, CannotBumpFee +from electrum.address_synchronizer import AddTransactionException +from electrum.wallet import Multisig_Wallet, CannotBumpFee from .amountedit import AmountEdit, BTCAmountEdit, MyLineEdit, FeerateEdit from .qrcodewidget import QRCodeWidget, QRDialog diff --git a/electrum/gui/qt/transaction_dialog.py b/electrum/gui/qt/transaction_dialog.py index fe7624da..66d47f6f 100644 --- a/electrum/gui/qt/transaction_dialog.py +++ b/electrum/gui/qt/transaction_dialog.py @@ -37,7 +37,6 @@ from electrum.plugin import run_hook from electrum import simple_config from electrum.util import bfh -from electrum.wallet import AddTransactionException from electrum.transaction import SerializationError from .util import * diff --git a/electrum/tests/test_wallet_vertical.py b/electrum/tests/test_wallet_vertical.py index 96d42dcc..c95d9831 100644 --- a/electrum/tests/test_wallet_vertical.py +++ b/electrum/tests/test_wallet_vertical.py @@ -7,7 +7,8 @@ from typing import Sequence from electrum import storage, bitcoin, keystore, constants from electrum import Transaction from electrum import SimpleConfig -from electrum.wallet import TX_HEIGHT_UNCONFIRMED, TX_HEIGHT_UNCONF_PARENT, sweep, Multisig_Wallet, Standard_Wallet, Imported_Wallet +from electrum.address_synchronizer import TX_HEIGHT_UNCONFIRMED, TX_HEIGHT_UNCONF_PARENT +from electrum.wallet import sweep, Multisig_Wallet, Standard_Wallet, Imported_Wallet from electrum.util import bfh, bh2u from electrum.plugins.trustedcoin import trustedcoin diff --git a/electrum/wallet.py b/electrum/wallet.py index 416deffd..b207a4d6 100644 --- a/electrum/wallet.py +++ b/electrum/wallet.py @@ -28,7 +28,7 @@ import os -import threading +import sys import random import time import json @@ -36,12 +36,8 @@ import copy import errno import traceback from functools import partial -from collections import defaultdict from numbers import Number from decimal import Decimal -import itertools - -import sys from .i18n import _ from .util import (NotEnoughFunds, PrintError, UserCancelled, profiler, @@ -57,8 +53,7 @@ from .storage import multisig_type, STO_EV_PLAINTEXT, STO_EV_USER_PW, STO_EV_XPU from . import transaction, bitcoin, coinchooser, paymentrequest, contacts from .transaction import Transaction from .plugin import run_hook -from .synchronizer import Synchronizer -from .verifier import SPV +from .address_synchronizer import AddressSynchronizer from .paymentrequest import PR_PAID, PR_UNPAID, PR_UNKNOWN, PR_EXPIRED from .paymentrequest import InvoiceStore @@ -71,9 +66,6 @@ TX_STATUS = [ _('Local'), ] -TX_HEIGHT_LOCAL = -2 -TX_HEIGHT_UNCONF_PARENT = -1 -TX_HEIGHT_UNCONFIRMED = 0 def relayfee(network): @@ -158,65 +150,37 @@ def sweep(privkeys, network, config, recipient, fee=None, imax=100): return tx -class AddTransactionException(Exception): - pass - - -class UnrelatedTransactionException(AddTransactionException): - def __str__(self): - return _("Transaction is unrelated to this wallet.") - class CannotBumpFee(Exception): pass -class Abstract_Wallet(PrintError): + + +class Abstract_Wallet(AddressSynchronizer): """ Wallet classes are created to handle various address generation methods. Completion states (watching-only, single account, no seed, etc) are handled inside classes. """ max_change_outputs = 3 + gap_limit_for_change = 6 def __init__(self, storage): + AddressSynchronizer.__init__(self, storage) self.electrum_version = ELECTRUM_VERSION - self.storage = storage - self.network = None - # verifier (SPV) and synchronizer are started in start_threads - self.synchronizer = None - self.verifier = None - - self.gap_limit_for_change = 6 # constant - - # locks: if you need to take multiple ones, acquire them in the order they are defined here! - self.lock = threading.RLock() - self.transaction_lock = threading.RLock() - # saved fields self.use_change = storage.get('use_change', True) self.multiple_change = storage.get('multiple_change', False) self.labels = storage.get('labels', {}) self.frozen_addresses = set(storage.get('frozen_addresses',[])) - self.history = storage.get('addr_history',{}) # address -> list(txid, height) self.fiat_value = storage.get('fiat_value', {}) self.receive_requests = storage.get('payment_requests', {}) - # Verified transactions. txid -> (height, timestamp, block_pos). Access with self.lock. - self.verified_tx = storage.get('verified_tx3', {}) - # Transactions pending verification. txid -> tx_height. Access with self.lock. - self.unverified_tx = defaultdict(int) - self.load_keystore() self.load_addresses() self.test_addresses_sanity() - self.load_transactions() - self.load_local_history() - self.check_history() - self.load_unverified_transactions() - self.remove_local_transactions_we_dont_have() - # wallet.up_to_date is true when the wallet is synchronized - self.up_to_date = False + self.check_history() # save wallet type the first time if self.storage.get('wallet_type') is None: @@ -228,7 +192,6 @@ class Abstract_Wallet(PrintError): self.coin_price_cache = {} - def diagnostic_name(self): return self.basename() @@ -238,92 +201,16 @@ class Abstract_Wallet(PrintError): def get_master_public_key(self): return None - @profiler - def load_transactions(self): - # load txi, txo, tx_fees - self.txi = self.storage.get('txi', {}) - for txid, d in list(self.txi.items()): - for addr, lst in d.items(): - self.txi[txid][addr] = set([tuple(x) for x in lst]) - self.txo = self.storage.get('txo', {}) - self.tx_fees = self.storage.get('tx_fees', {}) - tx_list = self.storage.get('transactions', {}) - # load transactions - self.transactions = {} - for tx_hash, raw in tx_list.items(): - tx = Transaction(raw) - self.transactions[tx_hash] = tx - if self.txi.get(tx_hash) is None and self.txo.get(tx_hash) is None: - self.print_error("removing unreferenced tx", tx_hash) - self.transactions.pop(tx_hash) - # load spent_outpoints - _spent_outpoints = self.storage.get('spent_outpoints', {}) - self.spent_outpoints = defaultdict(dict) - for prevout_hash, d in _spent_outpoints.items(): - for prevout_n_str, spending_txid in d.items(): - prevout_n = int(prevout_n_str) - self.spent_outpoints[prevout_hash][prevout_n] = spending_txid - - @profiler - def load_local_history(self): - self._history_local = {} # address -> set(txid) - for txid in itertools.chain(self.txi, self.txo): - self._add_tx_to_local_history(txid) - - def remove_local_transactions_we_dont_have(self): - txid_set = set(self.txi) | set(self.txo) - for txid in txid_set: - tx_height = self.get_tx_height(txid)[0] - if tx_height == TX_HEIGHT_LOCAL and txid not in self.transactions: - self.remove_transaction(txid) - - @profiler - def save_transactions(self, write=False): - with self.transaction_lock: - tx = {} - for k,v in self.transactions.items(): - tx[k] = str(v) - self.storage.put('transactions', tx) - self.storage.put('txi', self.txi) - self.storage.put('txo', self.txo) - self.storage.put('tx_fees', self.tx_fees) - self.storage.put('addr_history', self.history) - self.storage.put('spent_outpoints', self.spent_outpoints) - if write: - self.storage.write() - - def save_verified_tx(self, write=False): - with self.lock: - self.storage.put('verified_tx3', self.verified_tx) - if write: - self.storage.write() - - def clear_history(self): - with self.lock: - with self.transaction_lock: - self.txi = {} - self.txo = {} - self.tx_fees = {} - self.spent_outpoints = defaultdict(dict) - self.history = {} - self.verified_tx = {} - self.transactions = {} - self.save_transactions() - @profiler def check_history(self): save = False - hist_addrs_mine = list(filter(lambda k: self.is_mine(k), self.history.keys())) hist_addrs_not_mine = list(filter(lambda k: not self.is_mine(k), self.history.keys())) - for addr in hist_addrs_not_mine: self.history.pop(addr) save = True - for addr in hist_addrs_mine: hist = self.history[addr] - for tx_hash, tx_height in hist: if self.txi.get(tx_hash) or self.txo.get(tx_hash): continue @@ -358,19 +245,6 @@ class Abstract_Wallet(PrintError): def is_deterministic(self): return self.keystore.is_deterministic() - def set_up_to_date(self, up_to_date): - with self.lock: - self.up_to_date = up_to_date - if up_to_date: - self.save_transactions(write=True) - # if the verifier is also up to date, persist that too; - # otherwise it will persist its results when it finishes - if self.verifier and self.verifier.is_up_to_date(): - self.save_verified_tx(write=True) - - def is_up_to_date(self): - with self.lock: return self.up_to_date - def set_label(self, name, text = None): changed = False old_text = self.labels.get(name) @@ -441,64 +315,6 @@ class Abstract_Wallet(PrintError): def get_public_keys(self, address): return [self.get_public_key(address)] - def add_unverified_tx(self, tx_hash, tx_height): - if tx_height in (TX_HEIGHT_UNCONFIRMED, TX_HEIGHT_UNCONF_PARENT) \ - and tx_hash in self.verified_tx: - with self.lock: - self.verified_tx.pop(tx_hash) - if self.verifier: - self.verifier.remove_spv_proof_for_tx(tx_hash) - - # tx will be verified only if height > 0 - if tx_hash not in self.verified_tx: - with self.lock: - self.unverified_tx[tx_hash] = tx_height - - def add_verified_tx(self, tx_hash, info): - # Remove from the unverified map and add to the verified map - with self.lock: - self.unverified_tx.pop(tx_hash, None) - self.verified_tx[tx_hash] = info # (tx_height, timestamp, pos) - height, conf, timestamp = self.get_tx_height(tx_hash) - self.network.trigger_callback('verified', tx_hash, height, conf, timestamp) - - def get_unverified_txs(self): - '''Returns a map from tx hash to transaction height''' - with self.lock: - return dict(self.unverified_tx) # copy - - def undo_verifications(self, blockchain, height): - '''Used by the verifier when a reorg has happened''' - txs = set() - with self.lock: - for tx_hash, item in list(self.verified_tx.items()): - tx_height, timestamp, pos = item - if tx_height >= height: - header = blockchain.read_header(tx_height) - # fixme: use block hash, not timestamp - if not header or header.get('timestamp') != timestamp: - self.verified_tx.pop(tx_hash, None) - txs.add(tx_hash) - return txs - - def get_local_height(self): - """ return last known height if we are offline """ - return self.network.get_local_height() if self.network else self.storage.get('stored_height', 0) - - def get_tx_height(self, tx_hash): - """ Given a transaction, returns (height, conf, timestamp) """ - with self.lock: - if tx_hash in self.verified_tx: - height, timestamp, pos = self.verified_tx[tx_hash] - conf = max(self.get_local_height() - height + 1, 0) - return height, conf, timestamp - elif tx_hash in self.unverified_tx: - height = self.unverified_tx[tx_hash] - return height, 0, None - else: - # local transaction - return TX_HEIGHT_LOCAL, 0, None - def get_txpos(self, tx_hash): "return position, even if the tx is unverified" with self.lock: @@ -757,24 +573,6 @@ class Abstract_Wallet(PrintError): h.append((tx_hash, tx_height)) return h - def _add_tx_to_local_history(self, txid): - with self.transaction_lock: - for addr in itertools.chain(self.txi.get(txid, []), self.txo.get(txid, [])): - cur_hist = self._history_local.get(addr, set()) - cur_hist.add(txid) - self._history_local[addr] = cur_hist - - def _remove_tx_from_local_history(self, txid): - with self.transaction_lock: - for addr in itertools.chain(self.txi.get(txid, []), self.txo.get(txid, [])): - cur_hist = self._history_local.get(addr, set()) - try: - cur_hist.remove(txid) - except KeyError: - pass - else: - self._history_local[addr] = cur_hist - def get_txin_address(self, txi): addr = txi.get('address') if addr and addr != "(pubkey)": @@ -798,235 +596,6 @@ class Abstract_Wallet(PrintError): addr = None return addr - def get_conflicting_transactions(self, tx): - """Returns a set of transaction hashes from the wallet history that are - directly conflicting with tx, i.e. they have common outpoints being - spent with tx. If the tx is already in wallet history, that will not be - reported as a conflict. - """ - conflicting_txns = set() - with self.transaction_lock: - for txin in tx.inputs(): - if txin['type'] == 'coinbase': - continue - prevout_hash = txin['prevout_hash'] - prevout_n = txin['prevout_n'] - spending_tx_hash = self.spent_outpoints[prevout_hash].get(prevout_n) - if spending_tx_hash is None: - continue - # this outpoint has already been spent, by spending_tx - assert spending_tx_hash in self.transactions - conflicting_txns |= {spending_tx_hash} - txid = tx.txid() - if txid in conflicting_txns: - # this tx is already in history, so it conflicts with itself - if len(conflicting_txns) > 1: - raise Exception('Found conflicting transactions already in wallet history.') - conflicting_txns -= {txid} - return conflicting_txns - - def add_transaction(self, tx_hash, tx, allow_unrelated=False): - assert tx_hash, tx_hash - assert tx, tx - assert tx.is_complete() - # we need self.transaction_lock but get_tx_height will take self.lock - # so we need to take that too here, to enforce order of locks - with self.lock, self.transaction_lock: - # NOTE: returning if tx in self.transactions might seem like a good idea - # BUT we track is_mine inputs in a txn, and during subsequent calls - # of add_transaction tx, we might learn of more-and-more inputs of - # being is_mine, as we roll the gap_limit forward - is_coinbase = tx.inputs()[0]['type'] == 'coinbase' - tx_height = self.get_tx_height(tx_hash)[0] - if not allow_unrelated: - # note that during sync, if the transactions are not properly sorted, - # it could happen that we think tx is unrelated but actually one of the inputs is is_mine. - # this is the main motivation for allow_unrelated - is_mine = any([self.is_mine(self.get_txin_address(txin)) for txin in tx.inputs()]) - is_for_me = any([self.is_mine(self.get_txout_address(txo)) for txo in tx.outputs()]) - if not is_mine and not is_for_me: - raise UnrelatedTransactionException() - # Find all conflicting transactions. - # In case of a conflict, - # 1. confirmed > mempool > local - # 2. this new txn has priority over existing ones - # When this method exits, there must NOT be any conflict, so - # either keep this txn and remove all conflicting (along with dependencies) - # or drop this txn - conflicting_txns = self.get_conflicting_transactions(tx) - if conflicting_txns: - existing_mempool_txn = any( - self.get_tx_height(tx_hash2)[0] in (TX_HEIGHT_UNCONFIRMED, TX_HEIGHT_UNCONF_PARENT) - for tx_hash2 in conflicting_txns) - existing_confirmed_txn = any( - self.get_tx_height(tx_hash2)[0] > 0 - for tx_hash2 in conflicting_txns) - if existing_confirmed_txn and tx_height <= 0: - # this is a non-confirmed tx that conflicts with confirmed txns; drop. - return False - if existing_mempool_txn and tx_height == TX_HEIGHT_LOCAL: - # this is a local tx that conflicts with non-local txns; drop. - return False - # keep this txn and remove all conflicting - to_remove = set() - to_remove |= conflicting_txns - for conflicting_tx_hash in conflicting_txns: - to_remove |= self.get_depending_transactions(conflicting_tx_hash) - for tx_hash2 in to_remove: - self.remove_transaction(tx_hash2) - # add inputs - def add_value_from_prev_output(): - dd = self.txo.get(prevout_hash, {}) - # note: this nested loop takes linear time in num is_mine outputs of prev_tx - for addr, outputs in dd.items(): - # note: instead of [(n, v, is_cb), ...]; we could store: {n -> (v, is_cb)} - for n, v, is_cb in outputs: - if n == prevout_n: - if addr and self.is_mine(addr): - if d.get(addr) is None: - d[addr] = set() - d[addr].add((ser, v)) - return - self.txi[tx_hash] = d = {} - for txi in tx.inputs(): - if txi['type'] == 'coinbase': - continue - prevout_hash = txi['prevout_hash'] - prevout_n = txi['prevout_n'] - ser = prevout_hash + ':%d' % prevout_n - self.spent_outpoints[prevout_hash][prevout_n] = tx_hash - add_value_from_prev_output() - # add outputs - self.txo[tx_hash] = d = {} - for n, txo in enumerate(tx.outputs()): - v = txo[2] - ser = tx_hash + ':%d'%n - addr = self.get_txout_address(txo) - if addr and self.is_mine(addr): - if d.get(addr) is None: - d[addr] = [] - d[addr].append((n, v, is_coinbase)) - # give v to txi that spends me - next_tx = self.spent_outpoints[tx_hash].get(n) - if next_tx is not None: - dd = self.txi.get(next_tx, {}) - if dd.get(addr) is None: - dd[addr] = set() - if (ser, v) not in dd[addr]: - dd[addr].add((ser, v)) - self._add_tx_to_local_history(next_tx) - # add to local history - self._add_tx_to_local_history(tx_hash) - # save - self.transactions[tx_hash] = tx - return True - - def remove_transaction(self, tx_hash): - def remove_from_spent_outpoints(): - # undo spends in spent_outpoints - if tx is not None: # if we have the tx, this branch is faster - for txin in tx.inputs(): - if txin['type'] == 'coinbase': - continue - prevout_hash = txin['prevout_hash'] - prevout_n = txin['prevout_n'] - self.spent_outpoints[prevout_hash].pop(prevout_n, None) - if not self.spent_outpoints[prevout_hash]: - self.spent_outpoints.pop(prevout_hash) - else: # expensive but always works - for prevout_hash, d in list(self.spent_outpoints.items()): - for prevout_n, spending_txid in d.items(): - if spending_txid == tx_hash: - self.spent_outpoints[prevout_hash].pop(prevout_n, None) - if not self.spent_outpoints[prevout_hash]: - self.spent_outpoints.pop(prevout_hash) - # Remove this tx itself; if nothing spends from it. - # It is not so clear what to do if other txns spend from it, but it will be - # removed when those other txns are removed. - if not self.spent_outpoints[tx_hash]: - self.spent_outpoints.pop(tx_hash) - - with self.transaction_lock: - self.print_error("removing tx from history", tx_hash) - tx = self.transactions.pop(tx_hash, None) - remove_from_spent_outpoints() - self._remove_tx_from_local_history(tx_hash) - self.txi.pop(tx_hash, None) - self.txo.pop(tx_hash, None) - - def receive_tx_callback(self, tx_hash, tx, tx_height): - self.add_unverified_tx(tx_hash, tx_height) - self.add_transaction(tx_hash, tx, allow_unrelated=True) - - def receive_history_callback(self, addr, hist, tx_fees): - with self.lock: - old_hist = self.get_address_history(addr) - for tx_hash, height in old_hist: - if (tx_hash, height) not in hist: - # make tx local - self.unverified_tx.pop(tx_hash, None) - self.verified_tx.pop(tx_hash, None) - if self.verifier: - self.verifier.remove_spv_proof_for_tx(tx_hash) - self.history[addr] = hist - - for tx_hash, tx_height in hist: - # add it in case it was previously unconfirmed - self.add_unverified_tx(tx_hash, tx_height) - # if addr is new, we have to recompute txi and txo - tx = self.transactions.get(tx_hash) - if tx is None: - continue - self.add_transaction(tx_hash, tx, allow_unrelated=True) - - # Store fees - self.tx_fees.update(tx_fees) - - def get_history(self, domain=None): - # get domain - if domain is None: - domain = self.get_addresses() - domain = set(domain) - # 1. Get the history of each address in the domain, maintain the - # delta of a tx as the sum of its deltas on domain addresses - tx_deltas = defaultdict(int) - for addr in domain: - h = self.get_address_history(addr) - for tx_hash, height in h: - delta = self.get_tx_delta(tx_hash, addr) - if delta is None or tx_deltas[tx_hash] is None: - tx_deltas[tx_hash] = None - else: - tx_deltas[tx_hash] += delta - - # 2. create sorted history - history = [] - for tx_hash in tx_deltas: - delta = tx_deltas[tx_hash] - height, conf, timestamp = self.get_tx_height(tx_hash) - history.append((tx_hash, height, conf, timestamp, delta)) - history.sort(key = lambda x: self.get_txpos(x[0])) - history.reverse() - - # 3. add balance - c, u, x = self.get_balance(domain) - balance = c + u + x - h2 = [] - for tx_hash, height, conf, timestamp, delta in history: - h2.append((tx_hash, height, conf, timestamp, delta, balance)) - if balance is None or delta is None: - balance = None - else: - balance -= delta - h2.reverse() - - # fixme: this may happen if history is incomplete - if balance not in [None, 0]: - self.print_error("Error: history not synchronized") - return [] - - return h2 - def balance_at_timestamp(self, domain, target_timestamp): h = self.get_history(domain) for tx_hash, height, conf, timestamp, value, balance in h: @@ -1285,36 +854,6 @@ class Abstract_Wallet(PrintError): return True return False - def load_unverified_transactions(self): - # review transactions that are in the history - for addr, hist in self.history.items(): - for tx_hash, tx_height in hist: - # add it in case it was previously unconfirmed - self.add_unverified_tx(tx_hash, tx_height) - - def start_threads(self, network): - self.network = network - if self.network is not None: - self.verifier = SPV(self.network, self) - self.synchronizer = Synchronizer(self, network) - network.add_jobs([self.verifier, self.synchronizer]) - else: - self.verifier = None - self.synchronizer = None - - def stop_threads(self): - if self.network: - self.network.remove_jobs([self.synchronizer, self.verifier]) - self.synchronizer.release() - self.synchronizer = None - self.verifier = None - # Now no references to the synchronizer or verifier - # remain so they will be GC-ed - self.storage.put('stored_height', self.get_local_height()) - self.save_transactions() - self.save_verified_tx() - self.storage.write() - def wait_until_synchronized(self, callback=None): def wait_for_wallet(): self.set_up_to_date(False) @@ -1605,7 +1144,7 @@ class Abstract_Wallet(PrintError): expiration = 0 conf = None if amount: - if self.up_to_date: + if self.is_up_to_date(): paid, conf = self.get_payment_status(address, amount) status = PR_PAID if paid else PR_UNPAID if status == PR_UNPAID and expiration is not None and time.time() > timestamp + expiration: @@ -1700,12 +1239,6 @@ class Abstract_Wallet(PrintError): def can_delete_address(self): return False - def add_address(self, address): - if address not in self.history: - self.history[address] = [] - if self.synchronizer: - self.synchronizer.add(address) - def has_password(self): return self.has_keystore_encryption() or self.has_storage_encryption() From b4b862b0cc7930396f1350783019f61db8bf7a12 Mon Sep 17 00:00:00 2001 From: tiagotrs Date: Wed, 18 Jul 2018 13:15:31 +0200 Subject: [PATCH 08/56] add warning that seed extension will not be included in the backup (#4555) --- electrum/gui/qt/seed_dialog.py | 3 ++- electrum/plugins/revealer/qt.py | 10 +++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/electrum/gui/qt/seed_dialog.py b/electrum/gui/qt/seed_dialog.py index 1006dfd9..86fc15ec 100644 --- a/electrum/gui/qt/seed_dialog.py +++ b/electrum/gui/qt/seed_dialog.py @@ -207,5 +207,6 @@ class SeedDialog(WindowModalDialog): title = _("Your wallet generation seed is:") slayout = SeedLayout(title=title, seed=seed, msg=True, passphrase=passphrase) vbox.addLayout(slayout) - run_hook('set_seed', seed, slayout.seed_e) + has_extension = True if passphrase else False + run_hook('set_seed', seed, has_extension, slayout.seed_e) vbox.addLayout(Buttons(CloseButton(self))) diff --git a/electrum/plugins/revealer/qt.py b/electrum/plugins/revealer/qt.py index 0ce60562..69c9fcd3 100644 --- a/electrum/plugins/revealer/qt.py +++ b/electrum/plugins/revealer/qt.py @@ -57,8 +57,9 @@ class Plugin(BasePlugin): make_dir(self.base_dir) @hook - def set_seed(self, seed, parent): + def set_seed(self, seed, has_extension, parent): self.cseed = seed.upper() + self.has_extension = has_extension parent.addButton(':icons/revealer.png', partial(self.setup_dialog, parent), "Revealer"+_(" secret backup utility")) def requires_settings(self): @@ -168,6 +169,10 @@ class Plugin(BasePlugin): "
","", self.base_dir+ self.filename+self.version+"_"+self.code_id,""])) dialog.close() + def ext_warning(self, dialog): + dialog.show_message(''.join(["",_("Warning: "), "", _("your seed extension will not be included in the encrypted backup.")])) + dialog.close() + def bdone(self, dialog): dialog.show_message(''.join([_("Digital Revealer ({}_{}) saved as PNG and PDF at:").format(self.version, self.code_id), "
","", self.base_dir + 'revealer_' +self.version + '_'+ self.code_id, ''])) @@ -360,6 +365,9 @@ class Plugin(BasePlugin): self.filename = self.wallet_name+'_'+ _('seed')+'_' self.was = self.wallet_name +' ' + _('seed') + if self.has_extension: + self.ext_warning(self.c_dialog) + if not calibration: self.toPdf(QImage(cypherseed)) QDesktopServices.openUrl (QUrl.fromLocalFile(os.path.abspath(self.base_dir+self.filename+self.version+'_'+self.code_id+'.pdf'))) From 1e715113ab28b13592dd05939cb463e926c26416 Mon Sep 17 00:00:00 2001 From: Janus Date: Wed, 18 Jul 2018 14:34:59 +0200 Subject: [PATCH 09/56] remove pbkdf2 dependency, use stdlib instead --- contrib/requirements/requirements.txt | 1 - electrum/keystore.py | 7 +++---- electrum/mnemonic.py | 3 +-- electrum/plugins/digitalbitbox/digitalbitbox.py | 4 ++-- electrum/storage.py | 6 +++--- run_electrum | 1 - 6 files changed, 9 insertions(+), 13 deletions(-) diff --git a/contrib/requirements/requirements.txt b/contrib/requirements/requirements.txt index 26b1b3d3..4fd62530 100644 --- a/contrib/requirements/requirements.txt +++ b/contrib/requirements/requirements.txt @@ -1,6 +1,5 @@ pyaes>=0.1a1 ecdsa>=0.9 -pbkdf2 requests qrcode protobuf diff --git a/electrum/keystore.py b/electrum/keystore.py index a8e068bb..c822be13 100644 --- a/electrum/keystore.py +++ b/electrum/keystore.py @@ -552,13 +552,12 @@ def bip39_normalize_passphrase(passphrase): return normalize('NFKD', passphrase or '') def bip39_to_seed(mnemonic, passphrase): - import pbkdf2, hashlib, hmac + import hashlib, hmac PBKDF2_ROUNDS = 2048 mnemonic = normalize('NFKD', ' '.join(mnemonic.split())) passphrase = bip39_normalize_passphrase(passphrase) - return pbkdf2.PBKDF2(mnemonic, 'mnemonic' + passphrase, - iterations = PBKDF2_ROUNDS, macmodule = hmac, - digestmodule = hashlib.sha512).read(64) + return hashlib.pbkdf2_hmac('sha512', mnemonic.encode('utf-8'), + b'mnemonic' + passphrase.encode('utf-8'), iterations = PBKDF2_ROUNDS) # returns tuple (is_checksum_valid, is_wordlist_valid) def bip39_is_checksum_valid(mnemonic): diff --git a/electrum/mnemonic.py b/electrum/mnemonic.py index 17c2cafc..a22913ae 100644 --- a/electrum/mnemonic.py +++ b/electrum/mnemonic.py @@ -30,7 +30,6 @@ import unicodedata import string import ecdsa -import pbkdf2 from .util import print_error from .bitcoin import is_old_seed, is_new_seed @@ -131,7 +130,7 @@ class Mnemonic(object): PBKDF2_ROUNDS = 2048 mnemonic = normalize_text(mnemonic) passphrase = normalize_text(passphrase) - return pbkdf2.PBKDF2(mnemonic, 'electrum' + passphrase, iterations = PBKDF2_ROUNDS, macmodule = hmac, digestmodule = hashlib.sha512).read(64) + return hashlib.pbkdf2_hmac('sha512', mnemonic.encode('utf-8'), b'electrum' + passphrase.encode('utf-8'), iterations = PBKDF2_ROUNDS) def mnemonic_encode(self, i): n = len(self.wordlist) diff --git a/electrum/plugins/digitalbitbox/digitalbitbox.py b/electrum/plugins/digitalbitbox/digitalbitbox.py index ed8c05f8..37db2644 100644 --- a/electrum/plugins/digitalbitbox/digitalbitbox.py +++ b/electrum/plugins/digitalbitbox/digitalbitbox.py @@ -120,8 +120,8 @@ class DigitalBitbox_Client(): def stretch_key(self, key): - import pbkdf2, hmac - return to_hexstr(pbkdf2.PBKDF2(key, b'Digital Bitbox', iterations = 20480, macmodule = hmac, digestmodule = hashlib.sha512).read(64)) + import hmac + return to_hexstr(hashlib.pbkdf2_hmac('sha512', key.encode('utf-8'), b'Digital Bitbox', iterations = 20480)) def backup_password_dialog(self): diff --git a/electrum/storage.py b/electrum/storage.py index 70fb19a5..0524cd23 100644 --- a/electrum/storage.py +++ b/electrum/storage.py @@ -29,7 +29,7 @@ import json import copy import re import stat -import pbkdf2, hmac, hashlib +import hmac, hashlib import base64 import zlib from collections import defaultdict @@ -165,7 +165,7 @@ class WalletStorage(PrintError): @staticmethod def get_eckey_from_password(password): - secret = pbkdf2.PBKDF2(password, '', iterations=1024, macmodule=hmac, digestmodule=hashlib.sha512).read(64) + secret = hashlib.pbkdf2_hmac('sha512', password.encode('utf-8'), b'', iterations=1024) ec_key = ecc.ECPrivkey.from_arbitrary_size_secret(secret) return ec_key @@ -637,7 +637,7 @@ class WalletStorage(PrintError): # version 1.9.8 created v6 wallets when an incorrect seed was entered in the restore dialog msg += '\n\nThis file was created because of a bug in version 1.9.8.' if self.get('master_public_keys') is None and self.get('master_private_keys') is None and self.get('imported_keys') is None: - # pbkdf2 was not included with the binaries, and wallet creation aborted. + # pbkdf2 (at that time an additional dependency) was not included with the binaries, and wallet creation aborted. msg += "\nIt does not contain any keys, and can safely be removed." else: # creation was complete if electrum was run from source diff --git a/run_electrum b/run_electrum index b5d5af00..96b055e0 100755 --- a/run_electrum +++ b/run_electrum @@ -46,7 +46,6 @@ def check_imports(): import ecdsa import requests import qrcode - import pbkdf2 import google.protobuf import jsonrpclib except ImportError as e: From e5661156f0d4667a91ea60cae5cd8189e0770a92 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Wed, 18 Jul 2018 15:32:26 +0200 Subject: [PATCH 10/56] follow-up e3888752d6508ebbb9b4e9a8a09555914cd68fd2 --- electrum/wallet.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/electrum/wallet.py b/electrum/wallet.py index b207a4d6..2ed0af36 100644 --- a/electrum/wallet.py +++ b/electrum/wallet.py @@ -53,7 +53,8 @@ from .storage import multisig_type, STO_EV_PLAINTEXT, STO_EV_USER_PW, STO_EV_XPU from . import transaction, bitcoin, coinchooser, paymentrequest, contacts from .transaction import Transaction from .plugin import run_hook -from .address_synchronizer import AddressSynchronizer +from .address_synchronizer import (AddressSynchronizer, TX_HEIGHT_LOCAL, + TX_HEIGHT_UNCONF_PARENT, TX_HEIGHT_UNCONFIRMED) from .paymentrequest import PR_PAID, PR_UNPAID, PR_UNKNOWN, PR_EXPIRED from .paymentrequest import InvoiceStore From 27b36486df456f46712a2a4dd275b703f53570f7 Mon Sep 17 00:00:00 2001 From: Yura Pakhuchiy Date: Wed, 18 Jul 2018 22:39:32 +0700 Subject: [PATCH 11/56] Trezor: fix spending coinbase outputs (#4565) Attempt to spend coinbase output results in error: a bytes-like object is required, not 'str' --- electrum/plugins/keepkey/keepkey.py | 2 +- electrum/plugins/trezor/trezor.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/electrum/plugins/keepkey/keepkey.py b/electrum/plugins/keepkey/keepkey.py index 053cfe39..e0f568b8 100644 --- a/electrum/plugins/keepkey/keepkey.py +++ b/electrum/plugins/keepkey/keepkey.py @@ -288,7 +288,7 @@ class KeepKeyPlugin(HW_PluginBase): for txin in tx.inputs(): txinputtype = self.types.TxInputType() if txin['type'] == 'coinbase': - prev_hash = "\0"*32 + prev_hash = b"\x00"*32 prev_index = 0xffffffff # signed int -1 else: if for_sig: diff --git a/electrum/plugins/trezor/trezor.py b/electrum/plugins/trezor/trezor.py index 94b0754f..77af6875 100644 --- a/electrum/plugins/trezor/trezor.py +++ b/electrum/plugins/trezor/trezor.py @@ -362,7 +362,7 @@ class TrezorPlugin(HW_PluginBase): for txin in tx.inputs(): txinputtype = self.types.TxInputType() if txin['type'] == 'coinbase': - prev_hash = "\0"*32 + prev_hash = b"\x00"*32 prev_index = 0xffffffff # signed int -1 else: if for_sig: From aa86440866a4d4ab7e8f8601ad2906f4689deec1 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Wed, 18 Jul 2018 18:42:04 +0200 Subject: [PATCH 12/56] fix #4566: bip39 passphrases with multiple spaces --- electrum/base_wizard.py | 11 ++++--- electrum/gui/qt/installwizard.py | 17 ++++++++-- electrum/tests/test_wallet_vertical.py | 45 +++++++++++++++++++++++++- 3 files changed, 66 insertions(+), 7 deletions(-) diff --git a/electrum/base_wizard.py b/electrum/base_wizard.py index fd3c8cc3..d5a0f83a 100644 --- a/electrum/base_wizard.py +++ b/electrum/base_wizard.py @@ -348,7 +348,7 @@ class BaseWizard(object): k = hardware_keystore(d) self.on_keystore(k) - def passphrase_dialog(self, run_next): + def passphrase_dialog(self, run_next, is_restoring=False): title = _('Seed extension') message = '\n'.join([ _('You may extend your seed with custom words.'), @@ -358,7 +358,10 @@ class BaseWizard(object): _('Note that this is NOT your encryption password.'), _('If you do not know what this is, leave this field empty.'), ]) - self.line_dialog(title=title, message=message, warning=warning, default='', test=lambda x:True, run_next=run_next) + warn_issue4566 = is_restoring and self.seed_type == 'bip39' + self.line_dialog(title=title, message=message, warning=warning, + default='', test=lambda x:True, run_next=run_next, + warn_issue4566=warn_issue4566) def restore_from_seed(self): self.opt_bip39 = True @@ -371,10 +374,10 @@ class BaseWizard(object): self.seed_type = 'bip39' if is_bip39 else bitcoin.seed_type(seed) if self.seed_type == 'bip39': f = lambda passphrase: self.on_restore_bip39(seed, passphrase) - self.passphrase_dialog(run_next=f) if is_ext else f('') + self.passphrase_dialog(run_next=f, is_restoring=True) if is_ext else f('') elif self.seed_type in ['standard', 'segwit']: f = lambda passphrase: self.run('create_keystore', seed, passphrase) - self.passphrase_dialog(run_next=f) if is_ext else f('') + self.passphrase_dialog(run_next=f, is_restoring=True) if is_ext else f('') elif self.seed_type == 'old': self.run('create_keystore', seed, '') elif self.seed_type == '2fa': diff --git a/electrum/gui/qt/installwizard.py b/electrum/gui/qt/installwizard.py index 79647235..cd45e6f3 100644 --- a/electrum/gui/qt/installwizard.py +++ b/electrum/gui/qt/installwizard.py @@ -32,6 +32,12 @@ WIF_HELP_TEXT = (_('WIF keys are typed in Electrum, based on script type.') + '\ 'p2wpkh-p2sh:KxZcY47uGp9a... \t-> 3NhNeZQXF...\n' + 'p2wpkh:KxZcY47uGp9a... \t-> bc1q3fjfk...') # note: full key is KxZcY47uGp9aVQAb6VVvuBs8SwHKgkSR2DbZUzjDzXf2N2GPhG9n +MSG_PASSPHRASE_WARN_ISSUE4566 = _("Warning") + ": "\ + + _("You have multiple consecutive whitespaces or leading/trailing " + "whitespaces in your passphrase.") + " " \ + + _("This is discouraged.") + " " \ + + _("Due to a bug, old versions of Electrum will NOT be creating the " + "same wallet as newer versions or other software.") class CosignWidget(QWidget): @@ -550,17 +556,24 @@ class InstallWizard(QDialog, MessageBoxMixin, BaseWizard): @wizard_dialog def line_dialog(self, run_next, title, message, default, test, warning='', - presets=()): + presets=(), warn_issue4566=False): vbox = QVBoxLayout() vbox.addWidget(WWLabel(message)) line = QLineEdit() line.setText(default) def f(text): self.next_button.setEnabled(test(text)) + if warn_issue4566: + text_whitespace_normalised = ' '.join(text.split()) + warn_issue4566_label.setVisible(text != text_whitespace_normalised) line.textEdited.connect(f) vbox.addWidget(line) vbox.addWidget(WWLabel(warning)) + warn_issue4566_label = WWLabel(MSG_PASSPHRASE_WARN_ISSUE4566) + warn_issue4566_label.setVisible(False) + vbox.addWidget(warn_issue4566_label) + for preset in presets: button = QPushButton(preset[0]) button.clicked.connect(lambda __, text=preset[1]: line.setText(text)) @@ -570,7 +583,7 @@ class InstallWizard(QDialog, MessageBoxMixin, BaseWizard): vbox.addLayout(hbox) self.exec_layout(vbox, title, next_enabled=test(default)) - return ' '.join(line.text().split()) + return line.text() @wizard_dialog def show_xpub_dialog(self, xpub, run_next): diff --git a/electrum/tests/test_wallet_vertical.py b/electrum/tests/test_wallet_vertical.py index c95d9831..1b03983a 100644 --- a/electrum/tests/test_wallet_vertical.py +++ b/electrum/tests/test_wallet_vertical.py @@ -18,6 +18,11 @@ from . import SequentialTestCase from .test_bitcoin import needs_test_with_all_ecc_implementations +_UNICODE_HORROR_HEX = 'e282bf20f09f988020f09f98882020202020e3818620e38191e3819fe381be20e3828fe3828b2077cda2cda2cd9d68cda16fcda2cda120ccb8cda26bccb5cd9f6eccb4cd98c7ab77ccb8cc9b73cd9820cc80cc8177cd98cda2e1b8a9ccb561d289cca1cda27420cca7cc9568cc816fccb572cd8fccb5726f7273cca120ccb6cda1cda06cc4afccb665cd9fcd9f20ccb6cd9d696ecda220cd8f74cc9568ccb7cca1cd9f6520cd9fcd9f64cc9b61cd9c72cc95cda16bcca2cca820cda168ccb465cd8f61ccb7cca2cca17274cc81cd8f20ccb4ccb7cda0c3b2ccb5ccb666ccb82075cca7cd986ec3adcc9bcd9c63cda2cd8f6fccb7cd8f64ccb8cda265cca1cd9d3fcd9e' +UNICODE_HORROR = bfh(_UNICODE_HORROR_HEX).decode('utf-8') +# '₿ 😀 😈 う けたま わる w͢͢͝h͡o͢͡ ̸͢k̵͟n̴͘ǫw̸̛s͘ ̀́w͘͢ḩ̵a҉̡͢t ̧̕h́o̵r͏̵rors̡ ̶͡͠lį̶e͟͟ ̶͝in͢ ͏t̕h̷̡͟e ͟͟d̛a͜r̕͡k̢̨ ͡h̴e͏a̷̢̡rt́͏ ̴̷͠ò̵̶f̸ u̧͘ní̛͜c͢͏o̷͏d̸͢e̡͝?͞' + + class WalletIntegrityHelper: gap_limit = 1 # make tests run faster @@ -68,7 +73,6 @@ class WalletIntegrityHelper: return w -# TODO passphrase/seed_extension class TestWalletKeystoreAddressIntegrityForMainnet(SequentialTestCase): @needs_test_with_all_ecc_implementations @@ -111,6 +115,26 @@ class TestWalletKeystoreAddressIntegrityForMainnet(SequentialTestCase): self.assertEqual(w.get_receiving_addresses()[0], 'bc1q3g5tmkmlvxryhh843v4dz026avatc0zzr6h3af') self.assertEqual(w.get_change_addresses()[0], 'bc1qdy94n2q5qcp0kg7v9yzwe6wvfkhnvyzje7nx2p') + @needs_test_with_all_ecc_implementations + @mock.patch.object(storage.WalletStorage, '_write') + def test_electrum_seed_segwit_passphrase(self, mock_write): + seed_words = 'bitter grass shiver impose acquire brush forget axis eager alone wine silver' + self.assertEqual(bitcoin.seed_type(seed_words), 'segwit') + + ks = keystore.from_seed(seed_words, UNICODE_HORROR, False) + + WalletIntegrityHelper.check_seeded_keystore_sanity(self, ks) + self.assertTrue(isinstance(ks, keystore.BIP32_KeyStore)) + + self.assertEqual(ks.xprv, 'zprvAZDmEQiCLUcZXPfrBXoksCD2R6RMAzAre7SUyBotibisy9c7vGhLYvHaP3d9rYU12DKAWdZfscPNA7qEPgTkCDqX5sE93ryAJAQvkDbfLxU') + self.assertEqual(ks.xpub, 'zpub6nD7dvF6ArArjskKHZLmEL9ky8FqaSti1LN5maDWGwFrqwwGTp1b6ic4EHwciFNaYDmCXcQYxXSiF9BjcLCMPcaYkVN2nQD6QjYQ8vpSR3Z') + + w = WalletIntegrityHelper.create_standard_wallet(ks) + self.assertEqual(w.txin_type, 'p2wpkh') + + self.assertEqual(w.get_receiving_addresses()[0], 'bc1qx94dutas7ysn2my645cyttujrms5d9p57f6aam') + self.assertEqual(w.get_change_addresses()[0], 'bc1qcywwsy87sdp8vz5rfjh3sxdv6rt95kujdqq38g') + @needs_test_with_all_ecc_implementations @mock.patch.object(storage.WalletStorage, '_write') def test_electrum_seed_old(self, mock_write): @@ -183,6 +207,25 @@ class TestWalletKeystoreAddressIntegrityForMainnet(SequentialTestCase): self.assertEqual(w.get_receiving_addresses()[0], '16j7Dqk3Z9DdTdBtHcCVLaNQy9MTgywUUo') self.assertEqual(w.get_change_addresses()[0], '1GG5bVeWgAp5XW7JLCphse14QaC4qiHyWn') + @needs_test_with_all_ecc_implementations + @mock.patch.object(storage.WalletStorage, '_write') + def test_bip39_seed_bip44_standard_passphrase(self, mock_write): + seed_words = 'treat dwarf wealth gasp brass outside high rent blood crowd make initial' + self.assertEqual(keystore.bip39_is_checksum_valid(seed_words), (True, True)) + + ks = keystore.from_bip39_seed(seed_words, UNICODE_HORROR, "m/44'/0'/0'") + + self.assertTrue(isinstance(ks, keystore.BIP32_KeyStore)) + + self.assertEqual(ks.xprv, 'xprv9z8izheguGnLopSqkY7GcGFrP2Gu6rzBvvHo6uB9B8DWJhsows6WDZAsbBTaP3ncP2AVbTQphyEQkahrB9s1L7ihZtfz5WGQPMbXwsUtSik') + self.assertEqual(ks.xpub, 'xpub6D85QDBajeLe2JXJrZeGyQCaw47PWKi3J9DPuHakjTkVBWCxVQQkmMVMSSfnw39tj9FntbozpRtb1AJ8ubjeVSBhyK4M5mzdvsXZzKPwodT') + + w = WalletIntegrityHelper.create_standard_wallet(ks) + self.assertEqual(w.txin_type, 'p2pkh') + + self.assertEqual(w.get_receiving_addresses()[0], '1F88g2naBMhDB7pYFttPWGQgryba3hPevM') + self.assertEqual(w.get_change_addresses()[0], '1H4QD1rg2zQJ4UjuAVJr5eW1fEM8WMqyxh') + @needs_test_with_all_ecc_implementations @mock.patch.object(storage.WalletStorage, '_write') def test_bip39_seed_bip49_p2sh_segwit(self, mock_write): From 8f17f38b02406ff4f90940ea66dda4b5f93bf227 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Wed, 18 Jul 2018 20:17:03 +0200 Subject: [PATCH 13/56] trezor/kk: when using old fw, wizard did not display instructions properly --- electrum/plugins/keepkey/keepkey.py | 5 ++++- electrum/plugins/trezor/trezor.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/electrum/plugins/keepkey/keepkey.py b/electrum/plugins/keepkey/keepkey.py index e0f568b8..611fe844 100644 --- a/electrum/plugins/keepkey/keepkey.py +++ b/electrum/plugins/keepkey/keepkey.py @@ -141,7 +141,10 @@ class KeepKeyPlugin(HW_PluginBase): 'download the updated firmware from {}') .format(self.device, client.label(), self.firmware_URL)) self.print_error(msg) - handler.show_error(msg) + if handler: + handler.show_error(msg) + else: + raise Exception(msg) return None return client diff --git a/electrum/plugins/trezor/trezor.py b/electrum/plugins/trezor/trezor.py index 77af6875..7fecd1e6 100644 --- a/electrum/plugins/trezor/trezor.py +++ b/electrum/plugins/trezor/trezor.py @@ -157,7 +157,10 @@ class TrezorPlugin(HW_PluginBase): 'download the updated firmware from {}') .format(self.device, client.label(), self.firmware_URL)) self.print_error(msg) - handler.show_error(msg) + if handler: + handler.show_error(msg) + else: + raise Exception(msg) return None return client From 780b2d067c1ade3575e9511e9e3c71a4a39f50c2 Mon Sep 17 00:00:00 2001 From: Janus Date: Wed, 18 Jul 2018 13:31:41 +0200 Subject: [PATCH 14/56] Whitelist classes in verbose (-v) option --- electrum/commands.py | 2 +- electrum/interface.py | 1 + electrum/network.py | 1 + electrum/plugin.py | 1 + electrum/util.py | 16 +++++++++------- electrum/wallet.py | 1 + run_electrum | 4 ++-- 7 files changed, 16 insertions(+), 10 deletions(-) diff --git a/electrum/commands.py b/electrum/commands.py index a95d74ed..88d1c63c 100644 --- a/electrum/commands.py +++ b/electrum/commands.py @@ -829,7 +829,7 @@ def add_network_options(parser): def add_global_options(parser): group = parser.add_argument_group('global options') - group.add_argument("-v", "--verbose", action="store_true", dest="verbose", default=False, help="Show debugging information") + group.add_argument("-v", "--verbosity", dest="verbosity", default='', help="Set verbosity filter") group.add_argument("-D", "--dir", dest="electrum_path", help="electrum directory") group.add_argument("-P", "--portable", action="store_true", dest="portable", default=False, help="Use local 'electrum_data' directory") group.add_argument("-w", "--wallet", dest="wallet_path", help="wallet path") diff --git a/electrum/interface.py b/electrum/interface.py index 98305bb1..357d913a 100644 --- a/electrum/interface.py +++ b/electrum/interface.py @@ -59,6 +59,7 @@ def Connection(server, queue, config_path): class TcpConnection(threading.Thread, util.PrintError): + verbosity_filter = 'i' def __init__(self, server, queue, config_path): threading.Thread.__init__(self) diff --git a/electrum/network.py b/electrum/network.py index b836fbc7..c90ae823 100644 --- a/electrum/network.py +++ b/electrum/network.py @@ -170,6 +170,7 @@ class Network(util.DaemonThread): get_parameters(), get_server_height(), get_status_value(), is_connected(), set_parameters(), stop() """ + verbosity_filter = 'n' def __init__(self, config=None): if config is None: diff --git a/electrum/plugin.py b/electrum/plugin.py index c5223dc3..93ba9995 100644 --- a/electrum/plugin.py +++ b/electrum/plugin.py @@ -43,6 +43,7 @@ hooks = {} class Plugins(DaemonThread): + verbosity_filter = 'p' @profiler def __init__(self, config, is_local, gui_name): diff --git a/electrum/util.py b/electrum/util.py index 0e9f1285..aced2746 100644 --- a/electrum/util.py +++ b/electrum/util.py @@ -164,12 +164,14 @@ class MyEncoder(json.JSONEncoder): class PrintError(object): '''A handy base class''' + verbosity_filter = '' + def diagnostic_name(self): return self.__class__.__name__ def print_error(self, *msg): - # only prints with --verbose flag - print_error("[%s]" % self.diagnostic_name(), *msg) + if self.verbosity_filter in verbosity: + print_stderr("[%s]" % self.diagnostic_name(), *msg) def print_stderr(self, *msg): print_stderr("[%s]" % self.diagnostic_name(), *msg) @@ -213,6 +215,7 @@ class DebugMem(ThreadJob): class DaemonThread(threading.Thread, PrintError): """ daemon thread that terminates cleanly """ + verbosity_filter = 'd' def __init__(self): threading.Thread.__init__(self) @@ -263,15 +266,14 @@ class DaemonThread(threading.Thread, PrintError): self.print_error("stopped") -# TODO: disable -is_verbose = True +verbosity = '' def set_verbosity(b): - global is_verbose - is_verbose = b + global verbosity + verbosity = b def print_error(*args): - if not is_verbose: return + if not verbosity: return print_stderr(*args) def print_stderr(*args): diff --git a/electrum/wallet.py b/electrum/wallet.py index 2ed0af36..26efdcef 100644 --- a/electrum/wallet.py +++ b/electrum/wallet.py @@ -165,6 +165,7 @@ class Abstract_Wallet(AddressSynchronizer): max_change_outputs = 3 gap_limit_for_change = 6 + verbosity_filter = 'w' def __init__(self, storage): AddressSynchronizer.__init__(self, storage) diff --git a/run_electrum b/run_electrum index 96b055e0..46654ec1 100755 --- a/run_electrum +++ b/run_electrum @@ -355,7 +355,7 @@ if __name__ == '__main__': # config is an object passed to the various constructors (wallet, interface, gui) if is_android: config_options = { - 'verbose': True, + 'verbosity': '', 'cmd': 'gui', 'gui': 'kivy', } @@ -376,7 +376,7 @@ if __name__ == '__main__': config_options['electrum_path'] = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'electrum_data') # kivy sometimes freezes when we write to sys.stderr - set_verbosity(config_options.get('verbose') and config_options.get('gui')!='kivy') + set_verbosity(config_options.get('verbosity') if config_options.get('gui') != 'kivy' else '') # check uri uri = config_options.get('url') From 0025073b24cf168e2ec29a00640f9280b18f2877 Mon Sep 17 00:00:00 2001 From: ThomasV Date: Thu, 19 Jul 2018 10:15:22 +0200 Subject: [PATCH 15/56] move more methods from wallet to address_synchronizer --- electrum/address_synchronizer.py | 255 ++++++++++++++++++++++++++++- electrum/wallet.py | 266 +------------------------------ 2 files changed, 255 insertions(+), 266 deletions(-) diff --git a/electrum/address_synchronizer.py b/electrum/address_synchronizer.py index bbc8b0b2..63e4a595 100644 --- a/electrum/address_synchronizer.py +++ b/electrum/address_synchronizer.py @@ -25,6 +25,8 @@ import threading import itertools from collections import defaultdict +from . import bitcoin +from .bitcoin import COINBASE_MATURITY from .util import PrintError, profiler from .transaction import Transaction from .synchronizer import Synchronizer @@ -66,9 +68,50 @@ class AddressSynchronizer(PrintError): self.up_to_date = False self.load_transactions() self.load_local_history() + self.check_history() self.load_unverified_transactions() self.remove_local_transactions_we_dont_have() + def is_mine(self, address): + return address in self.history + + def get_addresses(self): + return sorted(self.history.keys()) + + def get_address_history(self, addr): + h = [] + # we need self.transaction_lock but get_tx_height will take self.lock + # so we need to take that too here, to enforce order of locks + with self.lock, self.transaction_lock: + related_txns = self._history_local.get(addr, set()) + for tx_hash in related_txns: + tx_height = self.get_tx_height(tx_hash)[0] + h.append((tx_hash, tx_height)) + return h + + def get_txin_address(self, txi): + addr = txi.get('address') + if addr and addr != "(pubkey)": + return addr + prevout_hash = txi.get('prevout_hash') + prevout_n = txi.get('prevout_n') + dd = self.txo.get(prevout_hash, {}) + for addr, l in dd.items(): + for n, v, is_cb in l: + if n == prevout_n: + return addr + return None + + def get_txout_address(self, txo): + _type, x, v = txo + if _type == TYPE_ADDRESS: + addr = x + elif _type == TYPE_PUBKEY: + addr = bitcoin.public_key_to_p2pkh(bfh(x)) + else: + addr = None + return addr + def load_unverified_transactions(self): # review transactions that are in the history for addr, hist in self.history.items(): @@ -322,6 +365,26 @@ class AddressSynchronizer(PrintError): for txid in itertools.chain(self.txi, self.txo): self._add_tx_to_local_history(txid) + @profiler + def check_history(self): + save = False + hist_addrs_mine = list(filter(lambda k: self.is_mine(k), self.history.keys())) + hist_addrs_not_mine = list(filter(lambda k: not self.is_mine(k), self.history.keys())) + for addr in hist_addrs_not_mine: + self.history.pop(addr) + save = True + for addr in hist_addrs_mine: + hist = self.history[addr] + for tx_hash, tx_height in hist: + if self.txi.get(tx_hash) or self.txo.get(tx_hash): + continue + tx = self.transactions.get(tx_hash) + if tx is not None: + self.add_transaction(tx_hash, tx, allow_unrelated=True) + save = True + if save: + self.save_transactions() + def remove_local_transactions_we_dont_have(self): txid_set = set(self.txi) | set(self.txo) for txid in txid_set: @@ -362,10 +425,22 @@ class AddressSynchronizer(PrintError): self.transactions = {} self.save_transactions() + def get_txpos(self, tx_hash): + "return position, even if the tx is unverified" + with self.lock: + if tx_hash in self.verified_tx: + height, timestamp, pos = self.verified_tx[tx_hash] + return height, pos + elif tx_hash in self.unverified_tx: + height = self.unverified_tx[tx_hash] + return (height, 0) if height > 0 else ((1e9 - height), 0) + else: + return (1e9+1, 0) + def get_history(self, domain=None): # get domain if domain is None: - domain = self.get_addresses() + domain = self.history.keys() domain = set(domain) # 1. Get the history of each address in the domain, maintain the # delta of a tx as the sum of its deltas on domain addresses @@ -492,3 +567,181 @@ class AddressSynchronizer(PrintError): def is_up_to_date(self): with self.lock: return self.up_to_date + + def get_num_tx(self, address): + """ return number of transactions where address is involved """ + return len(self.history.get(address, [])) + + def get_tx_delta(self, tx_hash, address): + "effect of tx on address" + delta = 0 + # substract the value of coins sent from address + d = self.txi.get(tx_hash, {}).get(address, []) + for n, v in d: + delta -= v + # add the value of the coins received at address + d = self.txo.get(tx_hash, {}).get(address, []) + for n, v, cb in d: + delta += v + return delta + + def get_tx_value(self, txid): + " effect of tx on the entire domain" + delta = 0 + for addr, d in self.txi.get(txid, {}).items(): + for n, v in d: + delta -= v + for addr, d in self.txo.get(txid, {}).items(): + for n, v, cb in d: + delta += v + return delta + + def get_wallet_delta(self, tx): + """ effect of tx on wallet """ + is_relevant = False # "related to wallet?" + is_mine = False + is_pruned = False + is_partial = False + v_in = v_out = v_out_mine = 0 + for txin in tx.inputs(): + addr = self.get_txin_address(txin) + if self.is_mine(addr): + is_mine = True + is_relevant = True + d = self.txo.get(txin['prevout_hash'], {}).get(addr, []) + for n, v, cb in d: + if n == txin['prevout_n']: + value = v + break + else: + value = None + if value is None: + is_pruned = True + else: + v_in += value + else: + is_partial = True + if not is_mine: + is_partial = False + for addr, value in tx.get_outputs(): + v_out += value + if self.is_mine(addr): + v_out_mine += value + is_relevant = True + if is_pruned: + # some inputs are mine: + fee = None + if is_mine: + v = v_out_mine - v_out + else: + # no input is mine + v = v_out_mine + else: + v = v_out_mine - v_in + if is_partial: + # some inputs are mine, but not all + fee = None + else: + # all inputs are mine + fee = v_in - v_out + if not is_mine: + fee = None + return is_relevant, is_mine, v, fee + + def get_addr_io(self, address): + h = self.get_address_history(address) + received = {} + sent = {} + for tx_hash, height in h: + l = self.txo.get(tx_hash, {}).get(address, []) + for n, v, is_cb in l: + received[tx_hash + ':%d'%n] = (height, v, is_cb) + for tx_hash, height in h: + l = self.txi.get(tx_hash, {}).get(address, []) + for txi, v in l: + sent[txi] = height + return received, sent + + def get_addr_utxo(self, address): + coins, spent = self.get_addr_io(address) + for txi in spent: + coins.pop(txi) + out = {} + for txo, v in coins.items(): + tx_height, value, is_cb = v + prevout_hash, prevout_n = txo.split(':') + x = { + 'address':address, + 'value':value, + 'prevout_n':int(prevout_n), + 'prevout_hash':prevout_hash, + 'height':tx_height, + 'coinbase':is_cb + } + out[txo] = x + return out + + # return the total amount ever received by an address + def get_addr_received(self, address): + received, sent = self.get_addr_io(address) + return sum([v for height, v, is_cb in received.values()]) + + # return the balance of a bitcoin address: confirmed and matured, unconfirmed, unmatured + def get_addr_balance(self, address): + received, sent = self.get_addr_io(address) + c = u = x = 0 + local_height = self.get_local_height() + for txo, (tx_height, v, is_cb) in received.items(): + if is_cb and tx_height + COINBASE_MATURITY > local_height: + x += v + elif tx_height > 0: + c += v + else: + u += v + if txo in sent: + if sent[txo] > 0: + c -= v + else: + u -= v + return c, u, x + + def get_utxos(self, domain=None, excluded=None, mature=False, confirmed_only=False): + coins = [] + if domain is None: + domain = self.get_addresses() + domain = set(domain) + if excluded: + domain = set(domain) - excluded + for addr in domain: + utxos = self.get_addr_utxo(addr) + for x in utxos.values(): + if confirmed_only and x['height'] <= 0: + continue + if mature and x['coinbase'] and x['height'] + COINBASE_MATURITY > self.get_local_height(): + continue + coins.append(x) + continue + return coins + + def get_balance(self, domain=None): + if domain is None: + domain = self.get_addresses() + domain = set(domain) + cc = uu = xx = 0 + for addr in domain: + c, u, x = self.get_addr_balance(addr) + cc += c + uu += u + xx += x + return cc, uu, xx + + def is_used(self, address): + h = self.history.get(address,[]) + if len(h) == 0: + return False + c, u, x = self.get_addr_balance(address) + return c + u + x == 0 + + def is_empty(self, address): + c, u, x = self.get_addr_balance(address) + return c+u+x == 0 diff --git a/electrum/wallet.py b/electrum/wallet.py index 26efdcef..93e230f8 100644 --- a/electrum/wallet.py +++ b/electrum/wallet.py @@ -182,8 +182,6 @@ class Abstract_Wallet(AddressSynchronizer): self.load_addresses() self.test_addresses_sanity() - self.check_history() - # save wallet type the first time if self.storage.get('wallet_type') is None: self.storage.put('wallet_type', self.wallet_type) @@ -203,26 +201,6 @@ class Abstract_Wallet(AddressSynchronizer): def get_master_public_key(self): return None - @profiler - def check_history(self): - save = False - hist_addrs_mine = list(filter(lambda k: self.is_mine(k), self.history.keys())) - hist_addrs_not_mine = list(filter(lambda k: not self.is_mine(k), self.history.keys())) - for addr in hist_addrs_not_mine: - self.history.pop(addr) - save = True - for addr in hist_addrs_mine: - hist = self.history[addr] - for tx_hash, tx_height in hist: - if self.txi.get(tx_hash) or self.txo.get(tx_hash): - continue - tx = self.transactions.get(tx_hash) - if tx is not None: - self.add_transaction(tx_hash, tx, allow_unrelated=True) - save = True - if save: - self.save_transactions() - def basename(self): return os.path.basename(self.storage.path) @@ -290,9 +268,6 @@ class Abstract_Wallet(AddressSynchronizer): except: return - def is_mine(self, address): - return address in self.get_addresses() - def is_change(self, address): if not self.is_mine(address): return False @@ -317,101 +292,9 @@ class Abstract_Wallet(AddressSynchronizer): def get_public_keys(self, address): return [self.get_public_key(address)] - def get_txpos(self, tx_hash): - "return position, even if the tx is unverified" - with self.lock: - if tx_hash in self.verified_tx: - height, timestamp, pos = self.verified_tx[tx_hash] - return height, pos - elif tx_hash in self.unverified_tx: - height = self.unverified_tx[tx_hash] - return (height, 0) if height > 0 else ((1e9 - height), 0) - else: - return (1e9+1, 0) - def is_found(self): return self.history.values() != [[]] * len(self.history) - def get_num_tx(self, address): - """ return number of transactions where address is involved """ - return len(self.history.get(address, [])) - - def get_tx_delta(self, tx_hash, address): - "effect of tx on address" - delta = 0 - # substract the value of coins sent from address - d = self.txi.get(tx_hash, {}).get(address, []) - for n, v in d: - delta -= v - # add the value of the coins received at address - d = self.txo.get(tx_hash, {}).get(address, []) - for n, v, cb in d: - delta += v - return delta - - def get_tx_value(self, txid): - " effect of tx on the entire domain" - delta = 0 - for addr, d in self.txi.get(txid, {}).items(): - for n, v in d: - delta -= v - for addr, d in self.txo.get(txid, {}).items(): - for n, v, cb in d: - delta += v - return delta - - def get_wallet_delta(self, tx): - """ effect of tx on wallet """ - is_relevant = False # "related to wallet?" - is_mine = False - is_pruned = False - is_partial = False - v_in = v_out = v_out_mine = 0 - for txin in tx.inputs(): - addr = self.get_txin_address(txin) - if self.is_mine(addr): - is_mine = True - is_relevant = True - d = self.txo.get(txin['prevout_hash'], {}).get(addr, []) - for n, v, cb in d: - if n == txin['prevout_n']: - value = v - break - else: - value = None - if value is None: - is_pruned = True - else: - v_in += value - else: - is_partial = True - if not is_mine: - is_partial = False - for addr, value in tx.get_outputs(): - v_out += value - if self.is_mine(addr): - v_out_mine += value - is_relevant = True - if is_pruned: - # some inputs are mine: - fee = None - if is_mine: - v = v_out_mine - v_out - else: - # no input is mine - v = v_out_mine - else: - v = v_out_mine - v_in - if is_partial: - # some inputs are mine, but not all - fee = None - else: - # all inputs are mine - fee = v_in - v_out - if not is_mine: - fee = None - return is_relevant, is_mine, v, fee - def get_tx_info(self, tx): is_relevant, is_mine, v, fee = self.get_wallet_delta(tx) exp_n = None @@ -461,143 +344,16 @@ class Abstract_Wallet(AddressSynchronizer): return tx_hash, status, label, can_broadcast, can_bump, amount, fee, height, conf, timestamp, exp_n - def get_addr_io(self, address): - h = self.get_address_history(address) - received = {} - sent = {} - for tx_hash, height in h: - l = self.txo.get(tx_hash, {}).get(address, []) - for n, v, is_cb in l: - received[tx_hash + ':%d'%n] = (height, v, is_cb) - for tx_hash, height in h: - l = self.txi.get(tx_hash, {}).get(address, []) - for txi, v in l: - sent[txi] = height - return received, sent - - def get_addr_utxo(self, address): - coins, spent = self.get_addr_io(address) - for txi in spent: - coins.pop(txi) - out = {} - for txo, v in coins.items(): - tx_height, value, is_cb = v - prevout_hash, prevout_n = txo.split(':') - x = { - 'address':address, - 'value':value, - 'prevout_n':int(prevout_n), - 'prevout_hash':prevout_hash, - 'height':tx_height, - 'coinbase':is_cb - } - out[txo] = x - return out - - # return the total amount ever received by an address - def get_addr_received(self, address): - received, sent = self.get_addr_io(address) - return sum([v for height, v, is_cb in received.values()]) - - # return the balance of a bitcoin address: confirmed and matured, unconfirmed, unmatured - def get_addr_balance(self, address): - received, sent = self.get_addr_io(address) - c = u = x = 0 - local_height = self.get_local_height() - for txo, (tx_height, v, is_cb) in received.items(): - if is_cb and tx_height + COINBASE_MATURITY > local_height: - x += v - elif tx_height > 0: - c += v - else: - u += v - if txo in sent: - if sent[txo] > 0: - c -= v - else: - u -= v - return c, u, x - def get_spendable_coins(self, domain, config): confirmed_only = config.get('confirmed_only', False) - return self.get_utxos(domain, exclude_frozen=True, mature=True, confirmed_only=confirmed_only) - - def get_utxos(self, domain = None, exclude_frozen = False, mature = False, confirmed_only = False): - coins = [] - if domain is None: - domain = self.get_addresses() - domain = set(domain) - if exclude_frozen: - domain = set(domain) - self.frozen_addresses - for addr in domain: - utxos = self.get_addr_utxo(addr) - for x in utxos.values(): - if confirmed_only and x['height'] <= 0: - continue - if mature and x['coinbase'] and x['height'] + COINBASE_MATURITY > self.get_local_height(): - continue - coins.append(x) - continue - return coins + return self.get_utxos(domain, excluded=self.frozen_addresses, mature=True, confirmed_only=confirmed_only) def dummy_address(self): return self.get_receiving_addresses()[0] - def get_addresses(self): - out = [] - out += self.get_receiving_addresses() - out += self.get_change_addresses() - return out - def get_frozen_balance(self): return self.get_balance(self.frozen_addresses) - def get_balance(self, domain=None): - if domain is None: - domain = self.get_addresses() - domain = set(domain) - cc = uu = xx = 0 - for addr in domain: - c, u, x = self.get_addr_balance(addr) - cc += c - uu += u - xx += x - return cc, uu, xx - - def get_address_history(self, addr): - h = [] - # we need self.transaction_lock but get_tx_height will take self.lock - # so we need to take that too here, to enforce order of locks - with self.lock, self.transaction_lock: - related_txns = self._history_local.get(addr, set()) - for tx_hash in related_txns: - tx_height = self.get_tx_height(tx_hash)[0] - h.append((tx_hash, tx_height)) - return h - - def get_txin_address(self, txi): - addr = txi.get('address') - if addr and addr != "(pubkey)": - return addr - prevout_hash = txi.get('prevout_hash') - prevout_n = txi.get('prevout_n') - dd = self.txo.get(prevout_hash, {}) - for addr, l in dd.items(): - for n, v, is_cb in l: - if n == prevout_n: - return addr - return None - - def get_txout_address(self, txo): - _type, x, v = txo - if _type == TYPE_ADDRESS: - addr = x - elif _type == TYPE_PUBKEY: - addr = bitcoin.public_key_to_p2pkh(bfh(x)) - else: - addr = None - return addr - def balance_at_timestamp(self, domain, target_timestamp): h = self.get_history(domain) for tx_hash, height, conf, timestamp, value, balance in h: @@ -884,17 +640,6 @@ class Abstract_Wallet(AddressSynchronizer): def can_export(self): return not self.is_watching_only() and hasattr(self.keystore, 'get_private_key') - def is_used(self, address): - h = self.history.get(address,[]) - if len(h) == 0: - return False - c, u, x = self.get_addr_balance(address) - return c + u + x == 0 - - def is_empty(self, address): - c, u, x = self.get_addr_balance(address) - return c+u+x == 0 - def address_is_old(self, address, age_limit=2): age = -1 h = self.history.get(address, []) @@ -1458,15 +1203,9 @@ class Imported_Wallet(Simple_Wallet): def is_beyond_limit(self, address): return False - def is_mine(self, address): - return address in self.addresses - def get_fingerprint(self): return '' - def get_addresses(self, include_change=False): - return sorted(self.addresses.keys()) - def get_receiving_addresses(self): return self.get_addresses() @@ -1699,9 +1438,6 @@ class Deterministic_Wallet(Abstract_Wallet): return False return True - def is_mine(self, address): - return address in self._addr_to_addr_index - def get_address_index(self, address): return self._addr_to_addr_index[address] From b96b5af10191511efc1ef8cd93d655c028503a03 Mon Sep 17 00:00:00 2001 From: ThomasV Date: Thu, 19 Jul 2018 10:25:46 +0200 Subject: [PATCH 16/56] fix imports --- electrum/address_synchronizer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/electrum/address_synchronizer.py b/electrum/address_synchronizer.py index 63e4a595..196fcd67 100644 --- a/electrum/address_synchronizer.py +++ b/electrum/address_synchronizer.py @@ -26,7 +26,7 @@ import itertools from collections import defaultdict from . import bitcoin -from .bitcoin import COINBASE_MATURITY +from .bitcoin import COINBASE_MATURITY, TYPE_ADDRESS, TYPE_PUBKEY from .util import PrintError, profiler from .transaction import Transaction from .synchronizer import Synchronizer From f9f6ea436519a3df65591a039a8e18ff358a711c Mon Sep 17 00:00:00 2001 From: Janus Date: Thu, 19 Jul 2018 12:43:53 +0200 Subject: [PATCH 17/56] commands: tolerate lack of argument to 'verbosity' --- electrum/commands.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/electrum/commands.py b/electrum/commands.py index 88d1c63c..e9276d3b 100644 --- a/electrum/commands.py +++ b/electrum/commands.py @@ -829,7 +829,9 @@ def add_network_options(parser): def add_global_options(parser): group = parser.add_argument_group('global options') - group.add_argument("-v", "--verbosity", dest="verbosity", default='', help="Set verbosity filter") + # const is for when no argument is given to verbosity + # default is for when the flag is missing + group.add_argument("-v", "--verbosity", dest="verbosity", help="Set verbosity filter", default='', const='', nargs='?') group.add_argument("-D", "--dir", dest="electrum_path", help="electrum directory") group.add_argument("-P", "--portable", action="store_true", dest="portable", default=False, help="Use local 'electrum_data' directory") group.add_argument("-w", "--wallet", dest="wallet_path", help="wallet path") From 1fb0b6d7bd6e3a6deeb101a5d166de556a76de1c Mon Sep 17 00:00:00 2001 From: Janus Date: Thu, 19 Jul 2018 13:33:57 +0200 Subject: [PATCH 18/56] plugins/ledger: just hardcode BTCHIP_DEBUG to False --- electrum/plugins/ledger/ledger.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/electrum/plugins/ledger/ledger.py b/electrum/plugins/ledger/ledger.py index 73604a9a..bf7e6d2f 100644 --- a/electrum/plugins/ledger/ledger.py +++ b/electrum/plugins/ledger/ledger.py @@ -12,7 +12,7 @@ from electrum.transaction import Transaction from electrum.wallet import Standard_Wallet from ..hw_wallet import HW_PluginBase from ..hw_wallet.plugin import is_any_tx_output_on_change_branch -from electrum.util import print_error, is_verbose, bfh, bh2u, versiontuple +from electrum.util import print_error, bfh, bh2u, versiontuple from electrum.base_wizard import ScriptTypeNotSupported try: @@ -24,7 +24,7 @@ try: from btchip.btchipFirmwareWizard import checkFirmware, updateFirmware from btchip.btchipException import BTChipException BTCHIP = True - BTCHIP_DEBUG = is_verbose + BTCHIP_DEBUG = False except ImportError: BTCHIP = False From cc77ba523f3892ccbca681a5293c1fed34ab449b Mon Sep 17 00:00:00 2001 From: SomberNight Date: Thu, 19 Jul 2018 13:47:49 +0200 Subject: [PATCH 19/56] fix minor undefined stuff in address_synchronizer --- electrum/address_synchronizer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/electrum/address_synchronizer.py b/electrum/address_synchronizer.py index 196fcd67..730a9198 100644 --- a/electrum/address_synchronizer.py +++ b/electrum/address_synchronizer.py @@ -27,10 +27,11 @@ from collections import defaultdict from . import bitcoin from .bitcoin import COINBASE_MATURITY, TYPE_ADDRESS, TYPE_PUBKEY -from .util import PrintError, profiler +from .util import PrintError, profiler, bfh from .transaction import Transaction from .synchronizer import Synchronizer from .verifier import SPV +from .i18n import _ TX_HEIGHT_LOCAL = -2 TX_HEIGHT_UNCONF_PARENT = -1 From 01193be241fa1cafa263658750ed6b999e4701dc Mon Sep 17 00:00:00 2001 From: SomberNight Date: Thu, 19 Jul 2018 13:55:05 +0200 Subject: [PATCH 20/56] logging: when not giving args to -v, log everything, as before --- electrum/commands.py | 2 +- electrum/util.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/electrum/commands.py b/electrum/commands.py index e9276d3b..62d2ed6f 100644 --- a/electrum/commands.py +++ b/electrum/commands.py @@ -831,7 +831,7 @@ def add_global_options(parser): group = parser.add_argument_group('global options') # const is for when no argument is given to verbosity # default is for when the flag is missing - group.add_argument("-v", "--verbosity", dest="verbosity", help="Set verbosity filter", default='', const='', nargs='?') + group.add_argument("-v", "--verbosity", dest="verbosity", help="Set verbosity filter", default='', const='*', nargs='?') group.add_argument("-D", "--dir", dest="electrum_path", help="electrum directory") group.add_argument("-P", "--portable", action="store_true", dest="portable", default=False, help="Use local 'electrum_data' directory") group.add_argument("-w", "--wallet", dest="wallet_path", help="wallet path") diff --git a/electrum/util.py b/electrum/util.py index aced2746..485b3458 100644 --- a/electrum/util.py +++ b/electrum/util.py @@ -170,7 +170,7 @@ class PrintError(object): return self.__class__.__name__ def print_error(self, *msg): - if self.verbosity_filter in verbosity: + if self.verbosity_filter in verbosity or verbosity == '*': print_stderr("[%s]" % self.diagnostic_name(), *msg) def print_stderr(self, *msg): @@ -266,7 +266,7 @@ class DaemonThread(threading.Thread, PrintError): self.print_error("stopped") -verbosity = '' +verbosity = '*' def set_verbosity(b): global verbosity verbosity = b From 0100af9389966d9d3ad8c372e816c7cd9905c6fd Mon Sep 17 00:00:00 2001 From: SomberNight Date: Thu, 19 Jul 2018 13:59:38 +0200 Subject: [PATCH 21/56] fix #4572 --- electrum/network.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/electrum/network.py b/electrum/network.py index c90ae823..5b8a2c86 100644 --- a/electrum/network.py +++ b/electrum/network.py @@ -876,13 +876,17 @@ class Network(util.DaemonThread): if not connect: self.connection_down(interface.server) return - # If not finished, get the next chunk - if index >= len(blockchain.checkpoints) and blockchain.height() < interface.tip: - self.request_chunk(interface, index+1) + if index >= len(blockchain.checkpoints): + # If not finished, get the next chunk + if blockchain.height() < interface.tip: + self.request_chunk(interface, index+1) + else: + interface.mode = 'default' + interface.print_error('catch up done', blockchain.height()) + blockchain.catch_up = None else: - interface.mode = 'default' - interface.print_error('catch up done', blockchain.height()) - blockchain.catch_up = None + # the verifier must have asked for this chunk + pass self.notify('updated') def on_get_header(self, interface, response): From cb6bde49b473a2f7805627749630880d83331d5b Mon Sep 17 00:00:00 2001 From: SomberNight Date: Thu, 19 Jul 2018 14:36:30 +0200 Subject: [PATCH 22/56] fix some wine build failures on branches/forks --- contrib/build-wine/build-electrum-git.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/build-wine/build-electrum-git.sh b/contrib/build-wine/build-electrum-git.sh index b3a78fd2..f98a77a1 100755 --- a/contrib/build-wine/build-electrum-git.sh +++ b/contrib/build-wine/build-electrum-git.sh @@ -25,7 +25,7 @@ pushd $WINEPREFIX/drive_c/electrum git submodule init git submodule update -VERSION=`git describe --tags --dirty` +VERSION=`git describe --tags --dirty || printf 'custom'` echo "Last commit: $VERSION" pushd ./contrib/deterministic-build/electrum-locale From 801d3113ab44789c70a896b1865a133c0af3675f Mon Sep 17 00:00:00 2001 From: SomberNight Date: Thu, 19 Jul 2018 14:53:04 +0200 Subject: [PATCH 23/56] wine build: remove pgp.mit.edu from keyservers sometimes slow, and does not return all the pubkeys asked for (so build fails) --- contrib/build-wine/prepare-wine.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/contrib/build-wine/prepare-wine.sh b/contrib/build-wine/prepare-wine.sh index 8df8c135..ffa31e62 100755 --- a/contrib/build-wine/prepare-wine.sh +++ b/contrib/build-wine/prepare-wine.sh @@ -96,8 +96,7 @@ KEYRING_PYTHON_DEV="keyring-electrum-build-python-dev.gpg" for server in $(shuf -e ha.pool.sks-keyservers.net \ hkp://p80.pool.sks-keyservers.net:80 \ keyserver.ubuntu.com \ - hkp://keyserver.ubuntu.com:80 \ - pgp.mit.edu) ; do + hkp://keyserver.ubuntu.com:80) ; do retry gpg --no-default-keyring --keyring $KEYRING_PYTHON_DEV --keyserver "$server" --recv-keys $KEYLIST_PYTHON_DEV \ && break || : ; done From 597295e35931f001dce80cc30937911d6b2229b0 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Thu, 19 Jul 2018 18:16:23 +0200 Subject: [PATCH 24/56] address_synchronizer fixes is_mine: wallet expects get_address_index to work imported wallets: history did not include addr keys after creation deterministic wallets: get_addresses() should be sorted in derivation order --- electrum/address_synchronizer.py | 4 ++++ electrum/wallet.py | 37 ++++++++++++++++++++++++-------- 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/electrum/address_synchronizer.py b/electrum/address_synchronizer.py index 730a9198..58785040 100644 --- a/electrum/address_synchronizer.py +++ b/electrum/address_synchronizer.py @@ -67,6 +67,10 @@ class AddressSynchronizer(PrintError): self.unverified_tx = defaultdict(int) # true when synchronized self.up_to_date = False + + self.load_and_cleanup() + + def load_and_cleanup(self): self.load_transactions() self.load_local_history() self.check_history() diff --git a/electrum/wallet.py b/electrum/wallet.py index 93e230f8..2cb48edf 100644 --- a/electrum/wallet.py +++ b/electrum/wallet.py @@ -169,6 +169,7 @@ class Abstract_Wallet(AddressSynchronizer): def __init__(self, storage): AddressSynchronizer.__init__(self, storage) + self.electrum_version = ELECTRUM_VERSION # saved fields self.use_change = storage.get('use_change', True) @@ -178,10 +179,6 @@ class Abstract_Wallet(AddressSynchronizer): self.fiat_value = storage.get('fiat_value', {}) self.receive_requests = storage.get('payment_requests', {}) - self.load_keystore() - self.load_addresses() - self.test_addresses_sanity() - # save wallet type the first time if self.storage.get('wallet_type') is None: self.storage.put('wallet_type', self.wallet_type) @@ -192,6 +189,12 @@ class Abstract_Wallet(AddressSynchronizer): self.coin_price_cache = {} + def load_and_cleanup(self): + self.load_keystore() + self.load_addresses() + self.test_addresses_sanity() + super().load_and_cleanup() + def diagnostic_name(self): return self.basename() @@ -268,6 +271,15 @@ class Abstract_Wallet(AddressSynchronizer): except: return + def is_mine(self, address): + if not super().is_mine(address): + return False + try: + self.get_address_index(address) + except KeyError: + return False + return True + def is_change(self, address): if not self.is_mine(address): return False @@ -1218,9 +1230,9 @@ class Imported_Wallet(Simple_Wallet): if address in self.addresses: return '' self.addresses[address] = {} - self.storage.put('addresses', self.addresses) - self.storage.write() self.add_address(address) + self.save_addresses() + self.save_transactions(write=True) return address def delete_address(self, address): @@ -1268,7 +1280,7 @@ class Imported_Wallet(Simple_Wallet): else: self.keystore.delete_imported_key(pubkey) self.save_keystore() - self.storage.put('addresses', self.addresses) + self.save_addresses() self.storage.write() @@ -1296,9 +1308,9 @@ class Imported_Wallet(Simple_Wallet): raise NotImplementedError(txin_type) self.addresses[addr] = {'type':txin_type, 'pubkey':pubkey, 'redeem_script':redeem_script} self.save_keystore() - self.save_addresses() - self.storage.write() self.add_address(addr) + self.save_addresses() + self.save_transactions(write=True) return addr def get_redeem_script(self, address): @@ -1337,6 +1349,13 @@ class Deterministic_Wallet(Abstract_Wallet): def has_seed(self): return self.keystore.has_seed() + def get_addresses(self): + # overloaded so that addresses are ordered based on derivation + out = [] + out += self.get_receiving_addresses() + out += self.get_change_addresses() + return out + def get_receiving_addresses(self): return self.receiving_addresses From f7dce426cb81ac999e6f0ca02ef50b8122b4d71f Mon Sep 17 00:00:00 2001 From: SomberNight Date: Thu, 19 Jul 2018 19:52:06 +0200 Subject: [PATCH 25/56] fix #4574 --- electrum/gui/qt/paytoedit.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/electrum/gui/qt/paytoedit.py b/electrum/gui/qt/paytoedit.py index 716a5f58..e33b90e4 100644 --- a/electrum/gui/qt/paytoedit.py +++ b/electrum/gui/qt/paytoedit.py @@ -34,8 +34,7 @@ from .qrtextedit import ScanQRTextEdit from .completion_text_edit import CompletionTextEdit from . import util -RE_ADDRESS = '[1-9A-HJ-NP-Za-km-z]{26,}' -RE_ALIAS = '(.*?)\s*\<([1-9A-HJ-NP-Za-km-z]{26,})\>' +RE_ALIAS = '(.*?)\s*\<([0-9A-Za-z]{1,})\>' frozen_style = "QWidget { background-color:none; border:none;}" normal_style = "QPlainTextEdit { }" From 281805a0a4bb7e01a8ec623be9ea23d7505bb9a0 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Fri, 20 Jul 2018 16:38:18 +0200 Subject: [PATCH 26/56] linux sdist: 'typing' was not included, which is needed on py3.4 not making typing conditioned on py version as then freeze_packages would not pick it up. --- contrib/deterministic-build/requirements.txt | 6 ++++-- contrib/requirements/requirements.txt | 1 + setup.py | 1 - 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/contrib/deterministic-build/requirements.txt b/contrib/deterministic-build/requirements.txt index 67d1fafd..1b226f9d 100644 --- a/contrib/deterministic-build/requirements.txt +++ b/contrib/deterministic-build/requirements.txt @@ -16,8 +16,6 @@ idna==2.7 \ jsonrpclib-pelix==0.3.1 \ --hash=sha256:5417b1508d5a50ec64f6e5b88907f111155d52607b218ff3ba9a777afb2e49e3 \ --hash=sha256:bd89a6093bc4d47dc8a096197aacb827359944a4533be5193f3845f57b9f91b4 -pbkdf2==1.3 \ - --hash=sha256:ac6397369f128212c43064a2b4878038dab78dab41875364554aaf2a684e6979 pip==10.0.1 \ --hash=sha256:717cdffb2833be8409433a93746744b59505f42146e8d37de6c62b430e25d6d7 \ --hash=sha256:f2bd08e0cd1b06e10218feaf6fef299f473ba706582eb3bd9d52203fdbd7ee68 @@ -56,6 +54,10 @@ setuptools==40.0.0 \ six==1.11.0 \ --hash=sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9 \ --hash=sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb +typing==3.6.4 \ + --hash=sha256:3a887b021a77b292e151afb75323dea88a7bc1b3dfa92176cff8e44c8b68bddf \ + --hash=sha256:b2c689d54e1144bbcfd191b0832980a21c2dbcf7b5ff7a66248a60c90e951eb8 \ + --hash=sha256:d400a9344254803a2368533e4533a4200d21eb7b6b729c173bc38201a74db3f2 urllib3==1.23 \ --hash=sha256:a68ac5e15e76e7e5dd2b8f94007233e01effe3e50e8daddf69acfd81cb686baf \ --hash=sha256:b5725a0bd4ba422ab0e66e89e030c806576753ea3ee08554382c14e685d117b5 diff --git a/contrib/requirements/requirements.txt b/contrib/requirements/requirements.txt index 4fd62530..99b859c3 100644 --- a/contrib/requirements/requirements.txt +++ b/contrib/requirements/requirements.txt @@ -7,3 +7,4 @@ dnspython jsonrpclib-pelix PySocks>=1.6.6 qdarkstyle<3.0 +typing>=3.0.0 diff --git a/setup.py b/setup.py index 937375fb..8d732862 100755 --- a/setup.py +++ b/setup.py @@ -43,7 +43,6 @@ if platform.system() in ['Linux', 'FreeBSD', 'DragonFly']: extras_require = { 'hardware': requirements_hw, 'fast': ['pycryptodomex'], - ':python_version < "3.5"': ['typing>=3.0.0'], } extras_require['full'] = extras_require['hardware'] + extras_require['fast'] From f8e13c5c330c892698356ec35858785ce8dc012f Mon Sep 17 00:00:00 2001 From: SomberNight Date: Sat, 21 Jul 2018 16:15:45 +0200 Subject: [PATCH 27/56] kivy: use correct i18n --- electrum/gui/kivy/main_window.py | 2 +- electrum/gui/kivy/uix/dialogs/crash_reporter.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/electrum/gui/kivy/main_window.py b/electrum/gui/kivy/main_window.py index 5414543f..052bd5f0 100644 --- a/electrum/gui/kivy/main_window.py +++ b/electrum/gui/kivy/main_window.py @@ -10,12 +10,12 @@ import threading from electrum.bitcoin import TYPE_ADDRESS from electrum.storage import WalletStorage from electrum.wallet import Wallet -from electrum.i18n import _ from electrum.paymentrequest import InvoiceStore from electrum.util import profiler, InvalidPassword from electrum.plugin import run_hook from electrum.util import format_satoshis, format_satoshis_plain from electrum.paymentrequest import PR_UNPAID, PR_PAID, PR_UNKNOWN, PR_EXPIRED +from .i18n import _ from kivy.app import App from kivy.core.window import Window diff --git a/electrum/gui/kivy/uix/dialogs/crash_reporter.py b/electrum/gui/kivy/uix/dialogs/crash_reporter.py index 8603d806..04582b95 100644 --- a/electrum/gui/kivy/uix/dialogs/crash_reporter.py +++ b/electrum/gui/kivy/uix/dialogs/crash_reporter.py @@ -9,9 +9,9 @@ from kivy.lang import Builder from kivy.uix.label import Label from kivy.utils import platform +from electrum.gui.kivy.i18n import _ from electrum.base_crash_reporter import BaseCrashReporter -from electrum.i18n import _ Builder.load_string(''' From 4284f4feb3e35e2d5c9b1d274081d4da126bab92 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Sat, 21 Jul 2018 23:09:46 +0200 Subject: [PATCH 28/56] fix #4575 --- electrum/gui/kivy/main_window.py | 11 +++++++++-- electrum/gui/kivy/uix/dialogs/qr_dialog.py | 5 +++-- electrum/gui/kivy/uix/dialogs/tx_dialog.py | 5 +++-- electrum/gui/kivy/uix/qrcodewidget.py | 13 +++++++++++-- electrum/gui/qt/transaction_dialog.py | 8 +++++++- 5 files changed, 33 insertions(+), 9 deletions(-) diff --git a/electrum/gui/kivy/main_window.py b/electrum/gui/kivy/main_window.py index 052bd5f0..dfb0f8ff 100644 --- a/electrum/gui/kivy/main_window.py +++ b/electrum/gui/kivy/main_window.py @@ -391,9 +391,16 @@ class ElectrumWindow(App): popup.export = self.export_private_keys popup.open() - def qr_dialog(self, title, data, show_text=False): + def qr_dialog(self, title, data, show_text=False, text_for_clipboard=None): from .uix.dialogs.qr_dialog import QRDialog - popup = QRDialog(title, data, show_text) + def on_qr_failure(): + popup.dismiss() + msg = _('Failed to display QR code.') + if text_for_clipboard: + msg += '\n' + _('Text copied to clipboard.') + self._clipboard.copy(text_for_clipboard) + Clock.schedule_once(lambda dt: self.show_info(msg)) + popup = QRDialog(title, data, show_text, on_qr_failure) popup.open() def scan_qr(self, on_complete): diff --git a/electrum/gui/kivy/uix/dialogs/qr_dialog.py b/electrum/gui/kivy/uix/dialogs/qr_dialog.py index abc5a6de..b12eb6ce 100644 --- a/electrum/gui/kivy/uix/dialogs/qr_dialog.py +++ b/electrum/gui/kivy/uix/dialogs/qr_dialog.py @@ -36,11 +36,12 @@ Builder.load_string(''' ''') class QRDialog(Factory.Popup): - def __init__(self, title, data, show_text): + def __init__(self, title, data, show_text, failure_cb=None): Factory.Popup.__init__(self) self.title = title self.data = data self.show_text = show_text + self.failure_cb = failure_cb def on_open(self): - self.ids.qr.set_data(self.data) + self.ids.qr.set_data(self.data, self.failure_cb) diff --git a/electrum/gui/kivy/uix/dialogs/tx_dialog.py b/electrum/gui/kivy/uix/dialogs/tx_dialog.py index 5a179fe0..a99acb94 100644 --- a/electrum/gui/kivy/uix/dialogs/tx_dialog.py +++ b/electrum/gui/kivy/uix/dialogs/tx_dialog.py @@ -179,6 +179,7 @@ class TxDialog(Factory.Popup): def show_qr(self): from electrum.bitcoin import base_encode, bfh - text = bfh(str(self.tx)) + raw_tx = str(self.tx) + text = bfh(raw_tx) text = base_encode(text, base=43) - self.app.qr_dialog(_("Raw Transaction"), text) + self.app.qr_dialog(_("Raw Transaction"), text, text_for_clipboard=raw_tx) diff --git a/electrum/gui/kivy/uix/qrcodewidget.py b/electrum/gui/kivy/uix/qrcodewidget.py index 1989c5a0..56e20cdd 100644 --- a/electrum/gui/kivy/uix/qrcodewidget.py +++ b/electrum/gui/kivy/uix/qrcodewidget.py @@ -5,6 +5,7 @@ from threading import Thread from functools import partial import qrcode +from qrcode import exceptions from kivy.uix.floatlayout import FloatLayout from kivy.graphics.texture import Texture @@ -50,15 +51,23 @@ class QRCodeWidget(FloatLayout): self.data = None self.qr = None self._qrtexture = None + self.failure_cb = None def on_data(self, instance, value): if not (self.canvas or value): return - self.update_qr() + try: + self.update_qr() + except qrcode.exceptions.DataOverflowError: + if self.failure_cb: + self.failure_cb() + else: + raise - def set_data(self, data): + def set_data(self, data, failure_cb=None): if self.data == data: return + self.failure_cb = failure_cb MinSize = 210 if len(data) < 128 else 500 self.setMinimumSize((MinSize, MinSize)) self.data = data diff --git a/electrum/gui/qt/transaction_dialog.py b/electrum/gui/qt/transaction_dialog.py index 66d47f6f..4e8e6e67 100644 --- a/electrum/gui/qt/transaction_dialog.py +++ b/electrum/gui/qt/transaction_dialog.py @@ -31,6 +31,9 @@ from PyQt5.QtCore import * from PyQt5.QtGui import * from PyQt5.QtWidgets import * +import qrcode +from qrcode import exceptions + from electrum.bitcoin import base_encode from electrum.i18n import _ from electrum.plugin import run_hook @@ -183,8 +186,11 @@ class TxDialog(QDialog, MessageBoxMixin): text = base_encode(text, base=43) try: self.main_window.show_qrcode(text, 'Transaction', parent=self) + except qrcode.exceptions.DataOverflowError: + self.show_error(_('Failed to display QR code.') + '\n' + + _('Transaction is too large in size.')) except Exception as e: - self.show_message(str(e)) + self.show_error(_('Failed to display QR code.') + '\n' + str(e)) def sign(self): def sign_done(success): From a830747f8348ec6be02e221d7fcb387d6e6392cd Mon Sep 17 00:00:00 2001 From: SomberNight Date: Sat, 21 Jul 2018 23:23:25 +0200 Subject: [PATCH 29/56] kivy: update history screen on fee histogram related: #4573 --- electrum/gui/kivy/main_window.py | 1 + 1 file changed, 1 insertion(+) diff --git a/electrum/gui/kivy/main_window.py b/electrum/gui/kivy/main_window.py index dfb0f8ff..2f68cf00 100644 --- a/electrum/gui/kivy/main_window.py +++ b/electrum/gui/kivy/main_window.py @@ -483,6 +483,7 @@ class ElectrumWindow(App): interests = ['updated', 'status', 'new_transaction', 'verified', 'interfaces'] self.network.register_callback(self.on_network_event, interests) self.network.register_callback(self.on_fee, ['fee']) + self.network.register_callback(self.on_history, ['fee_histogram']) self.network.register_callback(self.on_quotes, ['on_quotes']) self.network.register_callback(self.on_history, ['on_history']) # load wallet From 89aa9eb0a7ecdde51ae9e4989d06e14b0122e514 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Sun, 22 Jul 2018 19:40:10 +0200 Subject: [PATCH 30/56] revealer: minor fix and clean-up --- electrum/plugins/revealer/qt.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/electrum/plugins/revealer/qt.py b/electrum/plugins/revealer/qt.py index 69c9fcd3..198d1cc4 100644 --- a/electrum/plugins/revealer/qt.py +++ b/electrum/plugins/revealer/qt.py @@ -39,9 +39,9 @@ class Plugin(BasePlugin): BasePlugin.__init__(self, parent, config, name) self.base_dir = config.electrum_path()+'/revealer/' - if self.config.get('calibration_h') == None: + if self.config.get('calibration_h') is None: self.config.set_key('calibration_h', 0) - if self.config.get('calibration_v') == None: + if self.config.get('calibration_v') is None: self.config.set_key('calibration_v', 0) self.calibration_h = self.config.get('calibration_h') @@ -268,7 +268,7 @@ class Plugin(BasePlugin): max_letters = 17 max_lines = 6 max_words = 3 - if len(txt) > 102: + else: fontsize = 9 linespace = 10 max_letters = 24 @@ -596,8 +596,8 @@ class Plugin(BasePlugin): qr_qt = self.paintQR(self.hex_noise.upper() +self.code_id) target = QRectF(base_img.width()-65-qr_size, base_img.height()-65-qr_size, - qr_size, qr_size ); - painter.drawImage(target, qr_qt); + qr_size, qr_size ) + painter.drawImage(target, qr_qt) painter.setPen(QPen(Qt.black, 4)) painter.drawLine(base_img.width()-65-qr_size, base_img.height()-65-qr_size, From d2abaf54e80621d2a4076c7dd996e08038e9eade Mon Sep 17 00:00:00 2001 From: SomberNight Date: Mon, 23 Jul 2018 19:55:47 +0200 Subject: [PATCH 31/56] verifier: small refactor --- electrum/verifier.py | 64 +++++++++++++++++++++++++++----------------- 1 file changed, 39 insertions(+), 25 deletions(-) diff --git a/electrum/verifier.py b/electrum/verifier.py index 0a78ad05..82ad678b 100644 --- a/electrum/verifier.py +++ b/electrum/verifier.py @@ -20,12 +20,18 @@ # ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. + +from typing import Sequence, Optional + from .util import ThreadJob, bh2u from .bitcoin import Hash, hash_decode, hash_encode from .transaction import Transaction -class InnerNodeOfSpvProofIsValidTx(Exception): pass +class MerkleVerificationFailure(Exception): pass +class MissingBlockHeader(MerkleVerificationFailure): pass +class MerkleRootMismatch(MerkleVerificationFailure): pass +class InnerNodeOfSpvProofIsValidTx(MerkleVerificationFailure): pass class SPV(ThreadJob): @@ -85,28 +91,17 @@ class SPV(ThreadJob): tx_hash = params[0] tx_height = merkle.get('block_height') pos = merkle.get('pos') - try: - merkle_root = self.hash_merkle_root(merkle['merkle'], tx_hash, pos) - except InnerNodeOfSpvProofIsValidTx: - self.print_error("merkle verification failed for {} (inner node looks like tx)" - .format(tx_hash)) - return + merkle_branch = merkle.get('merkle') header = self.network.blockchain().read_header(tx_height) - # FIXME: if verification fails below, - # we should make a fresh connection to a server to - # recover from this, as this TX will now never verify - if not header: - self.print_error( - "merkle verification failed for {} (missing header {})" - .format(tx_hash, tx_height)) - return - if header.get('merkle_root') != merkle_root: - self.print_error( - "merkle verification failed for {} (merkle root mismatch {} != {})" - .format(tx_hash, header.get('merkle_root'), merkle_root)) + try: + verify_tx_is_in_block(tx_hash, merkle_branch, pos, header, tx_height) + except MerkleVerificationFailure as e: + self.print_error(str(e)) + # FIXME: we should make a fresh connection to a server + # to recover from this, as this TX will now never verify return # we passed all the tests - self.merkle_roots[tx_hash] = merkle_root + self.merkle_roots[tx_hash] = header.get('merkle_root') try: # note: we could pop in the beginning, but then we would request # this proof again in case of verification failure from the same server @@ -118,11 +113,17 @@ class SPV(ThreadJob): self.wallet.save_verified_tx(write=True) @classmethod - def hash_merkle_root(cls, merkle_s, target_hash, pos): - h = hash_decode(target_hash) - for i in range(len(merkle_s)): - item = merkle_s[i] - h = Hash(hash_decode(item) + h) if ((pos >> i) & 1) else Hash(h + hash_decode(item)) + def hash_merkle_root(cls, merkle_branch: Sequence[str], tx_hash: str, leaf_pos_in_tree: int): + """Return calculated merkle root.""" + try: + h = hash_decode(tx_hash) + merkle_branch_bytes = [hash_decode(item) for item in merkle_branch] + int(leaf_pos_in_tree) # raise if invalid + except Exception as e: + raise MerkleVerificationFailure(e) + + for i, item in enumerate(merkle_branch_bytes): + h = Hash(item + h) if ((leaf_pos_in_tree >> i) & 1) else Hash(h + item) cls._raise_if_valid_tx(bh2u(h)) return hash_encode(h) @@ -156,3 +157,16 @@ class SPV(ThreadJob): def is_up_to_date(self): return not self.requested_merkle + + +def verify_tx_is_in_block(tx_hash: str, merkle_branch: Sequence[str], + leaf_pos_in_tree: int, block_header: Optional[dict], + block_height: int) -> None: + """Raise MerkleVerificationFailure if verification fails.""" + if not block_header: + raise MissingBlockHeader("merkle verification failed for {} (missing header {})" + .format(tx_hash, block_height)) + calc_merkle_root = SPV.hash_merkle_root(merkle_branch, tx_hash, leaf_pos_in_tree) + if block_header.get('merkle_root') != calc_merkle_root: + raise MerkleRootMismatch("merkle verification failed for {} ({} != {})".format( + tx_hash, block_header.get('merkle_root'), calc_merkle_root)) From 53130da6828de132f5611d8e137cf94b8eda5206 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Mon, 23 Jul 2018 19:57:41 +0200 Subject: [PATCH 32/56] storage: factor out 'JsonDB' --- electrum/storage.py | 157 ++++++++++++++++++++++++-------------------- 1 file changed, 86 insertions(+), 71 deletions(-) diff --git a/electrum/storage.py b/electrum/storage.py index 0524cd23..b951e4b7 100644 --- a/electrum/storage.py +++ b/electrum/storage.py @@ -67,15 +67,84 @@ def get_derivation_used_for_hw_device_encryption(): # storage encryption version STO_EV_PLAINTEXT, STO_EV_USER_PW, STO_EV_XPUB_PW = range(0, 3) -class WalletStorage(PrintError): - def __init__(self, path, manual_upgrades=False): - self.print_error("wallet path", path) - self.manual_upgrades = manual_upgrades - self.lock = threading.RLock() +class JsonDB(PrintError): + + def __init__(self, path): + self.db_lock = threading.RLock() self.data = {} self.path = path self.modified = False + + def get(self, key, default=None): + with self.db_lock: + v = self.data.get(key) + if v is None: + v = default + else: + v = copy.deepcopy(v) + return v + + def put(self, key, value): + try: + json.dumps(key, cls=util.MyEncoder) + json.dumps(value, cls=util.MyEncoder) + except: + self.print_error("json error: cannot save", key) + return + with self.db_lock: + if value is not None: + if self.data.get(key) != value: + self.modified = True + self.data[key] = copy.deepcopy(value) + elif key in self.data: + self.modified = True + self.data.pop(key) + + @profiler + def write(self): + with self.db_lock: + self._write() + + def _write(self): + if threading.currentThread().isDaemon(): + self.print_error('warning: daemon thread cannot write db') + return + if not self.modified: + return + s = json.dumps(self.data, indent=4, sort_keys=True, cls=util.MyEncoder) + s = self.encrypt_before_writing(s) + + temp_path = "%s.tmp.%s" % (self.path, os.getpid()) + with open(temp_path, "w", encoding='utf-8') as f: + f.write(s) + f.flush() + os.fsync(f.fileno()) + + mode = os.stat(self.path).st_mode if os.path.exists(self.path) else stat.S_IREAD | stat.S_IWRITE + # perform atomic write on POSIX systems + try: + os.rename(temp_path, self.path) + except: + os.remove(self.path) + os.rename(temp_path, self.path) + os.chmod(self.path, mode) + self.print_error("saved", self.path) + self.modified = False + + def encrypt_before_writing(self, plaintext: str) -> str: + return plaintext + + def file_exists(self): + return self.path and os.path.exists(self.path) + + +class WalletStorage(JsonDB): + + def __init__(self, path, manual_upgrades=False): + self.print_error("wallet path", path) + JsonDB.__init__(self, path) + self.manual_upgrades = manual_upgrades self.pubkey = None if self.file_exists(): with open(self.path, "r", encoding='utf-8') as f: @@ -160,9 +229,6 @@ class WalletStorage(PrintError): except: return STO_EV_PLAINTEXT - def file_exists(self): - return self.path and os.path.exists(self.path) - @staticmethod def get_eckey_from_password(password): secret = hashlib.pbkdf2_hmac('sha512', password.encode('utf-8'), b'', iterations=1024) @@ -189,6 +255,17 @@ class WalletStorage(PrintError): s = s.decode('utf8') self.load_data(s) + def encrypt_before_writing(self, plaintext: str) -> str: + s = plaintext + if self.pubkey: + s = bytes(s, 'utf8') + c = zlib.compress(s) + enc_magic = self._get_encryption_magic() + public_key = ecc.ECPubkey(bfh(self.pubkey)) + s = public_key.encrypt_message(c, enc_magic) + s = s.decode('utf8') + return s + def check_password(self, password): """Raises an InvalidPassword exception on invalid password""" if not self.is_encrypted(): @@ -211,71 +288,9 @@ class WalletStorage(PrintError): self.pubkey = None self._encryption_version = STO_EV_PLAINTEXT # make sure next storage.write() saves changes - with self.lock: + with self.db_lock: self.modified = True - def get(self, key, default=None): - with self.lock: - v = self.data.get(key) - if v is None: - v = default - else: - v = copy.deepcopy(v) - return v - - def put(self, key, value): - try: - json.dumps(key, cls=util.MyEncoder) - json.dumps(value, cls=util.MyEncoder) - except: - self.print_error("json error: cannot save", key) - return - with self.lock: - if value is not None: - if self.data.get(key) != value: - self.modified = True - self.data[key] = copy.deepcopy(value) - elif key in self.data: - self.modified = True - self.data.pop(key) - - @profiler - def write(self): - with self.lock: - self._write() - - def _write(self): - if threading.currentThread().isDaemon(): - self.print_error('warning: daemon thread cannot write wallet') - return - if not self.modified: - return - s = json.dumps(self.data, indent=4, sort_keys=True, cls=util.MyEncoder) - if self.pubkey: - s = bytes(s, 'utf8') - c = zlib.compress(s) - enc_magic = self._get_encryption_magic() - public_key = ecc.ECPubkey(bfh(self.pubkey)) - s = public_key.encrypt_message(c, enc_magic) - s = s.decode('utf8') - - temp_path = "%s.tmp.%s" % (self.path, os.getpid()) - with open(temp_path, "w", encoding='utf-8') as f: - f.write(s) - f.flush() - os.fsync(f.fileno()) - - mode = os.stat(self.path).st_mode if os.path.exists(self.path) else stat.S_IREAD | stat.S_IWRITE - # perform atomic write on POSIX systems - try: - os.rename(temp_path, self.path) - except: - os.remove(self.path) - os.rename(temp_path, self.path) - os.chmod(self.path, mode) - self.print_error("saved", self.path) - self.modified = False - def requires_split(self): d = self.get('accounts', {}) return len(d) > 1 From 61aa19539cb2e0843bbacc91c523ef7833035785 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Tue, 24 Jul 2018 16:13:49 +0200 Subject: [PATCH 33/56] some packaging clean-up --- MANIFEST.in | 13 +++++++------ setup.py | 21 ++++----------------- 2 files changed, 11 insertions(+), 23 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index b028b14c..d09f4f5d 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,15 +3,16 @@ include README.rst include electrum.conf.sample include electrum.desktop include *.py -include electrum +include run_electrum include contrib/requirements/requirements.txt include contrib/requirements/requirements-hw.txt -recursive-include lib *.py -recursive-include gui *.py -recursive-include plugins *.py recursive-include packages *.py recursive-include packages cacert.pem include icons.qrc -recursive-include icons * -recursive-include scripts * +graft icons +graft electrum +prune electrum/tests + +global-exclude __pycache__ +global-exclude *.py[co] diff --git a/setup.py b/setup.py index 8d732862..18c5cb88 100755 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ # python setup.py sdist --format=zip,gztar -from setuptools import setup +from setuptools import setup, find_packages import os import sys import platform @@ -57,28 +57,15 @@ setup( 'electrum.gui', 'electrum.gui.qt', 'electrum.plugins', - 'electrum.plugins.audio_modem', - 'electrum.plugins.cosigner_pool', - 'electrum.plugins.email_requests', - 'electrum.plugins.greenaddress_instant', - 'electrum.plugins.hw_wallet', - 'electrum.plugins.keepkey', - 'electrum.plugins.labels', - 'electrum.plugins.ledger', - 'electrum.plugins.revealer', - 'electrum.plugins.trezor', - 'electrum.plugins.digitalbitbox', - 'electrum.plugins.trustedcoin', - 'electrum.plugins.virtualkeyboard', - ], + ] + [('electrum.plugins.'+pkg) for pkg in find_packages('electrum/plugins')], package_dir={ 'electrum': 'electrum' }, package_data={ '': ['*.txt', '*.json', '*.ttf', '*.otf'], 'electrum': [ - 'electrum/wordlist/*.txt', - 'electrum/locale/*/LC_MESSAGES/electrum.mo', + 'wordlist/*.txt', + 'locale/*/LC_MESSAGES/electrum.mo', ], }, scripts=['electrum/electrum'], From 579d48cf0cca3038961814f83dfc9028a410dd69 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Tue, 24 Jul 2018 18:25:22 +0200 Subject: [PATCH 34/56] follow-up a830747f8348ec6be02e221d7fcb387d6e6392cd on_history expects fewer arguments than what the fee_histogram callback gives --- electrum/gui/kivy/main_window.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/electrum/gui/kivy/main_window.py b/electrum/gui/kivy/main_window.py index 2f68cf00..b55ee129 100644 --- a/electrum/gui/kivy/main_window.py +++ b/electrum/gui/kivy/main_window.py @@ -159,6 +159,9 @@ class ElectrumWindow(App): Logger.info("on_history") self._trigger_update_history() + def on_fee_histogram(self, *args): + self._trigger_update_history() + def _get_bu(self): decimal_point = self.electrum_config.get('decimal_point', 5) return decimal_point_to_base_unit_name(decimal_point) @@ -483,7 +486,7 @@ class ElectrumWindow(App): interests = ['updated', 'status', 'new_transaction', 'verified', 'interfaces'] self.network.register_callback(self.on_network_event, interests) self.network.register_callback(self.on_fee, ['fee']) - self.network.register_callback(self.on_history, ['fee_histogram']) + self.network.register_callback(self.on_fee_histogram, ['fee_histogram']) self.network.register_callback(self.on_quotes, ['on_quotes']) self.network.register_callback(self.on_history, ['on_history']) # load wallet From a799a00dc52d475c1a849eda4522a752a50f2d61 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Tue, 24 Jul 2018 18:57:49 +0200 Subject: [PATCH 35/56] fix #4577 --- electrum/base_wizard.py | 18 ++++++++++++++---- electrum/gui/kivy/uix/dialogs/installwizard.py | 7 ++++++- electrum/wallet.py | 2 +- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/electrum/base_wizard.py b/electrum/base_wizard.py index d5a0f83a..8e130a7a 100644 --- a/electrum/base_wizard.py +++ b/electrum/base_wizard.py @@ -34,7 +34,7 @@ from .keystore import bip44_derivation, purpose48_derivation from .wallet import Imported_Wallet, Standard_Wallet, Multisig_Wallet, wallet_types, Wallet from .storage import STO_EV_USER_PW, STO_EV_XPUB_PW, get_derivation_used_for_hw_device_encryption from .i18n import _ -from .util import UserCancelled, InvalidPassword +from .util import UserCancelled, InvalidPassword, WalletFileException # hardware device setup purpose HWD_SETUP_NEW_WALLET, HWD_SETUP_DECRYPT_WALLET = range(0, 2) @@ -106,10 +106,20 @@ class BaseWizard(object): self.choice_dialog(title=title, message=message, choices=choices, run_next=self.on_wallet_type) def upgrade_storage(self): + exc = None def on_finished(): - self.wallet = Wallet(self.storage) - self.terminate() - self.waiting_dialog(partial(self.storage.upgrade), _('Upgrading wallet format...'), on_finished=on_finished) + if exc is None: + self.wallet = Wallet(self.storage) + self.terminate() + else: + raise exc + def do_upgrade(): + nonlocal exc + try: + self.storage.upgrade() + except Exception as e: + exc = e + self.waiting_dialog(do_upgrade, _('Upgrading wallet format...'), on_finished=on_finished) def load_2fa(self): self.storage.put('wallet_type', '2fa') diff --git a/electrum/gui/kivy/uix/dialogs/installwizard.py b/electrum/gui/kivy/uix/dialogs/installwizard.py index d5b51e07..04af3734 100644 --- a/electrum/gui/kivy/uix/dialogs/installwizard.py +++ b/electrum/gui/kivy/uix/dialogs/installwizard.py @@ -957,7 +957,12 @@ class InstallWizard(BaseWizard, Widget): # on completion hide message Clock.schedule_once(lambda dt: app.info_bubble.hide(now=True), -1) if on_finished: - Clock.schedule_once(lambda dt: on_finished(), -1) + def protected_on_finished(): + try: + on_finished() + except Exception as e: + self.show_error(str(e)) + Clock.schedule_once(lambda dt: protected_on_finished(), -1) app = App.get_running_app() app.show_info_bubble( diff --git a/electrum/wallet.py b/electrum/wallet.py index 2cb48edf..3f45e9ed 100644 --- a/electrum/wallet.py +++ b/electrum/wallet.py @@ -1661,4 +1661,4 @@ class Wallet(object): return Multisig_Wallet if wallet_type in wallet_constructors: return wallet_constructors[wallet_type] - raise RuntimeError("Unknown wallet type: " + str(wallet_type)) + raise WalletFileException("Unknown wallet type: " + str(wallet_type)) From 6899ca25278701137022d9ffa3ba229951d3eebd Mon Sep 17 00:00:00 2001 From: SomberNight Date: Tue, 24 Jul 2018 20:20:47 +0200 Subject: [PATCH 36/56] docker-wine: update a package version the previous version is no longer available. this suggests that it's difficult to reproduce old builds. not sure about long term solution. --- contrib/build-wine/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/build-wine/docker/Dockerfile b/contrib/build-wine/docker/Dockerfile index 0d2a63b1..f46e9812 100644 --- a/contrib/build-wine/docker/Dockerfile +++ b/contrib/build-wine/docker/Dockerfile @@ -8,7 +8,7 @@ RUN dpkg --add-architecture i386 && \ wget=1.19.4-1ubuntu2.1 \ gnupg2=2.2.4-1ubuntu1.1 \ dirmngr=2.2.4-1ubuntu1.1 \ - software-properties-common=0.96.24.32.3 \ + software-properties-common=0.96.24.32.4 \ && \ wget -nc https://dl.winehq.org/wine-builds/Release.key && \ apt-key add Release.key && \ From 02c30e3d520d130866e8edc4cd13eb229a853f49 Mon Sep 17 00:00:00 2001 From: Jean-Christophe Rona Date: Wed, 25 Jul 2018 20:07:39 +0200 Subject: [PATCH 37/56] Add support for Archos Safe-T mini hardware wallet (#4445) commit 10c46477f3a6f2fbc0596345511e0994253081eb Author: SomberNight Date: Wed Jul 25 19:40:05 2018 +0200 backport changes of trezor plugin commit 213619e880f709188c1ea6272758896748e681a8 Merge: a855b75b6 6899ca252 Author: Jean-Christophe Rona Date: Wed Jul 25 18:45:19 2018 +0200 Merge branch 'master' into safe-t-mini commit a855b75b6f5af5f707c4680d0bac79eb66a85ace Author: Jean-Christophe Rona Date: Wed Jul 25 18:37:12 2018 +0200 Safe-T: Switch to safet 0.1.3 to remove the rlp dependency commit 9bee44ca33289158c91c03d47dec45de6577f17b Author: SomberNight Date: Wed Jul 18 14:01:10 2018 +0200 safe-t: bump min fw to 1.0.5 older fw has a bug when restoring from seed commit 01816607e8ba308cb5cff96b5fb844e4f6b8fcc1 Author: SomberNight Date: Wed Jul 18 13:57:17 2018 +0200 safe-t: fix rlp version to avoid eth stuff commit 430206bea1fa10b762ff953fbc7652ce0d0e939d Merge: a999ae266 b4b862b0c Author: SomberNight Date: Wed Jul 18 13:29:41 2018 +0200 Merge branch 'master' into pr/4445 commit a999ae266f499f180946d53d4e860cc871d562ab Author: Jean-Christophe Rona Date: Tue Jun 19 14:18:03 2018 +0200 Safe-T mini: Remove supported coins This is not really useful there. commit 7922df1031b2c4b132f7f9c90232480b5bf9585c Author: Jean-Christophe Rona Date: Tue May 29 16:43:37 2018 +0200 Safe-T mini: Add support for the Safe-T mini --- contrib/build-osx/osx.spec | 6 +- contrib/build-wine/deterministic.spec | 4 + .../deterministic-build/requirements-hw.txt | 3 + contrib/requirements/requirements-hw.txt | 1 + electrum/plugins/safe_t/__init__.py | 8 + electrum/plugins/safe_t/client.py | 11 + electrum/plugins/safe_t/clientbase.py | 252 +++++++++ electrum/plugins/safe_t/cmdline.py | 14 + electrum/plugins/safe_t/qt.py | 492 +++++++++++++++++ electrum/plugins/safe_t/safe_t.py | 508 ++++++++++++++++++ electrum/plugins/safe_t/transport.py | 95 ++++ electrum/storage.py | 4 +- electrum/transaction.py | 2 +- icons.qrc | 2 + icons/safe-t.png | Bin 0 -> 3871 bytes icons/safe-t_unpaired.png | Bin 0 -> 3719 bytes 16 files changed, 1398 insertions(+), 4 deletions(-) create mode 100644 electrum/plugins/safe_t/__init__.py create mode 100644 electrum/plugins/safe_t/client.py create mode 100644 electrum/plugins/safe_t/clientbase.py create mode 100644 electrum/plugins/safe_t/cmdline.py create mode 100644 electrum/plugins/safe_t/qt.py create mode 100644 electrum/plugins/safe_t/safe_t.py create mode 100644 electrum/plugins/safe_t/transport.py create mode 100644 icons/safe-t.png create mode 100644 icons/safe-t_unpaired.png diff --git a/contrib/build-osx/osx.spec b/contrib/build-osx/osx.spec index 5e2e6d4d..0cd01c66 100644 --- a/contrib/build-osx/osx.spec +++ b/contrib/build-osx/osx.spec @@ -23,6 +23,7 @@ block_cipher = None # see https://github.com/pyinstaller/pyinstaller/issues/2005 hiddenimports = [] hiddenimports += collect_submodules('trezorlib') +hiddenimports += collect_submodules('safetlib') hiddenimports += collect_submodules('btchip') hiddenimports += collect_submodules('keepkeylib') hiddenimports += collect_submodules('websocket') @@ -33,10 +34,11 @@ datas = [ (electrum+'electrum/locale', PYPKG + '/locale') ] datas += collect_data_files('trezorlib') +datas += collect_data_files('safetlib') datas += collect_data_files('btchip') datas += collect_data_files('keepkeylib') -# Add libusb so Trezor will work +# Add libusb so Trezor and Safe-T mini will work binaries = [(electrum + "contrib/build-osx/libusb-1.0.dylib", ".")] binaries += [(electrum + "contrib/build-osx/libsecp256k1.0.dylib", ".")] @@ -57,6 +59,8 @@ a = Analysis([electrum+ MAIN_SCRIPT, electrum+'electrum/plugins/email_requests/qt.py', electrum+'electrum/plugins/trezor/client.py', electrum+'electrum/plugins/trezor/qt.py', + electrum+'electrum/plugins/safe_t/client.py', + electrum+'electrum/plugins/safe_t/qt.py', electrum+'electrum/plugins/keepkey/qt.py', electrum+'electrum/plugins/ledger/qt.py', ], diff --git a/contrib/build-wine/deterministic.spec b/contrib/build-wine/deterministic.spec index 6561c62c..ae3fba80 100644 --- a/contrib/build-wine/deterministic.spec +++ b/contrib/build-wine/deterministic.spec @@ -18,6 +18,7 @@ home = 'C:\\electrum\\' # see https://github.com/pyinstaller/pyinstaller/issues/2005 hiddenimports = [] hiddenimports += collect_submodules('trezorlib') +hiddenimports += collect_submodules('safetlib') hiddenimports += collect_submodules('btchip') hiddenimports += collect_submodules('keepkeylib') hiddenimports += collect_submodules('websocket') @@ -37,6 +38,7 @@ datas = [ ('C:\\Program Files (x86)\\ZBar\\bin\\', '.') ] datas += collect_data_files('trezorlib') +datas += collect_data_files('safetlib') datas += collect_data_files('btchip') datas += collect_data_files('keepkeylib') @@ -54,6 +56,8 @@ a = Analysis([home+'run_electrum', home+'electrum/plugins/email_requests/qt.py', home+'electrum/plugins/trezor/client.py', home+'electrum/plugins/trezor/qt.py', + home+'electrum/plugins/safe_t/client.py', + home+'electrum/plugins/safe_t/qt.py', home+'electrum/plugins/keepkey/qt.py', home+'electrum/plugins/ledger/qt.py', #home+'packages/requests/utils.py' diff --git a/contrib/deterministic-build/requirements-hw.txt b/contrib/deterministic-build/requirements-hw.txt index 29fae2fd..f06b43e1 100644 --- a/contrib/deterministic-build/requirements-hw.txt +++ b/contrib/deterministic-build/requirements-hw.txt @@ -94,6 +94,9 @@ pyblake2==1.1.2 \ requests==2.19.1 \ --hash=sha256:63b52e3c866428a224f97cab011de738c36aec0185aa91cfacd418b5d58911d1 \ --hash=sha256:ec22d826a36ed72a7358ff3fe56cbd4ba69dd7a6718ffd450ff0e9df7a47ce6a +safet==0.1.3 \ + --hash=sha256:ba80fe9f6ba317ab9514a8726cd3792e68eb46dd419f380d48ae4a0ccae646dc \ + --hash=sha256:e5d8e6a87c8bdf1cefd07004181b93fd7631557fdab09d143ba8d1b29291d6dc setuptools==40.0.0 \ --hash=sha256:012adb8e25fbfd64c652e99e7bab58799a3aaf05d39ab38561f69190a909015f \ --hash=sha256:d68abee4eed409fbe8c302ac4d8429a1ffef912cd047a903b5701c024048dd49 diff --git a/contrib/requirements/requirements-hw.txt b/contrib/requirements/requirements-hw.txt index e8c7b2e9..4fe6477f 100644 --- a/contrib/requirements/requirements-hw.txt +++ b/contrib/requirements/requirements-hw.txt @@ -1,5 +1,6 @@ Cython>=0.27 trezor[hidapi]>=0.9.0 +safet[hidapi]>=0.1.0 keepkey btchip-python websocket-client diff --git a/electrum/plugins/safe_t/__init__.py b/electrum/plugins/safe_t/__init__.py new file mode 100644 index 00000000..9bfb2d9b --- /dev/null +++ b/electrum/plugins/safe_t/__init__.py @@ -0,0 +1,8 @@ +from electrum.i18n import _ + +fullname = 'Safe-T mini Wallet' +description = _('Provides support for Safe-T mini hardware wallet') +requires = [('safetlib','github.com/archos-safe-t/python-safet')] +registers_keystore = ('hardware', 'safe_t', _("Safe-T mini wallet")) +available_for = ['qt', 'cmdline'] + diff --git a/electrum/plugins/safe_t/client.py b/electrum/plugins/safe_t/client.py new file mode 100644 index 00000000..568f753b --- /dev/null +++ b/electrum/plugins/safe_t/client.py @@ -0,0 +1,11 @@ +from safetlib.client import proto, BaseClient, ProtocolMixin +from .clientbase import SafeTClientBase + +class SafeTClient(SafeTClientBase, ProtocolMixin, BaseClient): + def __init__(self, transport, handler, plugin): + BaseClient.__init__(self, transport=transport) + ProtocolMixin.__init__(self, transport=transport) + SafeTClientBase.__init__(self, handler, plugin, proto) + + +SafeTClientBase.wrap_methods(SafeTClient) diff --git a/electrum/plugins/safe_t/clientbase.py b/electrum/plugins/safe_t/clientbase.py new file mode 100644 index 00000000..68a4544c --- /dev/null +++ b/electrum/plugins/safe_t/clientbase.py @@ -0,0 +1,252 @@ +import time +from struct import pack + +from electrum.i18n import _ +from electrum.util import PrintError, UserCancelled +from electrum.keystore import bip39_normalize_passphrase +from electrum.bitcoin import serialize_xpub + + +class GuiMixin(object): + # Requires: self.proto, self.device + + # ref: https://github.com/trezor/trezor-common/blob/44dfb07cfaafffada4b2ce0d15ba1d90d17cf35e/protob/types.proto#L89 + messages = { + 3: _("Confirm the transaction output on your {} device"), + 4: _("Confirm internal entropy on your {} device to begin"), + 5: _("Write down the seed word shown on your {}"), + 6: _("Confirm on your {} that you want to wipe it clean"), + 7: _("Confirm on your {} device the message to sign"), + 8: _("Confirm the total amount spent and the transaction fee on your " + "{} device"), + 10: _("Confirm wallet address on your {} device"), + 14: _("Choose on your {} device where to enter your passphrase"), + 'default': _("Check your {} device to continue"), + } + + def callback_Failure(self, msg): + # BaseClient's unfortunate call() implementation forces us to + # raise exceptions on failure in order to unwind the stack. + # However, making the user acknowledge they cancelled + # gets old very quickly, so we suppress those. The NotInitialized + # one is misnamed and indicates a passphrase request was cancelled. + if msg.code in (self.types.FailureType.PinCancelled, + self.types.FailureType.ActionCancelled, + self.types.FailureType.NotInitialized): + raise UserCancelled() + raise RuntimeError(msg.message) + + def callback_ButtonRequest(self, msg): + message = self.msg + if not message: + message = self.messages.get(msg.code, self.messages['default']) + self.handler.show_message(message.format(self.device), self.cancel) + return self.proto.ButtonAck() + + def callback_PinMatrixRequest(self, msg): + if msg.type == 2: + msg = _("Enter a new PIN for your {}:") + elif msg.type == 3: + msg = (_("Re-enter the new PIN for your {}.\n\n" + "NOTE: the positions of the numbers have changed!")) + else: + msg = _("Enter your current {} PIN:") + pin = self.handler.get_pin(msg.format(self.device)) + if len(pin) > 9: + self.handler.show_error(_('The PIN cannot be longer than 9 characters.')) + pin = '' # to cancel below + if not pin: + return self.proto.Cancel() + return self.proto.PinMatrixAck(pin=pin) + + def callback_PassphraseRequest(self, req): + if req and hasattr(req, 'on_device') and req.on_device is True: + return self.proto.PassphraseAck() + + if self.creating_wallet: + msg = _("Enter a passphrase to generate this wallet. Each time " + "you use this wallet your {} will prompt you for the " + "passphrase. If you forget the passphrase you cannot " + "access the bitcoins in the wallet.").format(self.device) + else: + msg = _("Enter the passphrase to unlock this wallet:") + passphrase = self.handler.get_passphrase(msg, self.creating_wallet) + if passphrase is None: + return self.proto.Cancel() + passphrase = bip39_normalize_passphrase(passphrase) + + ack = self.proto.PassphraseAck(passphrase=passphrase) + length = len(ack.passphrase) + if length > 50: + self.handler.show_error(_("Too long passphrase ({} > 50 chars).").format(length)) + return self.proto.Cancel() + return ack + + def callback_PassphraseStateRequest(self, msg): + return self.proto.PassphraseStateAck() + + def callback_WordRequest(self, msg): + self.step += 1 + msg = _("Step {}/24. Enter seed word as explained on " + "your {}:").format(self.step, self.device) + word = self.handler.get_word(msg) + # Unfortunately the device can't handle self.proto.Cancel() + return self.proto.WordAck(word=word) + + +class SafeTClientBase(GuiMixin, PrintError): + + def __init__(self, handler, plugin, proto): + assert hasattr(self, 'tx_api') # ProtocolMixin already constructed? + self.proto = proto + self.device = plugin.device + self.handler = handler + self.tx_api = plugin + self.types = plugin.types + self.msg = None + self.creating_wallet = False + self.used() + + def __str__(self): + return "%s/%s" % (self.label(), self.features.device_id) + + def label(self): + '''The name given by the user to the device.''' + return self.features.label + + def is_initialized(self): + '''True if initialized, False if wiped.''' + return self.features.initialized + + def is_pairable(self): + return not self.features.bootloader_mode + + def has_usable_connection_with_device(self): + try: + res = self.ping("electrum pinging device") + assert res == "electrum pinging device" + except BaseException: + return False + return True + + def used(self): + self.last_operation = time.time() + + def prevent_timeouts(self): + self.last_operation = float('inf') + + def timeout(self, cutoff): + '''Time out the client if the last operation was before cutoff.''' + if self.last_operation < cutoff: + self.print_error("timed out") + self.clear_session() + + @staticmethod + def expand_path(n): + '''Convert bip32 path to list of uint32 integers with prime flags + 0/-1/1' -> [0, 0x80000001, 0x80000001]''' + # This code is similar to code in safetlib where it unfortunately + # is not declared as a staticmethod. Our n has an extra element. + PRIME_DERIVATION_FLAG = 0x80000000 + path = [] + for x in n.split('/')[1:]: + prime = 0 + if x.endswith("'"): + x = x.replace('\'', '') + prime = PRIME_DERIVATION_FLAG + if x.startswith('-'): + prime = PRIME_DERIVATION_FLAG + path.append(abs(int(x)) | prime) + return path + + def cancel(self): + '''Provided here as in keepkeylib but not safetlib.''' + self.transport.write(self.proto.Cancel()) + + def i4b(self, x): + return pack('>I', x) + + def get_xpub(self, bip32_path, xtype): + address_n = self.expand_path(bip32_path) + creating = False + node = self.get_public_node(address_n, creating).node + return serialize_xpub(xtype, node.chain_code, node.public_key, node.depth, self.i4b(node.fingerprint), self.i4b(node.child_num)) + + def toggle_passphrase(self): + if self.features.passphrase_protection: + self.msg = _("Confirm on your {} device to disable passphrases") + else: + self.msg = _("Confirm on your {} device to enable passphrases") + enabled = not self.features.passphrase_protection + self.apply_settings(use_passphrase=enabled) + + def change_label(self, label): + self.msg = _("Confirm the new label on your {} device") + self.apply_settings(label=label) + + def change_homescreen(self, homescreen): + self.msg = _("Confirm on your {} device to change your home screen") + self.apply_settings(homescreen=homescreen) + + def set_pin(self, remove): + if remove: + self.msg = _("Confirm on your {} device to disable PIN protection") + elif self.features.pin_protection: + self.msg = _("Confirm on your {} device to change your PIN") + else: + self.msg = _("Confirm on your {} device to set a PIN") + self.change_pin(remove) + + def clear_session(self): + '''Clear the session to force pin (and passphrase if enabled) + re-entry. Does not leak exceptions.''' + self.print_error("clear session:", self) + self.prevent_timeouts() + try: + super(SafeTClientBase, self).clear_session() + except BaseException as e: + # If the device was removed it has the same effect... + self.print_error("clear_session: ignoring error", str(e)) + + def get_public_node(self, address_n, creating): + self.creating_wallet = creating + return super(SafeTClientBase, self).get_public_node(address_n) + + def close(self): + '''Called when Our wallet was closed or the device removed.''' + self.print_error("closing client") + self.clear_session() + # Release the device + self.transport.close() + + def firmware_version(self): + f = self.features + return (f.major_version, f.minor_version, f.patch_version) + + def atleast_version(self, major, minor=0, patch=0): + return self.firmware_version() >= (major, minor, patch) + + @staticmethod + def wrapper(func): + '''Wrap methods to clear any message box they opened.''' + + def wrapped(self, *args, **kwargs): + try: + self.prevent_timeouts() + return func(self, *args, **kwargs) + finally: + self.used() + self.handler.finished() + self.creating_wallet = False + self.msg = None + + return wrapped + + @staticmethod + def wrap_methods(cls): + for method in ['apply_settings', 'change_pin', + 'get_address', 'get_public_node', + 'load_device_by_mnemonic', 'load_device_by_xprv', + 'recovery_device', 'reset_device', 'sign_message', + 'sign_tx', 'wipe_device']: + setattr(cls, method, cls.wrapper(getattr(cls, method))) diff --git a/electrum/plugins/safe_t/cmdline.py b/electrum/plugins/safe_t/cmdline.py new file mode 100644 index 00000000..9c6346d3 --- /dev/null +++ b/electrum/plugins/safe_t/cmdline.py @@ -0,0 +1,14 @@ +from electrum.plugin import hook +from .safe_t import SafeTPlugin +from ..hw_wallet import CmdLineHandler + +class Plugin(SafeTPlugin): + handler = CmdLineHandler() + @hook + def init_keystore(self, keystore): + if not isinstance(keystore, self.keystore_class): + return + keystore.handler = self.handler + + def create_handler(self, window): + return self.handler diff --git a/electrum/plugins/safe_t/qt.py b/electrum/plugins/safe_t/qt.py new file mode 100644 index 00000000..b9580aa1 --- /dev/null +++ b/electrum/plugins/safe_t/qt.py @@ -0,0 +1,492 @@ +from functools import partial +import threading + +from PyQt5.Qt import Qt +from PyQt5.Qt import QGridLayout, QInputDialog, QPushButton +from PyQt5.Qt import QVBoxLayout, QLabel + +from electrum.gui.qt.util import * +from electrum.i18n import _ +from electrum.plugin import hook, DeviceMgr +from electrum.util import PrintError, UserCancelled, bh2u +from electrum.wallet import Wallet, Standard_Wallet + +from ..hw_wallet.qt import QtHandlerBase, QtPluginBase +from .safe_t import SafeTPlugin, TIM_NEW, TIM_RECOVER, TIM_MNEMONIC + + +PASSPHRASE_HELP_SHORT =_( + "Passphrases allow you to access new wallets, each " + "hidden behind a particular case-sensitive passphrase.") +PASSPHRASE_HELP = PASSPHRASE_HELP_SHORT + " " + _( + "You need to create a separate Electrum wallet for each passphrase " + "you use as they each generate different addresses. Changing " + "your passphrase does not lose other wallets, each is still " + "accessible behind its own passphrase.") +RECOMMEND_PIN = _( + "You should enable PIN protection. Your PIN is the only protection " + "for your bitcoins if your device is lost or stolen.") +PASSPHRASE_NOT_PIN = _( + "If you forget a passphrase you will be unable to access any " + "bitcoins in the wallet behind it. A passphrase is not a PIN. " + "Only change this if you are sure you understand it.") + + +class QtHandler(QtHandlerBase): + + pin_signal = pyqtSignal(object) + + def __init__(self, win, pin_matrix_widget_class, device): + super(QtHandler, self).__init__(win, device) + self.pin_signal.connect(self.pin_dialog) + self.pin_matrix_widget_class = pin_matrix_widget_class + + def get_pin(self, msg): + self.done.clear() + self.pin_signal.emit(msg) + self.done.wait() + return self.response + + def pin_dialog(self, msg): + # Needed e.g. when resetting a device + self.clear_dialog() + dialog = WindowModalDialog(self.top_level_window(), _("Enter PIN")) + matrix = self.pin_matrix_widget_class() + vbox = QVBoxLayout() + vbox.addWidget(QLabel(msg)) + vbox.addWidget(matrix) + vbox.addLayout(Buttons(CancelButton(dialog), OkButton(dialog))) + dialog.setLayout(vbox) + dialog.exec_() + self.response = str(matrix.get_value()) + self.done.set() + + +class QtPlugin(QtPluginBase): + # Derived classes must provide the following class-static variables: + # icon_file + # pin_matrix_widget_class + + def create_handler(self, window): + return QtHandler(window, self.pin_matrix_widget_class(), self.device) + + @hook + def receive_menu(self, menu, addrs, wallet): + if len(addrs) != 1: + return + for keystore in wallet.get_keystores(): + if type(keystore) == self.keystore_class: + def show_address(): + keystore.thread.add(partial(self.show_address, wallet, addrs[0], keystore)) + menu.addAction(_("Show on {}").format(self.device), show_address) + break + + def show_settings_dialog(self, window, keystore): + device_id = self.choose_device(window, keystore) + if device_id: + SettingsDialog(window, self, keystore, device_id).exec_() + + def request_safe_t_init_settings(self, wizard, method, device): + vbox = QVBoxLayout() + next_enabled = True + label = QLabel(_("Enter a label to name your device:")) + name = QLineEdit() + hl = QHBoxLayout() + hl.addWidget(label) + hl.addWidget(name) + hl.addStretch(1) + vbox.addLayout(hl) + + def clean_text(widget): + text = widget.toPlainText().strip() + return ' '.join(text.split()) + + if method in [TIM_NEW, TIM_RECOVER]: + gb = QGroupBox() + hbox1 = QHBoxLayout() + gb.setLayout(hbox1) + vbox.addWidget(gb) + gb.setTitle(_("Select your seed length:")) + bg = QButtonGroup() + for i, count in enumerate([12, 18, 24]): + rb = QRadioButton(gb) + rb.setText(_("%d words") % count) + bg.addButton(rb) + bg.setId(rb, i) + hbox1.addWidget(rb) + rb.setChecked(True) + cb_pin = QCheckBox(_('Enable PIN protection')) + cb_pin.setChecked(True) + else: + text = QTextEdit() + text.setMaximumHeight(60) + if method == TIM_MNEMONIC: + msg = _("Enter your BIP39 mnemonic:") + else: + msg = _("Enter the master private key beginning with xprv:") + def set_enabled(): + from electrum.keystore import is_xprv + wizard.next_button.setEnabled(is_xprv(clean_text(text))) + text.textChanged.connect(set_enabled) + next_enabled = False + + vbox.addWidget(QLabel(msg)) + vbox.addWidget(text) + pin = QLineEdit() + pin.setValidator(QRegExpValidator(QRegExp('[1-9]{0,9}'))) + pin.setMaximumWidth(100) + hbox_pin = QHBoxLayout() + hbox_pin.addWidget(QLabel(_("Enter your PIN (digits 1-9):"))) + hbox_pin.addWidget(pin) + hbox_pin.addStretch(1) + + if method in [TIM_NEW, TIM_RECOVER]: + vbox.addWidget(WWLabel(RECOMMEND_PIN)) + vbox.addWidget(cb_pin) + else: + vbox.addLayout(hbox_pin) + + passphrase_msg = WWLabel(PASSPHRASE_HELP_SHORT) + passphrase_warning = WWLabel(PASSPHRASE_NOT_PIN) + passphrase_warning.setStyleSheet("color: red") + cb_phrase = QCheckBox(_('Enable passphrases')) + cb_phrase.setChecked(False) + vbox.addWidget(passphrase_msg) + vbox.addWidget(passphrase_warning) + vbox.addWidget(cb_phrase) + + wizard.exec_layout(vbox, next_enabled=next_enabled) + + if method in [TIM_NEW, TIM_RECOVER]: + item = bg.checkedId() + pin = cb_pin.isChecked() + else: + item = ' '.join(str(clean_text(text)).split()) + pin = str(pin.text()) + + return (item, name.text(), pin, cb_phrase.isChecked()) + + +class Plugin(SafeTPlugin, QtPlugin): + icon_unpaired = ":icons/safe-t_unpaired.png" + icon_paired = ":icons/safe-t.png" + + @classmethod + def pin_matrix_widget_class(self): + from safetlib.qt.pinmatrix import PinMatrixWidget + return PinMatrixWidget + + +class SettingsDialog(WindowModalDialog): + '''This dialog doesn't require a device be paired with a wallet. + We want users to be able to wipe a device even if they've forgotten + their PIN.''' + + def __init__(self, window, plugin, keystore, device_id): + title = _("{} Settings").format(plugin.device) + super(SettingsDialog, self).__init__(window, title) + self.setMaximumWidth(540) + + devmgr = plugin.device_manager() + config = devmgr.config + handler = keystore.handler + thread = keystore.thread + hs_rows, hs_cols = (64, 128) + + def invoke_client(method, *args, **kw_args): + unpair_after = kw_args.pop('unpair_after', False) + + def task(): + client = devmgr.client_by_id(device_id) + if not client: + raise RuntimeError("Device not connected") + if method: + getattr(client, method)(*args, **kw_args) + if unpair_after: + devmgr.unpair_id(device_id) + return client.features + + thread.add(task, on_success=update) + + def update(features): + self.features = features + set_label_enabled() + if features.bootloader_hash: + bl_hash = bh2u(features.bootloader_hash) + bl_hash = "\n".join([bl_hash[:32], bl_hash[32:]]) + else: + bl_hash = "N/A" + noyes = [_("No"), _("Yes")] + endis = [_("Enable Passphrases"), _("Disable Passphrases")] + disen = [_("Disabled"), _("Enabled")] + setchange = [_("Set a PIN"), _("Change PIN")] + + version = "%d.%d.%d" % (features.major_version, + features.minor_version, + features.patch_version) + + device_label.setText(features.label) + pin_set_label.setText(noyes[features.pin_protection]) + passphrases_label.setText(disen[features.passphrase_protection]) + bl_hash_label.setText(bl_hash) + label_edit.setText(features.label) + device_id_label.setText(features.device_id) + initialized_label.setText(noyes[features.initialized]) + version_label.setText(version) + clear_pin_button.setVisible(features.pin_protection) + clear_pin_warning.setVisible(features.pin_protection) + pin_button.setText(setchange[features.pin_protection]) + pin_msg.setVisible(not features.pin_protection) + passphrase_button.setText(endis[features.passphrase_protection]) + language_label.setText(features.language) + + def set_label_enabled(): + label_apply.setEnabled(label_edit.text() != self.features.label) + + def rename(): + invoke_client('change_label', label_edit.text()) + + def toggle_passphrase(): + title = _("Confirm Toggle Passphrase Protection") + currently_enabled = self.features.passphrase_protection + if currently_enabled: + msg = _("After disabling passphrases, you can only pair this " + "Electrum wallet if it had an empty passphrase. " + "If its passphrase was not empty, you will need to " + "create a new wallet with the install wizard. You " + "can use this wallet again at any time by re-enabling " + "passphrases and entering its passphrase.") + else: + msg = _("Your current Electrum wallet can only be used with " + "an empty passphrase. You must create a separate " + "wallet with the install wizard for other passphrases " + "as each one generates a new set of addresses.") + msg += "\n\n" + _("Are you sure you want to proceed?") + if not self.question(msg, title=title): + return + invoke_client('toggle_passphrase', unpair_after=currently_enabled) + + def change_homescreen(): + dialog = QFileDialog(self, _("Choose Homescreen")) + filename, __ = dialog.getOpenFileName() + if not filename: + return # user cancelled + + if filename.endswith('.toif'): + img = open(filename, 'rb').read() + if img[:8] != b'TOIf\x90\x00\x90\x00': + handler.show_error('File is not a TOIF file with size of 144x144') + return + else: + from PIL import Image # FIXME + im = Image.open(filename) + if im.size != (128, 64): + handler.show_error('Image must be 128 x 64 pixels') + return + im = im.convert('1') + pix = im.load() + img = bytearray(1024) + for j in range(64): + for i in range(128): + if pix[i, j]: + o = (i + j * 128) + img[o // 8] |= (1 << (7 - o % 8)) + img = bytes(img) + invoke_client('change_homescreen', img) + + def clear_homescreen(): + invoke_client('change_homescreen', b'\x00') + + def set_pin(): + invoke_client('set_pin', remove=False) + + def clear_pin(): + invoke_client('set_pin', remove=True) + + def wipe_device(): + wallet = window.wallet + if wallet and sum(wallet.get_balance()): + title = _("Confirm Device Wipe") + msg = _("Are you SURE you want to wipe the device?\n" + "Your wallet still has bitcoins in it!") + if not self.question(msg, title=title, + icon=QMessageBox.Critical): + return + invoke_client('wipe_device', unpair_after=True) + + def slider_moved(): + mins = timeout_slider.sliderPosition() + timeout_minutes.setText(_("%2d minutes") % mins) + + def slider_released(): + config.set_session_timeout(timeout_slider.sliderPosition() * 60) + + # Information tab + info_tab = QWidget() + info_layout = QVBoxLayout(info_tab) + info_glayout = QGridLayout() + info_glayout.setColumnStretch(2, 1) + device_label = QLabel() + pin_set_label = QLabel() + passphrases_label = QLabel() + version_label = QLabel() + device_id_label = QLabel() + bl_hash_label = QLabel() + bl_hash_label.setWordWrap(True) + language_label = QLabel() + initialized_label = QLabel() + rows = [ + (_("Device Label"), device_label), + (_("PIN set"), pin_set_label), + (_("Passphrases"), passphrases_label), + (_("Firmware Version"), version_label), + (_("Device ID"), device_id_label), + (_("Bootloader Hash"), bl_hash_label), + (_("Language"), language_label), + (_("Initialized"), initialized_label), + ] + for row_num, (label, widget) in enumerate(rows): + info_glayout.addWidget(QLabel(label), row_num, 0) + info_glayout.addWidget(widget, row_num, 1) + info_layout.addLayout(info_glayout) + + # Settings tab + settings_tab = QWidget() + settings_layout = QVBoxLayout(settings_tab) + settings_glayout = QGridLayout() + + # Settings tab - Label + label_msg = QLabel(_("Name this {}. If you have multiple devices " + "their labels help distinguish them.") + .format(plugin.device)) + label_msg.setWordWrap(True) + label_label = QLabel(_("Device Label")) + label_edit = QLineEdit() + label_edit.setMinimumWidth(150) + label_edit.setMaxLength(plugin.MAX_LABEL_LEN) + label_apply = QPushButton(_("Apply")) + label_apply.clicked.connect(rename) + label_edit.textChanged.connect(set_label_enabled) + settings_glayout.addWidget(label_label, 0, 0) + settings_glayout.addWidget(label_edit, 0, 1, 1, 2) + settings_glayout.addWidget(label_apply, 0, 3) + settings_glayout.addWidget(label_msg, 1, 1, 1, -1) + + # Settings tab - PIN + pin_label = QLabel(_("PIN Protection")) + pin_button = QPushButton() + pin_button.clicked.connect(set_pin) + settings_glayout.addWidget(pin_label, 2, 0) + settings_glayout.addWidget(pin_button, 2, 1) + pin_msg = QLabel(_("PIN protection is strongly recommended. " + "A PIN is your only protection against someone " + "stealing your bitcoins if they obtain physical " + "access to your {}.").format(plugin.device)) + pin_msg.setWordWrap(True) + pin_msg.setStyleSheet("color: red") + settings_glayout.addWidget(pin_msg, 3, 1, 1, -1) + + # Settings tab - Homescreen + homescreen_label = QLabel(_("Homescreen")) + homescreen_change_button = QPushButton(_("Change...")) + homescreen_clear_button = QPushButton(_("Reset")) + homescreen_change_button.clicked.connect(change_homescreen) + try: + import PIL + except ImportError: + homescreen_change_button.setDisabled(True) + homescreen_change_button.setToolTip( + _("Required package 'PIL' is not available - Please install it.") + ) + homescreen_clear_button.clicked.connect(clear_homescreen) + homescreen_msg = QLabel(_("You can set the homescreen on your " + "device to personalize it. You must " + "choose a {} x {} monochrome black and " + "white image.").format(hs_rows, hs_cols)) + homescreen_msg.setWordWrap(True) + settings_glayout.addWidget(homescreen_label, 4, 0) + settings_glayout.addWidget(homescreen_change_button, 4, 1) + settings_glayout.addWidget(homescreen_clear_button, 4, 2) + settings_glayout.addWidget(homescreen_msg, 5, 1, 1, -1) + + # Settings tab - Session Timeout + timeout_label = QLabel(_("Session Timeout")) + timeout_minutes = QLabel() + timeout_slider = QSlider(Qt.Horizontal) + timeout_slider.setRange(1, 60) + timeout_slider.setSingleStep(1) + timeout_slider.setTickInterval(5) + timeout_slider.setTickPosition(QSlider.TicksBelow) + timeout_slider.setTracking(True) + timeout_msg = QLabel( + _("Clear the session after the specified period " + "of inactivity. Once a session has timed out, " + "your PIN and passphrase (if enabled) must be " + "re-entered to use the device.")) + timeout_msg.setWordWrap(True) + timeout_slider.setSliderPosition(config.get_session_timeout() // 60) + slider_moved() + timeout_slider.valueChanged.connect(slider_moved) + timeout_slider.sliderReleased.connect(slider_released) + settings_glayout.addWidget(timeout_label, 6, 0) + settings_glayout.addWidget(timeout_slider, 6, 1, 1, 3) + settings_glayout.addWidget(timeout_minutes, 6, 4) + settings_glayout.addWidget(timeout_msg, 7, 1, 1, -1) + settings_layout.addLayout(settings_glayout) + settings_layout.addStretch(1) + + # Advanced tab + advanced_tab = QWidget() + advanced_layout = QVBoxLayout(advanced_tab) + advanced_glayout = QGridLayout() + + # Advanced tab - clear PIN + clear_pin_button = QPushButton(_("Disable PIN")) + clear_pin_button.clicked.connect(clear_pin) + clear_pin_warning = QLabel( + _("If you disable your PIN, anyone with physical access to your " + "{} device can spend your bitcoins.").format(plugin.device)) + clear_pin_warning.setWordWrap(True) + clear_pin_warning.setStyleSheet("color: red") + advanced_glayout.addWidget(clear_pin_button, 0, 2) + advanced_glayout.addWidget(clear_pin_warning, 1, 0, 1, 5) + + # Advanced tab - toggle passphrase protection + passphrase_button = QPushButton() + passphrase_button.clicked.connect(toggle_passphrase) + passphrase_msg = WWLabel(PASSPHRASE_HELP) + passphrase_warning = WWLabel(PASSPHRASE_NOT_PIN) + passphrase_warning.setStyleSheet("color: red") + advanced_glayout.addWidget(passphrase_button, 3, 2) + advanced_glayout.addWidget(passphrase_msg, 4, 0, 1, 5) + advanced_glayout.addWidget(passphrase_warning, 5, 0, 1, 5) + + # Advanced tab - wipe device + wipe_device_button = QPushButton(_("Wipe Device")) + wipe_device_button.clicked.connect(wipe_device) + wipe_device_msg = QLabel( + _("Wipe the device, removing all data from it. The firmware " + "is left unchanged.")) + wipe_device_msg.setWordWrap(True) + wipe_device_warning = QLabel( + _("Only wipe a device if you have the recovery seed written down " + "and the device wallet(s) are empty, otherwise the bitcoins " + "will be lost forever.")) + wipe_device_warning.setWordWrap(True) + wipe_device_warning.setStyleSheet("color: red") + advanced_glayout.addWidget(wipe_device_button, 6, 2) + advanced_glayout.addWidget(wipe_device_msg, 7, 0, 1, 5) + advanced_glayout.addWidget(wipe_device_warning, 8, 0, 1, 5) + advanced_layout.addLayout(advanced_glayout) + advanced_layout.addStretch(1) + + tabs = QTabWidget(self) + tabs.addTab(info_tab, _("Information")) + tabs.addTab(settings_tab, _("Settings")) + tabs.addTab(advanced_tab, _("Advanced")) + dialog_vbox = QVBoxLayout(self) + dialog_vbox.addWidget(tabs) + dialog_vbox.addLayout(Buttons(CloseButton(self))) + + # Update information + invoke_client(None) diff --git a/electrum/plugins/safe_t/safe_t.py b/electrum/plugins/safe_t/safe_t.py new file mode 100644 index 00000000..4d4410de --- /dev/null +++ b/electrum/plugins/safe_t/safe_t.py @@ -0,0 +1,508 @@ +from binascii import hexlify, unhexlify +import traceback +import sys + +from electrum.util import bfh, bh2u, versiontuple, UserCancelled +from electrum.bitcoin import (b58_address_to_hash160, xpub_from_pubkey, deserialize_xpub, + TYPE_ADDRESS, TYPE_SCRIPT, is_address) +from electrum import constants +from electrum.i18n import _ +from electrum.plugin import BasePlugin, Device +from electrum.transaction import deserialize, Transaction +from electrum.keystore import Hardware_KeyStore, is_xpubkey, parse_xpubkey, xtype_from_derivation +from electrum.base_wizard import ScriptTypeNotSupported + +from ..hw_wallet import HW_PluginBase +from ..hw_wallet.plugin import is_any_tx_output_on_change_branch + + +# Safe-T mini initialization methods +TIM_NEW, TIM_RECOVER, TIM_MNEMONIC, TIM_PRIVKEY = range(0, 4) + +# script "generation" +SCRIPT_GEN_LEGACY, SCRIPT_GEN_P2SH_SEGWIT, SCRIPT_GEN_NATIVE_SEGWIT = range(0, 3) + + +class SafeTKeyStore(Hardware_KeyStore): + hw_type = 'safe_t' + device = 'Safe-T mini' + + def get_derivation(self): + return self.derivation + + def get_script_gen(self): + xtype = xtype_from_derivation(self.derivation) + if xtype in ('p2wpkh', 'p2wsh'): + return SCRIPT_GEN_NATIVE_SEGWIT + elif xtype in ('p2wpkh-p2sh', 'p2wsh-p2sh'): + return SCRIPT_GEN_P2SH_SEGWIT + else: + return SCRIPT_GEN_LEGACY + + def get_client(self, force_pair=True): + return self.plugin.get_client(self, force_pair) + + def decrypt_message(self, sequence, message, password): + raise RuntimeError(_('Encryption and decryption are not implemented by {}').format(self.device)) + + def sign_message(self, sequence, message, password): + client = self.get_client() + address_path = self.get_derivation() + "/%d/%d"%sequence + address_n = client.expand_path(address_path) + msg_sig = client.sign_message(self.plugin.get_coin_name(), address_n, message) + return msg_sig.signature + + def sign_transaction(self, tx, password): + if tx.is_complete(): + return + # previous transactions used as inputs + prev_tx = {} + # path of the xpubs that are involved + xpub_path = {} + for txin in tx.inputs(): + pubkeys, x_pubkeys = tx.get_sorted_pubkeys(txin) + tx_hash = txin['prevout_hash'] + if txin.get('prev_tx') is None and not Transaction.is_segwit_input(txin): + raise Exception(_('Offline signing with {} is not supported for legacy inputs.').format(self.device)) + prev_tx[tx_hash] = txin['prev_tx'] + for x_pubkey in x_pubkeys: + if not is_xpubkey(x_pubkey): + continue + xpub, s = parse_xpubkey(x_pubkey) + if xpub == self.get_master_public_key(): + xpub_path[xpub] = self.get_derivation() + + self.plugin.sign_transaction(self, tx, prev_tx, xpub_path) + + +class SafeTPlugin(HW_PluginBase): + # Derived classes provide: + # + # class-static variables: client_class, firmware_URL, handler_class, + # libraries_available, libraries_URL, minimum_firmware, + # wallet_class, types + + firmware_URL = 'https://safe-t.io' + libraries_URL = 'https://github.com/archos-safe-t/python-safet' + minimum_firmware = (1, 0, 5) + keystore_class = SafeTKeyStore + minimum_library = (0, 1, 0) + SUPPORTED_XTYPES = ('standard', 'p2wpkh-p2sh', 'p2wpkh', 'p2wsh-p2sh', 'p2wsh') + + MAX_LABEL_LEN = 32 + + def __init__(self, parent, config, name): + HW_PluginBase.__init__(self, parent, config, name) + + try: + # Minimal test if python-safet is installed + import safetlib + try: + library_version = safetlib.__version__ + except AttributeError: + # python-safet only introduced __version__ in 0.1.0 + library_version = 'unknown' + if library_version == 'unknown' or \ + versiontuple(library_version) < self.minimum_library: + self.libraries_available_message = ( + _("Library version for '{}' is too old.").format(name) + + '\nInstalled: {}, Needed: {}' + .format(library_version, self.minimum_library)) + self.print_stderr(self.libraries_available_message) + raise ImportError() + self.libraries_available = True + except ImportError: + self.libraries_available = False + return + + from . import client + from . import transport + import safetlib.messages + self.client_class = client.SafeTClient + self.types = safetlib.messages + self.DEVICE_IDS = ('Safe-T mini',) + + self.transport_handler = transport.SafeTTransport() + self.device_manager().register_enumerate_func(self.enumerate) + + def enumerate(self): + devices = self.transport_handler.enumerate_devices() + return [Device(d.get_path(), -1, d.get_path(), 'Safe-T mini', 0) for d in devices] + + def create_client(self, device, handler): + try: + self.print_error("connecting to device at", device.path) + transport = self.transport_handler.get_transport(device.path) + except BaseException as e: + self.print_error("cannot connect at", device.path, str(e)) + return None + + if not transport: + self.print_error("cannot connect at", device.path) + return + + self.print_error("connected to device at", device.path) + client = self.client_class(transport, handler, self) + + # Try a ping for device sanity + try: + client.ping('t') + except BaseException as e: + self.print_error("ping failed", str(e)) + return None + + if not client.atleast_version(*self.minimum_firmware): + msg = (_('Outdated {} firmware for device labelled {}. Please ' + 'download the updated firmware from {}') + .format(self.device, client.label(), self.firmware_URL)) + self.print_error(msg) + if handler: + handler.show_error(msg) + else: + raise Exception(msg) + return None + + return client + + def get_client(self, keystore, force_pair=True): + devmgr = self.device_manager() + handler = keystore.handler + with devmgr.hid_lock: + client = devmgr.client_for_keystore(self, handler, keystore, force_pair) + # returns the client for a given keystore. can use xpub + if client: + client.used() + return client + + def get_coin_name(self): + return "Testnet" if constants.net.TESTNET else "Bitcoin" + + def initialize_device(self, device_id, wizard, handler): + # Initialization method + msg = _("Choose how you want to initialize your {}.\n\n" + "The first two methods are secure as no secret information " + "is entered into your computer.\n\n" + "For the last two methods you input secrets on your keyboard " + "and upload them to your {}, and so you should " + "only do those on a computer you know to be trustworthy " + "and free of malware." + ).format(self.device, self.device) + choices = [ + # Must be short as QT doesn't word-wrap radio button text + (TIM_NEW, _("Let the device generate a completely new seed randomly")), + (TIM_RECOVER, _("Recover from a seed you have previously written down")), + (TIM_MNEMONIC, _("Upload a BIP39 mnemonic to generate the seed")), + (TIM_PRIVKEY, _("Upload a master private key")) + ] + def f(method): + import threading + settings = self.request_safe_t_init_settings(wizard, method, self.device) + t = threading.Thread(target=self._initialize_device_safe, args=(settings, method, device_id, wizard, handler)) + t.setDaemon(True) + t.start() + exit_code = wizard.loop.exec_() + if exit_code != 0: + # this method (initialize_device) was called with the expectation + # of leaving the device in an initialized state when finishing. + # signal that this is not the case: + raise UserCancelled() + wizard.choice_dialog(title=_('Initialize Device'), message=msg, choices=choices, run_next=f) + + def _initialize_device_safe(self, settings, method, device_id, wizard, handler): + exit_code = 0 + try: + self._initialize_device(settings, method, device_id, wizard, handler) + except UserCancelled: + exit_code = 1 + except BaseException as e: + traceback.print_exc(file=sys.stderr) + handler.show_error(str(e)) + exit_code = 1 + finally: + wizard.loop.exit(exit_code) + + def _initialize_device(self, settings, method, device_id, wizard, handler): + item, label, pin_protection, passphrase_protection = settings + + if method == TIM_RECOVER: + handler.show_error(_( + "You will be asked to enter 24 words regardless of your " + "seed's actual length. If you enter a word incorrectly or " + "misspell it, you cannot change it or go back - you will need " + "to start again from the beginning.\n\nSo please enter " + "the words carefully!"), + blocking=True) + + language = 'english' + devmgr = self.device_manager() + client = devmgr.client_by_id(device_id) + + if method == TIM_NEW: + strength = 64 * (item + 2) # 128, 192 or 256 + u2f_counter = 0 + skip_backup = False + client.reset_device(True, strength, passphrase_protection, + pin_protection, label, language, + u2f_counter, skip_backup) + elif method == TIM_RECOVER: + word_count = 6 * (item + 2) # 12, 18 or 24 + client.step = 0 + client.recovery_device(word_count, passphrase_protection, + pin_protection, label, language) + elif method == TIM_MNEMONIC: + pin = pin_protection # It's the pin, not a boolean + client.load_device_by_mnemonic(str(item), pin, + passphrase_protection, + label, language) + else: + pin = pin_protection # It's the pin, not a boolean + client.load_device_by_xprv(item, pin, passphrase_protection, + label, language) + + def _make_node_path(self, xpub, address_n): + _, depth, fingerprint, child_num, chain_code, key = deserialize_xpub(xpub) + node = self.types.HDNodeType( + depth=depth, + fingerprint=int.from_bytes(fingerprint, 'big'), + child_num=int.from_bytes(child_num, 'big'), + chain_code=chain_code, + public_key=key, + ) + return self.types.HDNodePathType(node=node, address_n=address_n) + + def setup_device(self, device_info, wizard, purpose): + devmgr = self.device_manager() + device_id = device_info.device.id_ + client = devmgr.client_by_id(device_id) + if client is None: + raise Exception(_('Failed to create a client for this device.') + '\n' + + _('Make sure it is in the correct state.')) + # fixme: we should use: client.handler = wizard + client.handler = self.create_handler(wizard) + if not device_info.initialized: + self.initialize_device(device_id, wizard, client.handler) + client.get_xpub('m', 'standard') + client.used() + + def get_xpub(self, device_id, derivation, xtype, wizard): + if xtype not in self.SUPPORTED_XTYPES: + raise ScriptTypeNotSupported(_('This type of script is not supported with {}.').format(self.device)) + devmgr = self.device_manager() + client = devmgr.client_by_id(device_id) + client.handler = wizard + xpub = client.get_xpub(derivation, xtype) + client.used() + return xpub + + def get_safet_input_script_type(self, script_gen, is_multisig): + if script_gen == SCRIPT_GEN_NATIVE_SEGWIT: + return self.types.InputScriptType.SPENDWITNESS + elif script_gen == SCRIPT_GEN_P2SH_SEGWIT: + return self.types.InputScriptType.SPENDP2SHWITNESS + else: + if is_multisig: + return self.types.InputScriptType.SPENDMULTISIG + else: + return self.types.InputScriptType.SPENDADDRESS + + def sign_transaction(self, keystore, tx, prev_tx, xpub_path): + self.prev_tx = prev_tx + self.xpub_path = xpub_path + client = self.get_client(keystore) + inputs = self.tx_inputs(tx, True, keystore.get_script_gen()) + outputs = self.tx_outputs(keystore.get_derivation(), tx, keystore.get_script_gen()) + signatures = client.sign_tx(self.get_coin_name(), inputs, outputs, lock_time=tx.locktime)[0] + signatures = [(bh2u(x) + '01') for x in signatures] + tx.update_signatures(signatures) + + def show_address(self, wallet, address, keystore=None): + if keystore is None: + keystore = wallet.get_keystore() + if not self.show_address_helper(wallet, address, keystore): + return + client = self.get_client(keystore) + if not client.atleast_version(1, 0): + keystore.handler.show_error(_("Your device firmware is too old")) + return + change, index = wallet.get_address_index(address) + derivation = keystore.derivation + address_path = "%s/%d/%d"%(derivation, change, index) + address_n = client.expand_path(address_path) + xpubs = wallet.get_master_public_keys() + if len(xpubs) == 1: + script_gen = keystore.get_script_gen() + script_type = self.get_safet_input_script_type(script_gen, is_multisig=False) + client.get_address(self.get_coin_name(), address_n, True, script_type=script_type) + else: + def f(xpub): + return self._make_node_path(xpub, [change, index]) + pubkeys = wallet.get_public_keys(address) + # sort xpubs using the order of pubkeys + sorted_pubkeys, sorted_xpubs = zip(*sorted(zip(pubkeys, xpubs))) + pubkeys = list(map(f, sorted_xpubs)) + multisig = self.types.MultisigRedeemScriptType( + pubkeys=pubkeys, + signatures=[b''] * wallet.n, + m=wallet.m, + ) + script_gen = keystore.get_script_gen() + script_type = self.get_safet_input_script_type(script_gen, is_multisig=True) + client.get_address(self.get_coin_name(), address_n, True, multisig=multisig, script_type=script_type) + + def tx_inputs(self, tx, for_sig=False, script_gen=SCRIPT_GEN_LEGACY): + inputs = [] + for txin in tx.inputs(): + txinputtype = self.types.TxInputType() + if txin['type'] == 'coinbase': + prev_hash = b"\x00"*32 + prev_index = 0xffffffff # signed int -1 + else: + if for_sig: + x_pubkeys = txin['x_pubkeys'] + if len(x_pubkeys) == 1: + x_pubkey = x_pubkeys[0] + xpub, s = parse_xpubkey(x_pubkey) + xpub_n = self.client_class.expand_path(self.xpub_path[xpub]) + txinputtype._extend_address_n(xpub_n + s) + txinputtype.script_type = self.get_safet_input_script_type(script_gen, is_multisig=False) + else: + def f(x_pubkey): + if is_xpubkey(x_pubkey): + xpub, s = parse_xpubkey(x_pubkey) + else: + xpub = xpub_from_pubkey(0, bfh(x_pubkey)) + s = [] + return self._make_node_path(xpub, s) + pubkeys = list(map(f, x_pubkeys)) + multisig = self.types.MultisigRedeemScriptType( + pubkeys=pubkeys, + signatures=list(map(lambda x: bfh(x)[:-1] if x else b'', txin.get('signatures'))), + m=txin.get('num_sig'), + ) + script_type = self.get_safet_input_script_type(script_gen, is_multisig=True) + txinputtype = self.types.TxInputType( + script_type=script_type, + multisig=multisig + ) + # find which key is mine + for x_pubkey in x_pubkeys: + if is_xpubkey(x_pubkey): + xpub, s = parse_xpubkey(x_pubkey) + if xpub in self.xpub_path: + xpub_n = self.client_class.expand_path(self.xpub_path[xpub]) + txinputtype._extend_address_n(xpub_n + s) + break + + prev_hash = unhexlify(txin['prevout_hash']) + prev_index = txin['prevout_n'] + + if 'value' in txin: + txinputtype.amount = txin['value'] + txinputtype.prev_hash = prev_hash + txinputtype.prev_index = prev_index + + if txin.get('scriptSig') is not None: + script_sig = bfh(txin['scriptSig']) + txinputtype.script_sig = script_sig + + txinputtype.sequence = txin.get('sequence', 0xffffffff - 1) + + inputs.append(txinputtype) + + return inputs + + def tx_outputs(self, derivation, tx, script_gen=SCRIPT_GEN_LEGACY): + + def create_output_by_derivation(info): + index, xpubs, m = info + if len(xpubs) == 1: + if script_gen == SCRIPT_GEN_NATIVE_SEGWIT: + script_type = self.types.OutputScriptType.PAYTOWITNESS + elif script_gen == SCRIPT_GEN_P2SH_SEGWIT: + script_type = self.types.OutputScriptType.PAYTOP2SHWITNESS + else: + script_type = self.types.OutputScriptType.PAYTOADDRESS + address_n = self.client_class.expand_path(derivation + "/%d/%d" % index) + txoutputtype = self.types.TxOutputType( + amount=amount, + script_type=script_type, + address_n=address_n, + ) + else: + if script_gen == SCRIPT_GEN_NATIVE_SEGWIT: + script_type = self.types.OutputScriptType.PAYTOWITNESS + elif script_gen == SCRIPT_GEN_P2SH_SEGWIT: + script_type = self.types.OutputScriptType.PAYTOP2SHWITNESS + else: + script_type = self.types.OutputScriptType.PAYTOMULTISIG + address_n = self.client_class.expand_path("/%d/%d" % index) + pubkeys = [self._make_node_path(xpub, address_n) for xpub in xpubs] + multisig = self.types.MultisigRedeemScriptType( + pubkeys=pubkeys, + signatures=[b''] * len(pubkeys), + m=m) + txoutputtype = self.types.TxOutputType( + multisig=multisig, + amount=amount, + address_n=self.client_class.expand_path(derivation + "/%d/%d" % index), + script_type=script_type) + return txoutputtype + + def create_output_by_address(): + txoutputtype = self.types.TxOutputType() + txoutputtype.amount = amount + if _type == TYPE_SCRIPT: + txoutputtype.script_type = self.types.OutputScriptType.PAYTOOPRETURN + txoutputtype.op_return_data = address[2:] + elif _type == TYPE_ADDRESS: + txoutputtype.script_type = self.types.OutputScriptType.PAYTOADDRESS + txoutputtype.address = address + return txoutputtype + + outputs = [] + has_change = False + any_output_on_change_branch = is_any_tx_output_on_change_branch(tx) + + for _type, address, amount in tx.outputs(): + use_create_by_derivation = False + + info = tx.output_info.get(address) + if info is not None and not has_change: + index, xpubs, m = info + on_change_branch = index[0] == 1 + # prioritise hiding outputs on the 'change' branch from user + # because no more than one change address allowed + # note: ^ restriction can be removed once we require fw + # that has https://github.com/trezor/trezor-mcu/pull/306 + if on_change_branch == any_output_on_change_branch: + use_create_by_derivation = True + has_change = True + + if use_create_by_derivation: + txoutputtype = create_output_by_derivation(info) + else: + txoutputtype = create_output_by_address() + outputs.append(txoutputtype) + + return outputs + + def electrum_tx_to_txtype(self, tx): + t = self.types.TransactionType() + if tx is None: + # probably for segwit input and we don't need this prev txn + return t + d = deserialize(tx.raw) + t.version = d['version'] + t.lock_time = d['lockTime'] + inputs = self.tx_inputs(tx) + t._extend_inputs(inputs) + for vout in d['outputs']: + o = t._add_bin_outputs() + o.amount = vout['value'] + o.script_pubkey = bfh(vout['scriptPubKey']) + return t + + # This function is called from the TREZOR libraries (via tx_api) + def get_tx(self, tx_hash): + tx = self.prev_tx[tx_hash] + return self.electrum_tx_to_txtype(tx) diff --git a/electrum/plugins/safe_t/transport.py b/electrum/plugins/safe_t/transport.py new file mode 100644 index 00000000..59dc915f --- /dev/null +++ b/electrum/plugins/safe_t/transport.py @@ -0,0 +1,95 @@ +from electrum.util import PrintError + + +class SafeTTransport(PrintError): + + @staticmethod + def all_transports(): + """Reimplemented safetlib.transport.all_transports so that we can + enable/disable specific transports. + """ + try: + # only to detect safetlib version + from safetlib.transport import all_transports + except ImportError: + # old safetlib. compat for safetlib < 0.9.2 + transports = [] + #try: + # from safetlib.transport_bridge import BridgeTransport + # transports.append(BridgeTransport) + #except BaseException: + # pass + try: + from safetlib.transport_hid import HidTransport + transports.append(HidTransport) + except BaseException: + pass + try: + from safetlib.transport_udp import UdpTransport + transports.append(UdpTransport) + except BaseException: + pass + try: + from safetlib.transport_webusb import WebUsbTransport + transports.append(WebUsbTransport) + except BaseException: + pass + else: + # new safetlib. + transports = [] + #try: + # from safetlib.transport.bridge import BridgeTransport + # transports.append(BridgeTransport) + #except BaseException: + # pass + try: + from safetlib.transport.hid import HidTransport + transports.append(HidTransport) + except BaseException: + pass + try: + from safetlib.transport.udp import UdpTransport + transports.append(UdpTransport) + except BaseException: + pass + try: + from safetlib.transport.webusb import WebUsbTransport + transports.append(WebUsbTransport) + except BaseException: + pass + return transports + return transports + + def enumerate_devices(self): + """Just like safetlib.transport.enumerate_devices, + but with exception catching, so that transports can fail separately. + """ + devices = [] + for transport in self.all_transports(): + try: + new_devices = transport.enumerate() + except BaseException as e: + self.print_error('enumerate failed for {}. error {}' + .format(transport.__name__, str(e))) + else: + devices.extend(new_devices) + return devices + + def get_transport(self, path=None): + """Reimplemented safetlib.transport.get_transport, + (1) for old safetlib + (2) to be able to disable specific transports + (3) to call our own enumerate_devices that catches exceptions + """ + if path is None: + try: + return self.enumerate_devices()[0] + except IndexError: + raise Exception("No Safe-T mini found") from None + + def match_prefix(a, b): + return a.startswith(b) or b.startswith(a) + transports = [t for t in self.all_transports() if match_prefix(path, t.PATH_PREFIX)] + if transports: + return transports[0].find_by_path(path) + raise Exception("Unknown path prefix '%s'" % path) diff --git a/electrum/storage.py b/electrum/storage.py index b951e4b7..40c13289 100644 --- a/electrum/storage.py +++ b/electrum/storage.py @@ -319,7 +319,7 @@ class WalletStorage(JsonDB): storage2.upgrade() storage2.write() result = [storage1.path, storage2.path] - elif wallet_type in ['bip44', 'trezor', 'keepkey', 'ledger', 'btchip', 'digitalbitbox']: + elif wallet_type in ['bip44', 'trezor', 'keepkey', 'ledger', 'btchip', 'digitalbitbox', 'safe_t']: mpk = storage.get('master_public_keys') for k in d.keys(): i = int(k) @@ -416,7 +416,7 @@ class WalletStorage(JsonDB): self.put('wallet_type', 'standard') self.put('keystore', d) - elif wallet_type in ['trezor', 'keepkey', 'ledger', 'digitalbitbox']: + elif wallet_type in ['trezor', 'keepkey', 'ledger', 'digitalbitbox', 'safe_t']: xpub = xpubs["x/0'"] derivation = self.get('derivation', bip44_derivation(0)) d = { diff --git a/electrum/transaction.py b/electrum/transaction.py index 54ca5948..e1a2f87e 100644 --- a/electrum/transaction.py +++ b/electrum/transaction.py @@ -672,7 +672,7 @@ class Transaction: `signatures` is expected to be a list of sigs with signatures[i] intended for self._inputs[i]. - This is used by the Trezor and KeepKey plugins. + This is used by the Trezor, KeepKey an Safe-T plugins. """ if self.is_complete(): return diff --git a/icons.qrc b/icons.qrc index 82cfba32..feb07a02 100644 --- a/icons.qrc +++ b/icons.qrc @@ -31,6 +31,8 @@ icons/qrcode.png icons/qrcode_white.png icons/preferences.png + icons/safe-t_unpaired.png + icons/safe-t.png icons/seed.png icons/status_connected.png icons/status_connected_proxy.png diff --git a/icons/safe-t.png b/icons/safe-t.png new file mode 100644 index 0000000000000000000000000000000000000000..7a574dec86a9bb621b622595baffb81359da8f09 GIT binary patch literal 3871 zcmV+)58&{LP)aB^>EX>4U6ba`-PAVE-2F#rGvnd3@N%}XuHOjal;%1_J8 zN##-i17i~|6H60IqeKG(0}BHPFf=eQHUyGJK(;wlDA51~m>QT_ni-oJngcP2&jkQT zwiL-a)I%}=01j(OL_t(|oYkCpR8`l#$3L5k0xA!uA_f^`asWZa2?S#jjYBk=7dz!O zDH&GVw6BTnYaN;wpNV}26O%M~sZrZRlk}yHL)2)VvF)l8iV5HpAWZiHQK_?#D+-az zet%rBaL+#Xo;v~ktaTUg+vn`xzF+p)XP@8x?IY*`NGSt=UTvR1zyyf4pRFA}Ex;|{ zmJq_$J+1AG=t^f&%KktKFa}5lMgj>yj0e^I9jI#i6a$|FB|?Zg-?Y{keiKki83~MS z`-}(10mJ>}uLBo>lfZG{MBDFXfAv_e1(Z^T02x3UFdBQWN&)(NTepuniM^+m0HwfD zA;e{GEA>b~DdhlQ1&{?y@U{`%$QM8!@SG6hoF^5yEufV00pLwwh$jwp>ZGNmg_c{l zXtCL_*=!&sw$@exL;xZ{5D*YRpec~RUV#J#1`-eu;4Am-yd;EJ;_gXq2`Huf4)8~J zowQ@MS}89pr=p^q%8CjsW;3-lwba$s(c0STR$bvkh7c1ILu_m;iAjkhjTlMNh$O

;tnQE?&4mNl6J`l$KIjQc7`gF=x-5ai?t_F?jG`MvWd# za!Lxx$;pfwJ(}p4Xm@(7TRqVo0gnNb9qrA{%^WW(qO7cps>(_%W-}G#6*M$77_Hjx z1P2Eb7avd3h$Ipc6G==+Bz;0U{Ra%tJEgY};+B!R4GAcvoD3Xuv~SzGl~pUBqp`8k zNHskQlgY%x-+zSV&pd<4WO6KiS_tvFk$Q~w>ug8EvHX0N{c_1arFZ}~n~lwHZ(`&6 z^{V3OM(Z&uV3wodolTpK`smTT_s+X2v*!Ei;c*WvC@3)MqepY`;zcYLi=+7)QpyoV z&JT2TKwCuvJK8K33-#Bo8E&K}#14-oH8l-Wpb3lF!s%0|kW%`^S5ZL$adB~u<}9Gx z)wwPO1T^!a<3&ZrTIK;mA|hCw`vP}PpWexCu~_)k(xns@6#Cp}VL>6^{PwpU&Dp?P zuFf@b31>JOP8>Hp9dwKE@Nl;8*r5>*ATBPB_uhY>tZc)>lVJ)AmBW#t3AGUcH7)i> zF92A%>UkoDM(CFJ>eY)E*RCa`PamK9t*xn54M_=7%E7MAbt<5gav=8T%554N8ZcK? zx!p)tgbx|QjF~eH9e3}ndwuTz^y$+ov&%q=zI4Gmnqdeza~UrMRFQsqp*T1P`4A z|K>H$ojXTpXehs4y_)Sicak=CtcT5Xj=v($MmVPyP&X;ca1SgjC}6?7`5ZiO0D#oA zRJQH-kk{XMgP2%PveWcAeNy{esuL~oIXXQsR_}P_btt7Q#37WkaQ^+bzr??N?|Up+ zzKp(o`vPEVvGKtNf8uu=Hge|6FFk9y1NY3FK~z)}0F8||*t2^#H*ehVN!`wfijHRg zp@WX*Y9WMXRs#A0T5#yLcTKyqlQ=_%4&}L3tGIi{J-2PP78^Tv?qK8k4Sor_b_Kz~ z!4#K$;b^`ogiwaT^#r^Mbjo{}Fm{~sAwDZRn`fT=PZE=oI-bwrLx*^0(`F7GJlL&c znkQ7%RBLQP=dP(IV5?(UQe3=l29?QVV!^_N{N$%U)r^PL)z$pz{Xep6#||!E_Bx-j zOQ@{2sAd@^A%sQ%JpnacCLz{!cCN`}V*Y~rdEy5@VC2Y7ODWh|Y#i9XpY7YWvHx#> z^ZRqHH+UsrTgJT2!A9b6#$m_9N52~eS1IQ;K74ka&s`s zfPn)j&@S6v5ke?e==>&NJ3}HOcyQ4o9$d6YIo)Y$YU0S@Jo54mlYcays>(`_{P>Ee z7BA+h#ZNn$cM2gMP#y1g0XvdDVFI&e&tmr6x!iHb9m-=aUAn~4yd!-2=`l{7JV|9m zrQ6x7o|2K7$y=K@(YtqV$D&^dAzoIU)1?A-BsnF8duPoeYg#rbDJiNPJpgXrxIsxt zDJ8|lSj=WBD=Vq0tW-r!IuRBYM*74FOq@80^a<(2#Ksa976#a(F10l^6crXxSXe-{ z*~0w`7xJSYKS^Mqa>cK&5JEZ5ben)32@MTp@{}n|&B|i(lqtj~#On&$&dnP)ICt(G zb#-;rovWk1p`Q9{*T~4oBq}gn*8OgoKcslFXRYG*Z)2Nf|SS z=%{E+fzCzs6Mq##=%!5cNC8!t0!_q3$1prTj@a1YBqYQ$EGmk~VZ(@wjC9#!y=0LP zV!Nt9bHNKCS{Jt1^uYq4RmSUM#`PR*LinwQr4TRgBwr<4NfO{6aYo#s80#;3a+ zxVw^BvuE?$^>6ByOC*P{pWx$$JdRvHMOE_|uC>_LoCgGkk`NR{X5TdC_0J-^-#CQx ze0#GHLK#%m6Ywi&8He&%RMO-=KG+3IKp{jGaK580CN_o%6T5F~@GqAx)m*wZ*~-H;&tkRtKALmA^#%{u zJc~_gp2NQ(rOa*r!`Z^RCd22&wQC6t4fRnI-DA>ZO|~)bT-oE*nCFUu#!@z4*{v#? z*8Yn#0XI7uqGMwCpLOf>9=QERSXdb2#&^#3lt^AUzsc{ubI)%^qKPkOwC&p07f=ZC zF`$|EWMyZu_SIMXv1449bj8qd$a=!9sfwpmHJ{;-^@OS@3tXr8J_%?hM)T&+=gkcp zx~wNooT$Bg|L-f;>A=21$5%GE5Kstl7I;EccF)Wi{C?Bh1P2Fu-9T52PuHYsbM(4$ zUx#1Fzph-=&j15Em4py`fkmpatn6$)*tXT%_Y}A!cDQyytFqZQ3;0H;XjV>VlEC10 z*S1iOW;Y(#Bg;EJ+QsN(-3&0VX^$j#q&+au73lbKeiLy-2(bm7o;V~Tf)BQBW#+xU z--6ZCaqCvv))N;3ntlEuAt9{Wu%4$DFZOx-Q1t?f0fAjNAJy@74crtEKnSr5xDU{5 z^AB%^PtPwF;8M!zvCb3S+|Ab=2}9Q$NaIY}B86~)#MxAWbH9`e-1ZB!eg z?+f$&#%n$1{)^JrVempg03k$^5F#5`qbd&y3S!mj|K^Q#>*&|7pT~ZR6dSy!g^nX3 zD9YnHjgk-)MRuq%6E_dM6tEp3#4=zuibyuV?74IJ=;J-4rg?VR)z{Z+UNayNEQ`qT zs6KaC7LlV0^XCg8JiCeLA%xfm%m9=xAc&5MVaxXI{P4*id32Zl>{CsMlAJ;Fnb607 zf!_2!qsSRFUsZIt{TIKiS4t^g1s3a-AIs0@KYsReE}TEFcZibDju$a-V5c{*P}Wq# z#4jG22dW7!gyOK;K3lcp-^EqbvfE>|TUIE4B9x?Mv-v{&hZ(gINrA1?(6P8KAsCP;4<8*VSQ_}m4qF~g!dg5KeVLb%8@HNO%%53bf z4pq*srIh^dd+)L4g$4_*KMX z9xbeQ^?RgyVHfdny<=*sYgoQ?2?d}1<7OTi8A={a0X(jLaNp{yC=4j(7 z%*|&zrFO&z4I{H}8uR;SlhtpmGvNRp6heGx*tx%vQqGf7o{>(t(by=jUb!l*R;z4n zZFP32Ke;HSoY&oYDUnhRl2RV_@(UJ9DHlj7^So6`L7tQ{vAcD0iIg%|O8J!sU+^`D zde%rOFZe*X3sOquqiK&tN*O7o{K()JJSnBryiu-0U2CM2M?4Vjh$q5zf97Ndq?B{9 zKT~I6fA@YzjsSUW&)r7q8q`)#HjoL#^N;Oo*8nqc6#HXYo)F@SCvEOg0(L}7*$)T- hz6PwST|zy9{|BUc9vVz0t(O1*002ovPDHLkV1np_YD)kB literal 0 HcmV?d00001 diff --git a/icons/safe-t_unpaired.png b/icons/safe-t_unpaired.png new file mode 100644 index 0000000000000000000000000000000000000000..c67a344f1aaaaaba7c42216848c037b660b756ff GIT binary patch literal 3719 zcmV;24tVj2P)aB^>EX>4U6ba`-PAVE-2F#rGvnd3@N%}XuHOjal;%1_J8 zN##-i17i~|6H60IqeKG(0}BHPFf=eQHUyGJK(;wlDA51~m>QT_ni-oJngcP2&jkQT zwiL-a)I%}=01eYgL_t(|oYkCrbXC=z$3HtaAXHf4VUu9YgS>bof|>*fB!~*~62U39 z)~Pf2Se-hvswyTg32$~xVm|&haL`lj^NG1?* z$jxv5I0=*7v(LTfzLNB_)?I|(KKpUMcb|Rs{_WpBihh8IL;wRiKaqd~P@VsFbo;ae zSAnZaDR)|hhJqy$Hb7Bm`GAm5^3paq>mX(`j~X0 zqoV_D9vHXvG=DPK7EnZP#Tb}varVp^Dk>`Y;=~C~RGgr^yqr^CedSNv0^)`nZeZN_ z@nmFXl97?YxbfpjNKEjj$NJS1{Soj8Fxk`o?YG}@w6v7U%1UZ$YH-!nQC(F{OG}I0 zqQlOhL4!z6Ng;hqI;m-Cq@|{jm7T?~;lr)63{*;8wNth&0YzjoaKzKTecLwHu6dfv zmoM8XreEQ3IC$tE|AAFcJ%z*J@JxPEDfOD2a_si&d{4uX;$l|*V)@^scmQsyl^KEyu2ad7g$Xn>O36qhIsZmjBk7P2bmwfO}v`Nr~M$`ZZ_Ip2g*Id78f~ zB4h02kMwmxXF(&n+gvUe%@;4)Ze$=NjZR|1go!vJ9k}XT96xpp5eZ9Or6naKCntNF z^MEQ}d3_29nC3-COH1vwEC6C+V|nJeXSsdGj2?EE%f-K~Sb?*|8S*-vB~I@D!4Evm z`M^eBdF|Z7xt@m4j@q6M`bBhfG&^?gG>He0oSe*C@4Q1^zU|@3HcqE@II=aNwj-da z#o_1$0BhF%lGstPmgxfq4B&-z>lplisS}BKZlo`wV`7*&YnH9+?w);j$m>6T{J75S(^2B92Tt-d94jlc*(CqS$jGq0+{(<% z40-+KpMS11XZgzKtHrl@8ctMH*ld!2s0jOh4Tr{{rfNsh~Oi*V7aoEb@6>OG^v;_UsKy{pZe|^EAha$S805 zECuY`0P3-a(c0P?R+rS;HEU>XZM94n5uRTCpENf&ho%0iO6|Pa>wxR62bwZgSNWWn z2Sr_79gjWyh~bge@bbEscyIUau+?8vQ>`-xEZ}%gLv2myr!StAI-MLUGT+Kqu3X`b z4I9E*x5e~qoVPsQ1kCU>So}gEM>7_2{``5aUbW@p44Ci>Slv4U_7&Atp}mDo?ujqI z4F8@+U((Xj!i5VLJk7&I#B!&~n}7|ThPb%60Gsx=gT-BDx*NOD3MeAip>b=kUbBYi zn2>D;_e&a1n)(eG@jF_qdZ5Vz4Uj%2oqd1)6AwK2LzL>(xTId=)TvWCvt{s;m4GG} z)@#3hmBz+Kh7KLdubz2^9lLfh@s?WxY^Ha@<$1P4qh3JEpeWlt&{jIAX=@8f+1`jx zNML_qfv34%DP@|KfVF@wJi6^2({Jw~&ZtqNczW$x?wonoHJiKL&8}TL+3@=MusB_N zgF%A^QC|6lr}-PDl-3PyCEx|1N8Cg9Ew^eH@p<|AJoO*{PFi|;_xv0zEM&{3%@h_C z^y!!u2(>5cO*Uom)U*CImI%9%HeRZaM2?E<)=S2jfeI1_59(T-?Mw?PR^eX zI-aptsHu19W*H8plt}<90Zm;dHOY5$uEXJA!NPla?8iT5?ARV#DY)C+9N52~9ox6F z|1WHI zJ)z=>rAv8Y>64!3T}r9@bk~Pn!0u#aXES@wZ05|H$Bj4MsJ-UgxpN#YI>aZR9N}15 z88y{4en+ncO77IDY}~wwfddD6CM{D+y`;<2s{(c>BQuk`XU`^YdOn$%nYtJ~0KRE! zqoU#j73Jl)>guSesiC%}M&~u@K~z)}SvlF{CYoECX})-o+}x?eCnS)Z60-KbqLli%Rbsyo(38P~2a}PR!OasUGGXEb zGHBT%6Ay z8zhUBQaf}BrW;-<)!{h}><_Lhce@*x%he+oa$4Rro_}#&xWxm*JEaxS${S-^GK?58 zLU&!CF?-G&w!Hnebu^a<6c)mY6_A$)v9WO7b;c()*03#KZd~Jgsky)^!D=OP9OFdg zHRjKmW3})dhYo&4@ii`d7G4%oaIo_!YVIy6#KVbv;FvIMLHoE9jhKJYYJU2SrT zjT)t&xAis&iHSV-;tTr3`g)i$1s;9WzIb3<`{<)a-g>hmp_7dwlHzH!63`R?lRhS0 zcU`Zu?&X&lIdY^q;lKfyGzo$UelIC8$|x+p9k74#~2!*%j{CSu)3-;~PrA`%*9^Kz+3`_-jEJ_0!>AR!BzgV%tES}p94?Prm z@c=GeGRigwI(=6}p6mLLw~cjEgwG4>)-iPG&=9rIKPF8!MH|DGEy0a>z9=a%p6F(# zcl}{GLl%+wfa$Sabg+oW9)6hi_V!RT(H2orQIsC@7?wpKJ>9>jZ6KtkLUlEm7ZW3t zQWq_Ipi=5XGy>-3=dtdUm&0*l+>$Ka&{0?ze(}JlL;qCH1K%OOj{~OHh}^Sa0l!_p z-pch8a&mGsx9>+E^-A4d>dFTn0xG3W0gvg@?wU1|H#WV=ph1IzZl5o1%QA&(!{NhW zulE|o#X4&)*f^=Al-dI<)}`g;=kxCNZNa{$z$ZzgH5*#J>H614Y^8(Ez9-@A&m7 z?d*w*0Mj}D!Gj0$U+Z7ziKRH>JrF&Ae$eIGX5KuV^&o`k z^>>|%>lIe%dq3L4%H_+sbm>w+=c)F-l=BCr)Jk9u8c#OB zoOyG3@52w6FmXaadkeJBU z9Xt4^$A1#gQ~J|SJ~bPcEP?DWy4bR^j5^F5?D``t`;{W{3b52F{YY^!|N66^bLRAE zt4p+edbHHMJ6Kt1I8*~19xNF$#BgVMgy~_&p&Beosh?p?b+qp@osyf&hkN%hXYO3T z%H6STyV*E)ENtBh=76BU892}=(<~mq=I;Lq>yCNnF4o(?6sv1?@7%?j)vLXp6YkE| z9ozN$B7640_rD)}PaHDD5KsT(n%*xMR;Qj)s!l1D3ur^b02bc2h`oRM*!KHiUiIT+<$)$;igZW zCWj9l@-CcC$btfqWy?gSP8ErX5g9np_{78*wq?tVYrG35BJy{Atz8d9i50=SOs9~34wxdHdHfcx?;|I> zAtLiI*3@|z@80juA)u&p-EF6=k)7q_15<$%{{Q&eMW7BijIk^$Qc8UtNSphWfZY+1 lA;4hZe}PurDWU$r{{b5f Date: Sun, 29 Jul 2018 03:56:10 +0200 Subject: [PATCH 38/56] network: handle reorg (sooner) in case of multiple forks at given height (#4537) --- electrum/network.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/electrum/network.py b/electrum/network.py index 5b8a2c86..86e48cc5 100644 --- a/electrum/network.py +++ b/electrum/network.py @@ -897,6 +897,7 @@ class Network(util.DaemonThread): self.connection_down(interface.server) return height = header.get('block_height') + #interface.print_error('got header', height, blockchain.hash_header(header)) if interface.request != height: interface.print_error("unsolicited header",interface.request, height) self.connection_down(interface.server) @@ -952,7 +953,7 @@ class Network(util.DaemonThread): elif branch.parent().check_header(header): interface.print_error('reorg', interface.bad, interface.tip) interface.blockchain = branch.parent() - next_height = None + next_height = interface.bad else: interface.print_error('checkpoint conflicts with existing fork', branch.path()) branch.write(b'', 0) @@ -1086,6 +1087,7 @@ class Network(util.DaemonThread): except InvalidHeader: self.connection_down(interface.server) return + #interface.print_error('notified of header', height, blockchain.hash_header(header)) if height < self.max_checkpoint(): self.connection_down(interface.server) return From eaf72aa9515bb3c09f68a483645dbeb96932616f Mon Sep 17 00:00:00 2001 From: SomberNight Date: Sun, 29 Jul 2018 04:00:02 +0200 Subject: [PATCH 39/56] network: handle one-block-long fork also add fixme about incorrect behaviour in case of a fork height higher than our local chain tip --- electrum/network.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/electrum/network.py b/electrum/network.py index 86e48cc5..07bad8dc 100644 --- a/electrum/network.py +++ b/electrum/network.py @@ -912,6 +912,9 @@ class Network(util.DaemonThread): next_height = height + 1 interface.blockchain.catch_up = interface.server elif chain: + # FIXME should await "initial chunk download". + # binary search will NOT do the correct thing if we don't yet + # have all headers up to the fork height interface.print_error("binary search") interface.mode = 'binary' interface.blockchain = chain @@ -973,8 +976,10 @@ class Network(util.DaemonThread): interface.blockchain = b interface.print_error("new chain", b.checkpoint) interface.mode = 'catch_up' - next_height = interface.bad + 1 - interface.blockchain.catch_up = interface.server + maybe_next_height = interface.bad + 1 + if maybe_next_height <= interface.tip: + next_height = maybe_next_height + interface.blockchain.catch_up = interface.server else: assert bh == interface.good if interface.blockchain.catch_up is None and bh < interface.tip: From 8e69174374aee87d73cd2f8005fbbe87c93eee9c Mon Sep 17 00:00:00 2001 From: SomberNight Date: Sun, 29 Jul 2018 04:29:19 +0200 Subject: [PATCH 40/56] logging: self.print_error should not print without -v flag --- electrum/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/electrum/util.py b/electrum/util.py index 485b3458..7b67e991 100644 --- a/electrum/util.py +++ b/electrum/util.py @@ -171,7 +171,7 @@ class PrintError(object): def print_error(self, *msg): if self.verbosity_filter in verbosity or verbosity == '*': - print_stderr("[%s]" % self.diagnostic_name(), *msg) + print_error("[%s]" % self.diagnostic_name(), *msg) def print_stderr(self, *msg): print_stderr("[%s]" % self.diagnostic_name(), *msg) From 629b9cb3b54cd067da9673c8fb84d49e417903bf Mon Sep 17 00:00:00 2001 From: SomberNight Date: Mon, 30 Jul 2018 19:15:05 +0200 Subject: [PATCH 41/56] fee estimation: split eta_to_fee into two methods --- electrum/simple_config.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/electrum/simple_config.py b/electrum/simple_config.py index e4d75dae..25efbb8d 100644 --- a/electrum/simple_config.py +++ b/electrum/simple_config.py @@ -4,7 +4,7 @@ import time import os import stat from decimal import Decimal -from typing import Union +from typing import Union, Optional from numbers import Real from copy import deepcopy @@ -296,19 +296,27 @@ class SimpleConfig(PrintError): return fee return get_fee_within_limits - @impose_hard_limits_on_fee - def eta_to_fee(self, slider_pos) -> Union[int, None]: + def eta_to_fee(self, slider_pos) -> Optional[int]: """Returns fee in sat/kbyte.""" slider_pos = max(slider_pos, 0) slider_pos = min(slider_pos, len(FEE_ETA_TARGETS)) if slider_pos < len(FEE_ETA_TARGETS): - target_blocks = FEE_ETA_TARGETS[slider_pos] - fee = self.fee_estimates.get(target_blocks) + num_blocks = FEE_ETA_TARGETS[slider_pos] + fee = self.eta_target_to_fee(num_blocks) else: + fee = self.eta_target_to_fee(1) + return fee + + @impose_hard_limits_on_fee + def eta_target_to_fee(self, num_blocks: int) -> Optional[int]: + """Returns fee in sat/kbyte.""" + if num_blocks == 1: fee = self.fee_estimates.get(2) if fee is not None: - fee += fee/2 + fee += fee / 2 fee = int(fee) + else: + fee = self.fee_estimates.get(num_blocks) return fee def fee_to_depth(self, target_fee: Real) -> int: From e1b2195cf7c4a3194812612ebae0b85ed56938aa Mon Sep 17 00:00:00 2001 From: SomberNight Date: Tue, 31 Jul 2018 12:30:43 +0200 Subject: [PATCH 42/56] fix #4591: pay to OP_RETURN on trezor --- electrum/plugins/keepkey/keepkey.py | 2 +- electrum/plugins/safe_t/safe_t.py | 2 +- electrum/plugins/trezor/trezor.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/electrum/plugins/keepkey/keepkey.py b/electrum/plugins/keepkey/keepkey.py index 611fe844..39c0cefc 100644 --- a/electrum/plugins/keepkey/keepkey.py +++ b/electrum/plugins/keepkey/keepkey.py @@ -382,7 +382,7 @@ class KeepKeyPlugin(HW_PluginBase): txoutputtype.amount = amount if _type == TYPE_SCRIPT: txoutputtype.script_type = self.types.PAYTOOPRETURN - txoutputtype.op_return_data = address[2:] + txoutputtype.op_return_data = bfh(address)[2:] elif _type == TYPE_ADDRESS: if is_segwit_address(address): txoutputtype.script_type = self.types.PAYTOWITNESS diff --git a/electrum/plugins/safe_t/safe_t.py b/electrum/plugins/safe_t/safe_t.py index 4d4410de..6b9f8b98 100644 --- a/electrum/plugins/safe_t/safe_t.py +++ b/electrum/plugins/safe_t/safe_t.py @@ -453,7 +453,7 @@ class SafeTPlugin(HW_PluginBase): txoutputtype.amount = amount if _type == TYPE_SCRIPT: txoutputtype.script_type = self.types.OutputScriptType.PAYTOOPRETURN - txoutputtype.op_return_data = address[2:] + txoutputtype.op_return_data = bfh(address)[2:] elif _type == TYPE_ADDRESS: txoutputtype.script_type = self.types.OutputScriptType.PAYTOADDRESS txoutputtype.address = address diff --git a/electrum/plugins/trezor/trezor.py b/electrum/plugins/trezor/trezor.py index 7fecd1e6..7d14008a 100644 --- a/electrum/plugins/trezor/trezor.py +++ b/electrum/plugins/trezor/trezor.py @@ -464,7 +464,7 @@ class TrezorPlugin(HW_PluginBase): txoutputtype.amount = amount if _type == TYPE_SCRIPT: txoutputtype.script_type = self.types.OutputScriptType.PAYTOOPRETURN - txoutputtype.op_return_data = address[2:] + txoutputtype.op_return_data = bfh(address)[2:] elif _type == TYPE_ADDRESS: txoutputtype.script_type = self.types.OutputScriptType.PAYTOADDRESS txoutputtype.address = address From c9c8b7656d3ceb5d4d26ae443cb2ad5eaab064d0 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Tue, 31 Jul 2018 13:03:34 +0200 Subject: [PATCH 43/56] follow-up prev. sanity check OP_RETURN outputs based on https://github.com/fyookball/electrum/pull/765/commits/86c63a3a084951c87789346e1d9de35f4cede055 --- electrum/plugins/hw_wallet/plugin.py | 16 +++++++++++++++- electrum/plugins/keepkey/keepkey.py | 4 ++-- electrum/plugins/safe_t/safe_t.py | 4 ++-- electrum/plugins/trezor/trezor.py | 4 ++-- 4 files changed, 21 insertions(+), 7 deletions(-) diff --git a/electrum/plugins/hw_wallet/plugin.py b/electrum/plugins/hw_wallet/plugin.py index a7f1b76e..76616870 100644 --- a/electrum/plugins/hw_wallet/plugin.py +++ b/electrum/plugins/hw_wallet/plugin.py @@ -26,7 +26,9 @@ from electrum.plugin import BasePlugin, hook from electrum.i18n import _ -from electrum.bitcoin import is_address +from electrum.bitcoin import is_address, TYPE_SCRIPT +from electrum.util import bfh +from electrum.transaction import opcodes class HW_PluginBase(BasePlugin): @@ -87,3 +89,15 @@ def is_any_tx_output_on_change_branch(tx): if index[0] == 1: return True return False + + +def trezor_validate_op_return_output_and_get_data(_type, address, amount): + if _type != TYPE_SCRIPT: + raise Exception("Unexpected output type: {}".format(_type)) + script = bfh(address) + if not (script[0] == opcodes.OP_RETURN and + script[1] == len(script) - 2 and script[1] <= 75): + raise Exception(_("Only OP_RETURN scripts, with one constant push, are supported.")) + if amount != 0: + raise Exception(_("Amount for OP_RETURN output must be zero.")) + return script[2:] diff --git a/electrum/plugins/keepkey/keepkey.py b/electrum/plugins/keepkey/keepkey.py index 39c0cefc..6860ea83 100644 --- a/electrum/plugins/keepkey/keepkey.py +++ b/electrum/plugins/keepkey/keepkey.py @@ -15,7 +15,7 @@ from electrum.wallet import Standard_Wallet from electrum.base_wizard import ScriptTypeNotSupported from ..hw_wallet import HW_PluginBase -from ..hw_wallet.plugin import is_any_tx_output_on_change_branch +from ..hw_wallet.plugin import is_any_tx_output_on_change_branch, trezor_validate_op_return_output_and_get_data # TREZOR initialization methods @@ -382,7 +382,7 @@ class KeepKeyPlugin(HW_PluginBase): txoutputtype.amount = amount if _type == TYPE_SCRIPT: txoutputtype.script_type = self.types.PAYTOOPRETURN - txoutputtype.op_return_data = bfh(address)[2:] + txoutputtype.op_return_data = trezor_validate_op_return_output_and_get_data(_type, address, amount) elif _type == TYPE_ADDRESS: if is_segwit_address(address): txoutputtype.script_type = self.types.PAYTOWITNESS diff --git a/electrum/plugins/safe_t/safe_t.py b/electrum/plugins/safe_t/safe_t.py index 6b9f8b98..ca5d189a 100644 --- a/electrum/plugins/safe_t/safe_t.py +++ b/electrum/plugins/safe_t/safe_t.py @@ -13,7 +13,7 @@ from electrum.keystore import Hardware_KeyStore, is_xpubkey, parse_xpubkey, xtyp from electrum.base_wizard import ScriptTypeNotSupported from ..hw_wallet import HW_PluginBase -from ..hw_wallet.plugin import is_any_tx_output_on_change_branch +from ..hw_wallet.plugin import is_any_tx_output_on_change_branch, trezor_validate_op_return_output_and_get_data # Safe-T mini initialization methods @@ -453,7 +453,7 @@ class SafeTPlugin(HW_PluginBase): txoutputtype.amount = amount if _type == TYPE_SCRIPT: txoutputtype.script_type = self.types.OutputScriptType.PAYTOOPRETURN - txoutputtype.op_return_data = bfh(address)[2:] + txoutputtype.op_return_data = trezor_validate_op_return_output_and_get_data(_type, address, amount) elif _type == TYPE_ADDRESS: txoutputtype.script_type = self.types.OutputScriptType.PAYTOADDRESS txoutputtype.address = address diff --git a/electrum/plugins/trezor/trezor.py b/electrum/plugins/trezor/trezor.py index 7d14008a..233bc33e 100644 --- a/electrum/plugins/trezor/trezor.py +++ b/electrum/plugins/trezor/trezor.py @@ -13,7 +13,7 @@ from electrum.keystore import Hardware_KeyStore, is_xpubkey, parse_xpubkey, xtyp from electrum.base_wizard import ScriptTypeNotSupported from ..hw_wallet import HW_PluginBase -from ..hw_wallet.plugin import is_any_tx_output_on_change_branch +from ..hw_wallet.plugin import is_any_tx_output_on_change_branch, trezor_validate_op_return_output_and_get_data # TREZOR initialization methods @@ -464,7 +464,7 @@ class TrezorPlugin(HW_PluginBase): txoutputtype.amount = amount if _type == TYPE_SCRIPT: txoutputtype.script_type = self.types.OutputScriptType.PAYTOOPRETURN - txoutputtype.op_return_data = bfh(address)[2:] + txoutputtype.op_return_data = trezor_validate_op_return_output_and_get_data(_type, address, amount) elif _type == TYPE_ADDRESS: txoutputtype.script_type = self.types.OutputScriptType.PAYTOADDRESS txoutputtype.address = address From 861640949e479689aeb9d30121ba1f9ddb40ae71 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Tue, 31 Jul 2018 14:03:08 +0200 Subject: [PATCH 44/56] kivy: on tx broadcast, truncate error message related #4593 --- electrum/gui/kivy/main_window.py | 1 + 1 file changed, 1 insertion(+) diff --git a/electrum/gui/kivy/main_window.py b/electrum/gui/kivy/main_window.py index b55ee129..176776ce 100644 --- a/electrum/gui/kivy/main_window.py +++ b/electrum/gui/kivy/main_window.py @@ -886,6 +886,7 @@ class ElectrumWindow(App): self.wallet.invoices.save() self.update_tab('invoices') else: + msg = msg[:500] if msg else _('There was an error broadcasting the transaction.') self.show_error(msg) if self.network and self.network.is_connected(): From 41e088693de72d9111ce1ce21e42b5aa017627dd Mon Sep 17 00:00:00 2001 From: SomberNight Date: Tue, 31 Jul 2018 15:51:05 +0200 Subject: [PATCH 45/56] verifier: better handle reorgs (and storage upgrade) --- electrum/address_synchronizer.py | 55 ++++++++++++++++++++------------ electrum/gui/qt/history_list.py | 2 +- electrum/storage.py | 12 ++++++- electrum/verifier.py | 4 ++- electrum/wallet.py | 6 ++-- 5 files changed, 52 insertions(+), 27 deletions(-) diff --git a/electrum/address_synchronizer.py b/electrum/address_synchronizer.py index 58785040..6d194e0c 100644 --- a/electrum/address_synchronizer.py +++ b/electrum/address_synchronizer.py @@ -31,6 +31,7 @@ from .util import PrintError, profiler, bfh from .transaction import Transaction from .synchronizer import Synchronizer from .verifier import SPV +from .blockchain import hash_header from .i18n import _ TX_HEIGHT_LOCAL = -2 @@ -45,6 +46,7 @@ class UnrelatedTransactionException(AddTransactionException): def __str__(self): return _("Transaction is unrelated to this wallet.") + class AddressSynchronizer(PrintError): """ inherited by wallet @@ -61,7 +63,7 @@ class AddressSynchronizer(PrintError): self.transaction_lock = threading.RLock() # address -> list(txid, height) self.history = storage.get('addr_history',{}) - # Verified transactions. txid -> (height, timestamp, block_pos). Access with self.lock. + # Verified transactions. txid -> (height, timestamp, block_pos, block_hash). Access with self.lock. self.verified_tx = storage.get('verified_tx3', {}) # Transactions pending verification. txid -> tx_height. Access with self.lock. self.unverified_tx = defaultdict(int) @@ -434,7 +436,7 @@ class AddressSynchronizer(PrintError): "return position, even if the tx is unverified" with self.lock: if tx_hash in self.verified_tx: - height, timestamp, pos = self.verified_tx[tx_hash] + height, timestamp, pos, header_hash = self.verified_tx[tx_hash] return height, pos elif tx_hash in self.unverified_tx: height = self.unverified_tx[tx_hash] @@ -462,7 +464,7 @@ class AddressSynchronizer(PrintError): history = [] for tx_hash in tx_deltas: delta = tx_deltas[tx_hash] - height, conf, timestamp = self.get_tx_height(tx_hash) + height, conf, timestamp, header_hash = self.get_tx_height(tx_hash) history.append((tx_hash, height, conf, timestamp, delta)) history.sort(key = lambda x: self.get_txpos(x[0])) history.reverse() @@ -503,24 +505,26 @@ class AddressSynchronizer(PrintError): self._history_local[addr] = cur_hist def add_unverified_tx(self, tx_hash, tx_height): - if tx_height in (TX_HEIGHT_UNCONFIRMED, TX_HEIGHT_UNCONF_PARENT) \ - and tx_hash in self.verified_tx: + if tx_hash in self.verified_tx: + if tx_height in (TX_HEIGHT_UNCONFIRMED, TX_HEIGHT_UNCONF_PARENT): + with self.lock: + self.verified_tx.pop(tx_hash) + if self.verifier: + self.verifier.remove_spv_proof_for_tx(tx_hash) + else: with self.lock: - self.verified_tx.pop(tx_hash) + # tx will be verified only if height > 0 + self.unverified_tx[tx_hash] = tx_height + # to remove pending proof requests: if self.verifier: self.verifier.remove_spv_proof_for_tx(tx_hash) - # tx will be verified only if height > 0 - if tx_hash not in self.verified_tx: - with self.lock: - self.unverified_tx[tx_hash] = tx_height - def add_verified_tx(self, tx_hash, info): # Remove from the unverified map and add to the verified map with self.lock: self.unverified_tx.pop(tx_hash, None) - self.verified_tx[tx_hash] = info # (tx_height, timestamp, pos) - height, conf, timestamp = self.get_tx_height(tx_hash) + self.verified_tx[tx_hash] = info # (tx_height, timestamp, pos, header_hash) + height, conf, timestamp, header_hash = self.get_tx_height(tx_hash) self.network.trigger_callback('verified', tx_hash, height, conf, timestamp) def get_unverified_txs(self): @@ -533,12 +537,21 @@ class AddressSynchronizer(PrintError): txs = set() with self.lock: for tx_hash, item in list(self.verified_tx.items()): - tx_height, timestamp, pos = item + tx_height, timestamp, pos, header_hash = item if tx_height >= height: header = blockchain.read_header(tx_height) - # fixme: use block hash, not timestamp - if not header or header.get('timestamp') != timestamp: + if not header or hash_header(header) != header_hash: self.verified_tx.pop(tx_hash, None) + # NOTE: we should add these txns to self.unverified_tx, + # but with what height? + # If on the new fork after the reorg, the txn is at the + # same height, we will not get a status update for the + # address. If the txn is not mined or at a diff height, + # we should get a status update. Unless we put tx into + # unverified_tx, it will turn into local. So we put it + # into unverified_tx with the old height, and if we get + # a status update, that will overwrite it. + self.unverified_tx[tx_hash] = tx_height txs.add(tx_hash) return txs @@ -547,18 +560,18 @@ class AddressSynchronizer(PrintError): return self.network.get_local_height() if self.network else self.storage.get('stored_height', 0) def get_tx_height(self, tx_hash): - """ Given a transaction, returns (height, conf, timestamp) """ + """ Given a transaction, returns (height, conf, timestamp, header_hash) """ with self.lock: if tx_hash in self.verified_tx: - height, timestamp, pos = self.verified_tx[tx_hash] + height, timestamp, pos, header_hash = self.verified_tx[tx_hash] conf = max(self.get_local_height() - height + 1, 0) - return height, conf, timestamp + return height, conf, timestamp, header_hash elif tx_hash in self.unverified_tx: height = self.unverified_tx[tx_hash] - return height, 0, None + return height, 0, None, None else: # local transaction - return TX_HEIGHT_LOCAL, 0, None + return TX_HEIGHT_LOCAL, 0, None, None def set_up_to_date(self, up_to_date): with self.lock: diff --git a/electrum/gui/qt/history_list.py b/electrum/gui/qt/history_list.py index 8f1a5ab3..602792a8 100644 --- a/electrum/gui/qt/history_list.py +++ b/electrum/gui/qt/history_list.py @@ -332,7 +332,7 @@ class HistoryList(MyTreeWidget, AcceptFileDragDrop): column_title = self.headerItem().text(column) column_data = item.text(column) tx_URL = block_explorer_URL(self.config, 'tx', tx_hash) - height, conf, timestamp = self.wallet.get_tx_height(tx_hash) + height, conf, timestamp, header_hash = self.wallet.get_tx_height(tx_hash) tx = self.wallet.transactions.get(tx_hash) is_relevant, is_mine, v, fee = self.wallet.get_wallet_delta(tx) is_unconfirmed = height <= 0 diff --git a/electrum/storage.py b/electrum/storage.py index 40c13289..156df968 100644 --- a/electrum/storage.py +++ b/electrum/storage.py @@ -44,7 +44,7 @@ from .keystore import bip44_derivation OLD_SEED_VERSION = 4 # electrum versions < 2.0 NEW_SEED_VERSION = 11 # electrum versions >= 2.0 -FINAL_SEED_VERSION = 17 # electrum >= 2.7 will set this to prevent +FINAL_SEED_VERSION = 18 # electrum >= 2.7 will set this to prevent # old versions from overwriting new format @@ -356,6 +356,7 @@ class WalletStorage(JsonDB): self.convert_version_15() self.convert_version_16() self.convert_version_17() + self.convert_version_18() self.put('seed_version', FINAL_SEED_VERSION) # just to be sure self.write() @@ -570,6 +571,15 @@ class WalletStorage(JsonDB): self.put('seed_version', 17) + def convert_version_18(self): + # delete verified_tx3 as its structure changed + if not self._is_upgrade_method_needed(17, 17): + return + + self.put('verified_tx3', None) + + self.put('seed_version', 18) + def convert_imported(self): if not self._is_upgrade_method_needed(0, 13): return diff --git a/electrum/verifier.py b/electrum/verifier.py index 82ad678b..f53f8d1e 100644 --- a/electrum/verifier.py +++ b/electrum/verifier.py @@ -26,6 +26,7 @@ from typing import Sequence, Optional from .util import ThreadJob, bh2u from .bitcoin import Hash, hash_decode, hash_encode from .transaction import Transaction +from .blockchain import hash_header class MerkleVerificationFailure(Exception): pass @@ -108,7 +109,8 @@ class SPV(ThreadJob): self.requested_merkle.remove(tx_hash) except KeyError: pass self.print_error("verified %s" % tx_hash) - self.wallet.add_verified_tx(tx_hash, (tx_height, header.get('timestamp'), pos)) + header_hash = hash_header(header) + self.wallet.add_verified_tx(tx_hash, (tx_height, header.get('timestamp'), pos, header_hash)) if self.is_up_to_date() and self.wallet.is_up_to_date(): self.wallet.save_verified_tx(write=True) diff --git a/electrum/wallet.py b/electrum/wallet.py index 3f45e9ed..430610f6 100644 --- a/electrum/wallet.py +++ b/electrum/wallet.py @@ -318,7 +318,7 @@ class Abstract_Wallet(AddressSynchronizer): if tx.is_complete(): if tx_hash in self.transactions.keys(): label = self.get_label(tx_hash) - height, conf, timestamp = self.get_tx_height(tx_hash) + height, conf, timestamp, header_hash = self.get_tx_height(tx_hash) if height > 0: if conf: status = _("{} confirmations").format(conf) @@ -839,7 +839,7 @@ class Abstract_Wallet(AddressSynchronizer): txid, n = txo.split(':') info = self.verified_tx.get(txid) if info: - tx_height, timestamp, pos = info + tx_height, timestamp, pos, header_hash = info conf = local_height - tx_height else: conf = 0 @@ -1091,7 +1091,7 @@ class Abstract_Wallet(AddressSynchronizer): def price_at_timestamp(self, txid, price_func): """Returns fiat price of bitcoin at the time tx got confirmed.""" - height, conf, timestamp = self.get_tx_height(txid) + height, conf, timestamp, header_hash = self.get_tx_height(txid) return price_func(timestamp if timestamp else time.time()) def unrealized_gains(self, domain, price_func, ccy): From a29e2218c8a01c859a57ba132e8d18cdaad10f8c Mon Sep 17 00:00:00 2001 From: SomberNight Date: Tue, 31 Jul 2018 16:49:57 +0200 Subject: [PATCH 46/56] wallet: introduce namedtuples TxMinedStatus and VerifiedTxInfo --- electrum/address_synchronizer.py | 59 +++++++++++++++++--------------- electrum/gui/kivy/uix/screens.py | 7 ++-- electrum/gui/qt/history_list.py | 12 ++++--- electrum/gui/stdio.py | 6 ++-- electrum/gui/text.py | 6 ++-- electrum/util.py | 11 ++++++ electrum/verifier.py | 5 +-- electrum/wallet.py | 31 ++++++++++------- 8 files changed, 79 insertions(+), 58 deletions(-) diff --git a/electrum/address_synchronizer.py b/electrum/address_synchronizer.py index 6d194e0c..62c6b623 100644 --- a/electrum/address_synchronizer.py +++ b/electrum/address_synchronizer.py @@ -27,7 +27,7 @@ from collections import defaultdict from . import bitcoin from .bitcoin import COINBASE_MATURITY, TYPE_ADDRESS, TYPE_PUBKEY -from .util import PrintError, profiler, bfh +from .util import PrintError, profiler, bfh, VerifiedTxInfo, TxMinedStatus from .transaction import Transaction from .synchronizer import Synchronizer from .verifier import SPV @@ -63,8 +63,11 @@ class AddressSynchronizer(PrintError): self.transaction_lock = threading.RLock() # address -> list(txid, height) self.history = storage.get('addr_history',{}) - # Verified transactions. txid -> (height, timestamp, block_pos, block_hash). Access with self.lock. - self.verified_tx = storage.get('verified_tx3', {}) + # Verified transactions. txid -> VerifiedTxInfo. Access with self.lock. + verified_tx = storage.get('verified_tx3', {}) + self.verified_tx = {} + for txid, (height, timestamp, txpos, header_hash) in verified_tx.items(): + self.verified_tx[txid] = VerifiedTxInfo(height, timestamp, txpos, header_hash) # Transactions pending verification. txid -> tx_height. Access with self.lock. self.unverified_tx = defaultdict(int) # true when synchronized @@ -92,7 +95,7 @@ class AddressSynchronizer(PrintError): with self.lock, self.transaction_lock: related_txns = self._history_local.get(addr, set()) for tx_hash in related_txns: - tx_height = self.get_tx_height(tx_hash)[0] + tx_height = self.get_tx_height(tx_hash).height h.append((tx_hash, tx_height)) return h @@ -195,7 +198,7 @@ class AddressSynchronizer(PrintError): # of add_transaction tx, we might learn of more-and-more inputs of # being is_mine, as we roll the gap_limit forward is_coinbase = tx.inputs()[0]['type'] == 'coinbase' - tx_height = self.get_tx_height(tx_hash)[0] + tx_height = self.get_tx_height(tx_hash).height if not allow_unrelated: # note that during sync, if the transactions are not properly sorted, # it could happen that we think tx is unrelated but actually one of the inputs is is_mine. @@ -214,10 +217,10 @@ class AddressSynchronizer(PrintError): conflicting_txns = self.get_conflicting_transactions(tx) if conflicting_txns: existing_mempool_txn = any( - self.get_tx_height(tx_hash2)[0] in (TX_HEIGHT_UNCONFIRMED, TX_HEIGHT_UNCONF_PARENT) + self.get_tx_height(tx_hash2).height in (TX_HEIGHT_UNCONFIRMED, TX_HEIGHT_UNCONF_PARENT) for tx_hash2 in conflicting_txns) existing_confirmed_txn = any( - self.get_tx_height(tx_hash2)[0] > 0 + self.get_tx_height(tx_hash2).height > 0 for tx_hash2 in conflicting_txns) if existing_confirmed_txn and tx_height <= 0: # this is a non-confirmed tx that conflicts with confirmed txns; drop. @@ -395,7 +398,7 @@ class AddressSynchronizer(PrintError): def remove_local_transactions_we_dont_have(self): txid_set = set(self.txi) | set(self.txo) for txid in txid_set: - tx_height = self.get_tx_height(txid)[0] + tx_height = self.get_tx_height(txid).height if tx_height == TX_HEIGHT_LOCAL and txid not in self.transactions: self.remove_transaction(txid) @@ -433,11 +436,11 @@ class AddressSynchronizer(PrintError): self.save_transactions() def get_txpos(self, tx_hash): - "return position, even if the tx is unverified" + """Returns (height, txpos) tuple, even if the tx is unverified.""" with self.lock: if tx_hash in self.verified_tx: - height, timestamp, pos, header_hash = self.verified_tx[tx_hash] - return height, pos + info = self.verified_tx[tx_hash] + return info.height, info.txpos elif tx_hash in self.unverified_tx: height = self.unverified_tx[tx_hash] return (height, 0) if height > 0 else ((1e9 - height), 0) @@ -464,16 +467,16 @@ class AddressSynchronizer(PrintError): history = [] for tx_hash in tx_deltas: delta = tx_deltas[tx_hash] - height, conf, timestamp, header_hash = self.get_tx_height(tx_hash) - history.append((tx_hash, height, conf, timestamp, delta)) + tx_mined_status = self.get_tx_height(tx_hash) + history.append((tx_hash, tx_mined_status, delta)) history.sort(key = lambda x: self.get_txpos(x[0])) history.reverse() # 3. add balance c, u, x = self.get_balance(domain) balance = c + u + x h2 = [] - for tx_hash, height, conf, timestamp, delta in history: - h2.append((tx_hash, height, conf, timestamp, delta, balance)) + for tx_hash, tx_mined_status, delta in history: + h2.append((tx_hash, tx_mined_status, delta, balance)) if balance is None or delta is None: balance = None else: @@ -519,13 +522,13 @@ class AddressSynchronizer(PrintError): if self.verifier: self.verifier.remove_spv_proof_for_tx(tx_hash) - def add_verified_tx(self, tx_hash, info): + def add_verified_tx(self, tx_hash: str, info: VerifiedTxInfo): # Remove from the unverified map and add to the verified map with self.lock: self.unverified_tx.pop(tx_hash, None) - self.verified_tx[tx_hash] = info # (tx_height, timestamp, pos, header_hash) - height, conf, timestamp, header_hash = self.get_tx_height(tx_hash) - self.network.trigger_callback('verified', tx_hash, height, conf, timestamp) + self.verified_tx[tx_hash] = info + tx_mined_status = self.get_tx_height(tx_hash) + self.network.trigger_callback('verified', tx_hash, tx_mined_status) def get_unverified_txs(self): '''Returns a map from tx hash to transaction height''' @@ -536,11 +539,11 @@ class AddressSynchronizer(PrintError): '''Used by the verifier when a reorg has happened''' txs = set() with self.lock: - for tx_hash, item in list(self.verified_tx.items()): - tx_height, timestamp, pos, header_hash = item + for tx_hash, info in list(self.verified_tx.items()): + tx_height = info.height if tx_height >= height: header = blockchain.read_header(tx_height) - if not header or hash_header(header) != header_hash: + if not header or hash_header(header) != info.header_hash: self.verified_tx.pop(tx_hash, None) # NOTE: we should add these txns to self.unverified_tx, # but with what height? @@ -559,19 +562,19 @@ class AddressSynchronizer(PrintError): """ return last known height if we are offline """ return self.network.get_local_height() if self.network else self.storage.get('stored_height', 0) - def get_tx_height(self, tx_hash): + def get_tx_height(self, tx_hash: str) -> TxMinedStatus: """ Given a transaction, returns (height, conf, timestamp, header_hash) """ with self.lock: if tx_hash in self.verified_tx: - height, timestamp, pos, header_hash = self.verified_tx[tx_hash] - conf = max(self.get_local_height() - height + 1, 0) - return height, conf, timestamp, header_hash + info = self.verified_tx[tx_hash] + conf = max(self.get_local_height() - info.height + 1, 0) + return TxMinedStatus(info.height, conf, info.timestamp, info.header_hash) elif tx_hash in self.unverified_tx: height = self.unverified_tx[tx_hash] - return height, 0, None, None + return TxMinedStatus(height, 0, None, None) else: # local transaction - return TX_HEIGHT_LOCAL, 0, None, None + return TxMinedStatus(TX_HEIGHT_LOCAL, 0, None, None) def set_up_to_date(self, up_to_date): with self.lock: diff --git a/electrum/gui/kivy/uix/screens.py b/electrum/gui/kivy/uix/screens.py index 94807913..474661ea 100644 --- a/electrum/gui/kivy/uix/screens.py +++ b/electrum/gui/kivy/uix/screens.py @@ -131,8 +131,8 @@ class HistoryScreen(CScreen): d = LabelDialog(_('Enter Transaction Label'), text, callback) d.open() - def get_card(self, tx_hash, height, conf, timestamp, value, balance): - status, status_str = self.app.wallet.get_tx_status(tx_hash, height, conf, timestamp) + def get_card(self, tx_hash, tx_mined_status, value, balance): + status, status_str = self.app.wallet.get_tx_status(tx_hash, tx_mined_status) icon = "atlas://electrum/gui/kivy/theming/light/" + TX_ICONS[status] label = self.app.wallet.get_label(tx_hash) if tx_hash else _('Pruned transaction outputs') ri = {} @@ -141,7 +141,7 @@ class HistoryScreen(CScreen): ri['icon'] = icon ri['date'] = status_str ri['message'] = label - ri['confirmations'] = conf + ri['confirmations'] = tx_mined_status.conf if value is not None: ri['is_mine'] = value < 0 if value < 0: value = - value @@ -158,7 +158,6 @@ class HistoryScreen(CScreen): return history = reversed(self.app.wallet.get_history()) history_card = self.screen.ids.history_container - count = 0 history_card.data = [self.get_card(*item) for item in history] diff --git a/electrum/gui/qt/history_list.py b/electrum/gui/qt/history_list.py index 602792a8..b1d11a46 100644 --- a/electrum/gui/qt/history_list.py +++ b/electrum/gui/qt/history_list.py @@ -29,7 +29,7 @@ import datetime from electrum.address_synchronizer import TX_HEIGHT_LOCAL from .util import * from electrum.i18n import _ -from electrum.util import block_explorer_URL, profiler, print_error +from electrum.util import block_explorer_URL, profiler, print_error, TxMinedStatus try: from electrum.plot import plot_history, NothingToPlotException @@ -237,7 +237,8 @@ class HistoryList(MyTreeWidget, AcceptFileDragDrop): value = tx_item['value'].value balance = tx_item['balance'].value label = tx_item['label'] - status, status_str = self.wallet.get_tx_status(tx_hash, height, conf, timestamp) + tx_mined_status = TxMinedStatus(height, conf, timestamp, None) + status, status_str = self.wallet.get_tx_status(tx_hash, tx_mined_status) has_invoice = self.wallet.invoices.paid.get(tx_hash) icon = self.icon_cache.get(":icons/" + TX_ICONS[status]) v_str = self.parent.format_amount(value, is_diff=True, whitespaces=True) @@ -304,10 +305,11 @@ class HistoryList(MyTreeWidget, AcceptFileDragDrop): label = self.wallet.get_label(txid) item.setText(3, label) - def update_item(self, tx_hash, height, conf, timestamp): + def update_item(self, tx_hash, tx_mined_status): if self.wallet is None: return - status, status_str = self.wallet.get_tx_status(tx_hash, height, conf, timestamp) + conf = tx_mined_status.conf + status, status_str = self.wallet.get_tx_status(tx_hash, tx_mined_status) icon = self.icon_cache.get(":icons/" + TX_ICONS[status]) items = self.findItems(tx_hash, Qt.UserRole|Qt.MatchContains|Qt.MatchRecursive, column=1) if items: @@ -332,7 +334,7 @@ class HistoryList(MyTreeWidget, AcceptFileDragDrop): column_title = self.headerItem().text(column) column_data = item.text(column) tx_URL = block_explorer_URL(self.config, 'tx', tx_hash) - height, conf, timestamp, header_hash = self.wallet.get_tx_height(tx_hash) + height = self.wallet.get_tx_height(tx_hash).height tx = self.wallet.transactions.get(tx_hash) is_relevant, is_mine, v, fee = self.wallet.get_wallet_delta(tx) is_unconfirmed = height <= 0 diff --git a/electrum/gui/stdio.py b/electrum/gui/stdio.py index a4756eae..bc9a8118 100644 --- a/electrum/gui/stdio.py +++ b/electrum/gui/stdio.py @@ -87,9 +87,9 @@ class ElectrumGui: + "%d"%(width[2]+delta)+"s"+"%"+"%d"%(width[3]+delta)+"s" messages = [] - for item in self.wallet.get_history(): - tx_hash, height, conf, timestamp, delta, balance = item - if conf: + for tx_hash, tx_mined_status, delta, balance in self.wallet.get_history(): + if tx_mined_status.conf: + timestamp = tx_mined_status.timestamp try: time_str = datetime.datetime.fromtimestamp(timestamp).isoformat(' ')[:-3] except Exception: diff --git a/electrum/gui/text.py b/electrum/gui/text.py index ee051563..07652ecc 100644 --- a/electrum/gui/text.py +++ b/electrum/gui/text.py @@ -109,9 +109,9 @@ class ElectrumGui: b = 0 self.history = [] - for item in self.wallet.get_history(): - tx_hash, height, conf, timestamp, value, balance = item - if conf: + for tx_hash, tx_mined_status, value, balance in self.wallet.get_history(): + if tx_mined_status.conf: + timestamp = tx_mined_status.timestamp try: time_str = datetime.datetime.fromtimestamp(timestamp).isoformat(' ')[:-3] except Exception: diff --git a/electrum/util.py b/electrum/util.py index 7b67e991..c6471be6 100644 --- a/electrum/util.py +++ b/electrum/util.py @@ -23,6 +23,7 @@ import binascii import os, sys, re, json from collections import defaultdict +from typing import NamedTuple from datetime import datetime import decimal from decimal import Decimal @@ -903,3 +904,13 @@ def make_dir(path, allow_symlink=True): raise Exception('Dangling link: ' + path) os.mkdir(path) os.chmod(path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) + + +TxMinedStatus = NamedTuple("TxMinedStatus", [("height", int), + ("conf", int), + ("timestamp", int), + ("header_hash", str)]) +VerifiedTxInfo = NamedTuple("VerifiedTxInfo", [("height", int), + ("timestamp", int), + ("txpos", int), + ("header_hash", str)]) diff --git a/electrum/verifier.py b/electrum/verifier.py index f53f8d1e..c85cbc3a 100644 --- a/electrum/verifier.py +++ b/electrum/verifier.py @@ -23,7 +23,7 @@ from typing import Sequence, Optional -from .util import ThreadJob, bh2u +from .util import ThreadJob, bh2u, VerifiedTxInfo from .bitcoin import Hash, hash_decode, hash_encode from .transaction import Transaction from .blockchain import hash_header @@ -110,7 +110,8 @@ class SPV(ThreadJob): except KeyError: pass self.print_error("verified %s" % tx_hash) header_hash = hash_header(header) - self.wallet.add_verified_tx(tx_hash, (tx_height, header.get('timestamp'), pos, header_hash)) + vtx_info = VerifiedTxInfo(tx_height, header.get('timestamp'), pos, header_hash) + self.wallet.add_verified_tx(tx_hash, vtx_info) if self.is_up_to_date() and self.wallet.is_up_to_date(): self.wallet.save_verified_tx(write=True) diff --git a/electrum/wallet.py b/electrum/wallet.py index 430610f6..5410d4ec 100644 --- a/electrum/wallet.py +++ b/electrum/wallet.py @@ -318,7 +318,8 @@ class Abstract_Wallet(AddressSynchronizer): if tx.is_complete(): if tx_hash in self.transactions.keys(): label = self.get_label(tx_hash) - height, conf, timestamp, header_hash = self.get_tx_height(tx_hash) + tx_mined_status = self.get_tx_height(tx_hash) + height, conf = tx_mined_status.height, tx_mined_status.conf if height > 0: if conf: status = _("{} confirmations").format(conf) @@ -368,8 +369,9 @@ class Abstract_Wallet(AddressSynchronizer): def balance_at_timestamp(self, domain, target_timestamp): h = self.get_history(domain) - for tx_hash, height, conf, timestamp, value, balance in h: - if timestamp > target_timestamp: + balance = 0 + for tx_hash, tx_mined_status, value, balance in h: + if tx_mined_status.timestamp > target_timestamp: return balance - value # return last balance return balance @@ -384,16 +386,17 @@ class Abstract_Wallet(AddressSynchronizer): fiat_income = Decimal(0) fiat_expenditures = Decimal(0) h = self.get_history(domain) - for tx_hash, height, conf, timestamp, value, balance in h: + for tx_hash, tx_mined_status, value, balance in h: + timestamp = tx_mined_status.timestamp if from_timestamp and (timestamp or time.time()) < from_timestamp: continue if to_timestamp and (timestamp or time.time()) >= to_timestamp: continue item = { - 'txid':tx_hash, - 'height':height, - 'confirmations':conf, - 'timestamp':timestamp, + 'txid': tx_hash, + 'height': tx_mined_status.height, + 'confirmations': tx_mined_status.conf, + 'timestamp': timestamp, 'value': Satoshis(value), 'balance': Satoshis(balance) } @@ -483,9 +486,12 @@ class Abstract_Wallet(AddressSynchronizer): return ', '.join(labels) return '' - def get_tx_status(self, tx_hash, height, conf, timestamp): + def get_tx_status(self, tx_hash, tx_mined_status): from .util import format_time extra = [] + height = tx_mined_status.height + conf = tx_mined_status.conf + timestamp = tx_mined_status.timestamp if conf == 0: tx = self.transactions.get(tx_hash) if not tx: @@ -839,8 +845,7 @@ class Abstract_Wallet(AddressSynchronizer): txid, n = txo.split(':') info = self.verified_tx.get(txid) if info: - tx_height, timestamp, pos, header_hash = info - conf = local_height - tx_height + conf = local_height - info.height else: conf = 0 l.append((conf, v)) @@ -1091,7 +1096,7 @@ class Abstract_Wallet(AddressSynchronizer): def price_at_timestamp(self, txid, price_func): """Returns fiat price of bitcoin at the time tx got confirmed.""" - height, conf, timestamp, header_hash = self.get_tx_height(txid) + timestamp = self.get_tx_height(txid).timestamp return price_func(timestamp if timestamp else time.time()) def unrealized_gains(self, domain, price_func, ccy): @@ -1258,7 +1263,7 @@ class Imported_Wallet(Simple_Wallet): self.verified_tx.pop(tx_hash, None) self.unverified_tx.pop(tx_hash, None) self.transactions.pop(tx_hash, None) - self.storage.put('verified_tx3', self.verified_tx) + self.save_verified_tx() self.save_transactions() self.set_label(address, None) From f64062b6f13dfc340fedcdfdf24156ae21414c1c Mon Sep 17 00:00:00 2001 From: SomberNight Date: Tue, 31 Jul 2018 20:25:53 +0200 Subject: [PATCH 47/56] add --noonion option to filter out onion servers closes #4531 --- electrum/commands.py | 1 + electrum/network.py | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/electrum/commands.py b/electrum/commands.py index 62d2ed6f..8d6f4ac5 100644 --- a/electrum/commands.py +++ b/electrum/commands.py @@ -826,6 +826,7 @@ def add_network_options(parser): parser.add_argument("-1", "--oneserver", action="store_true", dest="oneserver", default=None, help="connect to one server only") parser.add_argument("-s", "--server", dest="server", default=None, help="set server host:port:protocol, where protocol is either t (tcp) or s (ssl)") parser.add_argument("-p", "--proxy", dest="proxy", default=None, help="set proxy [type:]host[:port], where type is socks4,socks5 or http") + parser.add_argument("--noonion", action="store_true", dest="noonion", default=None, help="do not try to connect to onion servers") def add_global_options(parser): group = parser.add_argument_group('global options') diff --git a/electrum/network.py b/electrum/network.py index 07bad8dc..d31569d6 100644 --- a/electrum/network.py +++ b/electrum/network.py @@ -89,6 +89,10 @@ def filter_version(servers): return {k: v for k, v in servers.items() if is_recent(v.get('version'))} +def filter_noonion(servers): + return {k: v for k, v in servers.items() if not k.endswith('.onion')} + + def filter_protocol(hostmap, protocol='s'): '''Filters the hostmap for those implementing protocol. The result is a list in serialized form.''' @@ -409,6 +413,8 @@ class Network(util.DaemonThread): continue if host not in out: out[host] = {protocol: port} + if self.config.get('noonion'): + out = filter_noonion(out) return out @with_interface_lock From 2eb72d496f847aa7d2ff4aaba0a5db20cf92072e Mon Sep 17 00:00:00 2001 From: SomberNight Date: Tue, 31 Jul 2018 18:24:37 +0200 Subject: [PATCH 48/56] transaction: introduce TxOutput namedtuple --- electrum/address_synchronizer.py | 13 +++--- electrum/coinchooser.py | 4 +- electrum/commands.py | 6 +-- electrum/gui/kivy/main_window.py | 3 +- electrum/gui/kivy/uix/dialogs/__init__.py | 6 +-- electrum/gui/kivy/uix/screens.py | 3 +- electrum/gui/qt/main_window.py | 12 ++--- electrum/gui/qt/paytoedit.py | 13 +++--- electrum/gui/stdio.py | 4 +- electrum/gui/text.py | 4 +- electrum/paymentrequest.py | 7 +-- .../plugins/digitalbitbox/digitalbitbox.py | 6 +-- electrum/plugins/hw_wallet/plugin.py | 12 ++--- electrum/plugins/keepkey/keepkey.py | 5 ++- electrum/plugins/ledger/ledger.py | 10 ++--- electrum/plugins/safe_t/safe_t.py | 5 ++- electrum/plugins/trezor/trezor.py | 5 ++- electrum/plugins/trustedcoin/trustedcoin.py | 9 ++-- electrum/tests/test_wallet_vertical.py | 45 ++++++++++--------- electrum/transaction.py | 26 ++++++----- electrum/wallet.py | 33 +++++++------- 21 files changed, 122 insertions(+), 109 deletions(-) diff --git a/electrum/address_synchronizer.py b/electrum/address_synchronizer.py index 62c6b623..7b729f9e 100644 --- a/electrum/address_synchronizer.py +++ b/electrum/address_synchronizer.py @@ -28,7 +28,7 @@ from collections import defaultdict from . import bitcoin from .bitcoin import COINBASE_MATURITY, TYPE_ADDRESS, TYPE_PUBKEY from .util import PrintError, profiler, bfh, VerifiedTxInfo, TxMinedStatus -from .transaction import Transaction +from .transaction import Transaction, TxOutput from .synchronizer import Synchronizer from .verifier import SPV from .blockchain import hash_header @@ -112,12 +112,11 @@ class AddressSynchronizer(PrintError): return addr return None - def get_txout_address(self, txo): - _type, x, v = txo - if _type == TYPE_ADDRESS: - addr = x - elif _type == TYPE_PUBKEY: - addr = bitcoin.public_key_to_p2pkh(bfh(x)) + def get_txout_address(self, txo: TxOutput): + if txo.type == TYPE_ADDRESS: + addr = txo.address + elif txo.type == TYPE_PUBKEY: + addr = bitcoin.public_key_to_p2pkh(bfh(txo.address)) else: addr = None return addr diff --git a/electrum/coinchooser.py b/electrum/coinchooser.py index 0b8128b9..7dc355d5 100644 --- a/electrum/coinchooser.py +++ b/electrum/coinchooser.py @@ -26,7 +26,7 @@ from collections import defaultdict, namedtuple from math import floor, log10 from .bitcoin import sha256, COIN, TYPE_ADDRESS, is_address -from .transaction import Transaction +from .transaction import Transaction, TxOutput from .util import NotEnoughFunds, PrintError @@ -178,7 +178,7 @@ class CoinChooserBase(PrintError): # size of the change output, add it to the transaction. dust = sum(amount for amount in amounts if amount < dust_threshold) amounts = [amount for amount in amounts if amount >= dust_threshold] - change = [(TYPE_ADDRESS, addr, amount) + change = [TxOutput(TYPE_ADDRESS, addr, amount) for addr, amount in zip(change_addrs, amounts)] self.print_error('change:', change) if dust: diff --git a/electrum/commands.py b/electrum/commands.py index 8d6f4ac5..2240f57c 100644 --- a/electrum/commands.py +++ b/electrum/commands.py @@ -38,7 +38,7 @@ from .util import bfh, bh2u, format_satoshis, json_decode, print_error, json_enc from . import bitcoin from .bitcoin import is_address, hash_160, COIN, TYPE_ADDRESS from .i18n import _ -from .transaction import Transaction, multisig_script +from .transaction import Transaction, multisig_script, TxOutput from .paymentrequest import PR_PAID, PR_UNPAID, PR_UNKNOWN, PR_EXPIRED from .plugin import run_hook @@ -226,7 +226,7 @@ class Commands: txin['signatures'] = [None] txin['num_sig'] = 1 - outputs = [(TYPE_ADDRESS, x['address'], int(x['value'])) for x in outputs] + outputs = [TxOutput(TYPE_ADDRESS, x['address'], int(x['value'])) for x in outputs] tx = Transaction.from_io(inputs, outputs, locktime=locktime) tx.sign(keypairs) return tx.as_dict() @@ -415,7 +415,7 @@ class Commands: for address, amount in outputs: address = self._resolver(address) amount = satoshis(amount) - final_outputs.append((TYPE_ADDRESS, address, amount)) + final_outputs.append(TxOutput(TYPE_ADDRESS, address, amount)) coins = self.wallet.get_spendable_coins(domain, self.config) tx = self.wallet.make_unsigned_transaction(coins, final_outputs, self.config, fee, change_addr) diff --git a/electrum/gui/kivy/main_window.py b/electrum/gui/kivy/main_window.py index 176776ce..d0db73a2 100644 --- a/electrum/gui/kivy/main_window.py +++ b/electrum/gui/kivy/main_window.py @@ -713,13 +713,14 @@ class ElectrumWindow(App): self.fiat_balance = self.fx.format_amount(c+u+x) + ' [size=22dp]%s[/size]'% self.fx.ccy def get_max_amount(self): + from electrum.transaction import TxOutput if run_hook('abort_send', self): return '' inputs = self.wallet.get_spendable_coins(None, self.electrum_config) if not inputs: return '' addr = str(self.send_screen.screen.address) or self.wallet.dummy_address() - outputs = [(TYPE_ADDRESS, addr, '!')] + outputs = [TxOutput(TYPE_ADDRESS, addr, '!')] try: tx = self.wallet.make_unsigned_transaction(inputs, outputs, self.electrum_config) except NoDynamicFeeEstimates as e: diff --git a/electrum/gui/kivy/uix/dialogs/__init__.py b/electrum/gui/kivy/uix/dialogs/__init__.py index 9f8f7f2e..352675a3 100644 --- a/electrum/gui/kivy/uix/dialogs/__init__.py +++ b/electrum/gui/kivy/uix/dialogs/__init__.py @@ -206,9 +206,9 @@ class OutputList(RecycleView): def update(self, outputs): res = [] - for (type, address, amount) in outputs: - value = self.app.format_amount_and_units(amount) - res.append({'address': address, 'value': value}) + for o in outputs: + value = self.app.format_amount_and_units(o.value) + res.append({'address': o.address, 'value': value}) self.data = res diff --git a/electrum/gui/kivy/uix/screens.py b/electrum/gui/kivy/uix/screens.py index 474661ea..73930aef 100644 --- a/electrum/gui/kivy/uix/screens.py +++ b/electrum/gui/kivy/uix/screens.py @@ -20,6 +20,7 @@ from kivy.utils import platform from electrum.util import profiler, parse_URI, format_time, InvalidPassword, NotEnoughFunds, Fiat from electrum import bitcoin +from electrum.transaction import TxOutput from electrum.util import timestamp_to_datetime from electrum.paymentrequest import PR_UNPAID, PR_PAID, PR_UNKNOWN, PR_EXPIRED from electrum.plugin import run_hook @@ -256,7 +257,7 @@ class SendScreen(CScreen): except: self.app.show_error(_('Invalid amount') + ':\n' + self.screen.amount) return - outputs = [(bitcoin.TYPE_ADDRESS, address, amount)] + outputs = [TxOutput(bitcoin.TYPE_ADDRESS, address, amount)] message = self.screen.message amount = sum(map(lambda x:x[2], outputs)) if self.app.electrum_config.get('use_rbf'): diff --git a/electrum/gui/qt/main_window.py b/electrum/gui/qt/main_window.py index 1cdae70d..75667603 100644 --- a/electrum/gui/qt/main_window.py +++ b/electrum/gui/qt/main_window.py @@ -50,7 +50,7 @@ from electrum.util import (format_time, format_satoshis, format_fee_satoshis, export_meta, import_meta, bh2u, bfh, InvalidPassword, base_units, base_units_list, base_unit_name_to_decimal_point, decimal_point_to_base_unit_name, quantize_feerate) -from electrum.transaction import Transaction +from electrum.transaction import Transaction, TxOutput from electrum.address_synchronizer import AddTransactionException from electrum.wallet import Multisig_Wallet, CannotBumpFee @@ -1306,7 +1306,7 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError): outputs = self.payto_e.get_outputs(self.is_max) if not outputs: _type, addr = self.get_payto_or_dummy() - outputs = [(_type, addr, amount)] + outputs = [TxOutput(_type, addr, amount)] is_sweep = bool(self.tx_external_keypairs) make_tx = lambda fee_est: \ self.wallet.make_unsigned_transaction( @@ -1485,14 +1485,14 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError): self.show_error(_('No outputs')) return - for _type, addr, amount in outputs: - if addr is None: + for o in outputs: + if o.address is None: self.show_error(_('Bitcoin Address is None')) return - if _type == TYPE_ADDRESS and not bitcoin.is_address(addr): + if o.type == TYPE_ADDRESS and not bitcoin.is_address(o.address): self.show_error(_('Invalid Bitcoin Address')) return - if amount is None: + if o.value is None: self.show_error(_('Invalid Amount')) return diff --git a/electrum/gui/qt/paytoedit.py b/electrum/gui/qt/paytoedit.py index e33b90e4..376303b9 100644 --- a/electrum/gui/qt/paytoedit.py +++ b/electrum/gui/qt/paytoedit.py @@ -29,6 +29,7 @@ from decimal import Decimal from electrum import bitcoin from electrum.util import bfh +from electrum.transaction import TxOutput from .qrtextedit import ScanQRTextEdit from .completion_text_edit import CompletionTextEdit @@ -77,7 +78,7 @@ class PayToEdit(CompletionTextEdit, ScanQRTextEdit): x, y = line.split(',') out_type, out = self.parse_output(x) amount = self.parse_amount(y) - return out_type, out, amount + return TxOutput(out_type, out, amount) def parse_output(self, x): try: @@ -139,16 +140,16 @@ class PayToEdit(CompletionTextEdit, ScanQRTextEdit): is_max = False for i, line in enumerate(lines): try: - _type, to_address, amount = self.parse_address_and_amount(line) + output = self.parse_address_and_amount(line) except: self.errors.append((i, line.strip())) continue - outputs.append((_type, to_address, amount)) - if amount == '!': + outputs.append(output) + if output.value == '!': is_max = True else: - total += amount + total += output.value self.win.is_max = is_max self.outputs = outputs @@ -174,7 +175,7 @@ class PayToEdit(CompletionTextEdit, ScanQRTextEdit): amount = self.amount_edit.get_amount() _type, addr = self.payto_address - self.outputs = [(_type, addr, amount)] + self.outputs = [TxOutput(_type, addr, amount)] return self.outputs[:] diff --git a/electrum/gui/stdio.py b/electrum/gui/stdio.py index bc9a8118..d155efb0 100644 --- a/electrum/gui/stdio.py +++ b/electrum/gui/stdio.py @@ -4,6 +4,7 @@ _ = lambda x:x from electrum import WalletStorage, Wallet from electrum.util import format_satoshis, set_verbosity from electrum.bitcoin import is_address, COIN, TYPE_ADDRESS +from electrum.transaction import TxOutput import getpass, datetime # minimal fdisk like gui for console usage @@ -189,7 +190,8 @@ class ElectrumGui: if c == "n": return try: - tx = self.wallet.mktx([(TYPE_ADDRESS, self.str_recipient, amount)], password, self.config, fee) + tx = self.wallet.mktx([TxOutput(TYPE_ADDRESS, self.str_recipient, amount)], + password, self.config, fee) except Exception as e: print(str(e)) return diff --git a/electrum/gui/text.py b/electrum/gui/text.py index 07652ecc..61e3ac1d 100644 --- a/electrum/gui/text.py +++ b/electrum/gui/text.py @@ -6,6 +6,7 @@ import getpass import electrum from electrum.util import format_satoshis, set_verbosity from electrum.bitcoin import is_address, COIN, TYPE_ADDRESS +from electrum.transaction import TxOutput from .. import Wallet, WalletStorage _ = lambda x:x @@ -340,7 +341,8 @@ class ElectrumGui: else: password = None try: - tx = self.wallet.mktx([(TYPE_ADDRESS, self.str_recipient, amount)], password, self.config, fee) + tx = self.wallet.mktx([TxOutput(TYPE_ADDRESS, self.str_recipient, amount)], + password, self.config, fee) except Exception as e: self.show_message(str(e)) return diff --git a/electrum/paymentrequest.py b/electrum/paymentrequest.py index 03ffe602..7010b93f 100644 --- a/electrum/paymentrequest.py +++ b/electrum/paymentrequest.py @@ -42,6 +42,7 @@ from .util import print_error, bh2u, bfh from .util import export_meta, import_meta from .bitcoin import TYPE_ADDRESS +from .transaction import TxOutput REQUEST_HEADERS = {'Accept': 'application/bitcoin-paymentrequest', 'User-Agent': 'Electrum'} ACK_HEADERS = {'Content-Type':'application/bitcoin-payment','Accept':'application/bitcoin-paymentack','User-Agent':'Electrum'} @@ -123,7 +124,7 @@ class PaymentRequest: self.outputs = [] for o in self.details.outputs: addr = transaction.get_address_from_output_script(o.script)[1] - self.outputs.append((TYPE_ADDRESS, addr, o.amount)) + self.outputs.append(TxOutput(TYPE_ADDRESS, addr, o.amount)) self.memo = self.details.memo self.payment_url = self.details.payment_url @@ -225,8 +226,8 @@ class PaymentRequest: def get_address(self): o = self.outputs[0] - assert o[0] == TYPE_ADDRESS - return o[1] + assert o.type == TYPE_ADDRESS + return o.address def get_requestor(self): return self.requestor if self.requestor else self.get_address() diff --git a/electrum/plugins/digitalbitbox/digitalbitbox.py b/electrum/plugins/digitalbitbox/digitalbitbox.py index 37db2644..441d33b2 100644 --- a/electrum/plugins/digitalbitbox/digitalbitbox.py +++ b/electrum/plugins/digitalbitbox/digitalbitbox.py @@ -534,9 +534,9 @@ class DigitalBitbox_KeyStore(Hardware_KeyStore): self.give_error("No matching x_key for sign_transaction") # should never happen # Build pubkeyarray from outputs - for _type, address, amount in tx.outputs(): - assert _type == TYPE_ADDRESS - info = tx.output_info.get(address) + for o in tx.outputs(): + assert o.type == TYPE_ADDRESS + info = tx.output_info.get(o.address) if info is not None: index, xpubs, m = info changePath = self.get_derivation() + "/%d/%d" % index diff --git a/electrum/plugins/hw_wallet/plugin.py b/electrum/plugins/hw_wallet/plugin.py index 76616870..9d2e32ac 100644 --- a/electrum/plugins/hw_wallet/plugin.py +++ b/electrum/plugins/hw_wallet/plugin.py @@ -28,7 +28,7 @@ from electrum.plugin import BasePlugin, hook from electrum.i18n import _ from electrum.bitcoin import is_address, TYPE_SCRIPT from electrum.util import bfh -from electrum.transaction import opcodes +from electrum.transaction import opcodes, TxOutput class HW_PluginBase(BasePlugin): @@ -91,13 +91,13 @@ def is_any_tx_output_on_change_branch(tx): return False -def trezor_validate_op_return_output_and_get_data(_type, address, amount): - if _type != TYPE_SCRIPT: - raise Exception("Unexpected output type: {}".format(_type)) - script = bfh(address) +def trezor_validate_op_return_output_and_get_data(output: TxOutput) -> bytes: + if output.type != TYPE_SCRIPT: + raise Exception("Unexpected output type: {}".format(output.type)) + script = bfh(output.address) if not (script[0] == opcodes.OP_RETURN and script[1] == len(script) - 2 and script[1] <= 75): raise Exception(_("Only OP_RETURN scripts, with one constant push, are supported.")) - if amount != 0: + if output.value != 0: raise Exception(_("Amount for OP_RETURN output must be zero.")) return script[2:] diff --git a/electrum/plugins/keepkey/keepkey.py b/electrum/plugins/keepkey/keepkey.py index 6860ea83..66a208a4 100644 --- a/electrum/plugins/keepkey/keepkey.py +++ b/electrum/plugins/keepkey/keepkey.py @@ -382,7 +382,7 @@ class KeepKeyPlugin(HW_PluginBase): txoutputtype.amount = amount if _type == TYPE_SCRIPT: txoutputtype.script_type = self.types.PAYTOOPRETURN - txoutputtype.op_return_data = trezor_validate_op_return_output_and_get_data(_type, address, amount) + txoutputtype.op_return_data = trezor_validate_op_return_output_and_get_data(o) elif _type == TYPE_ADDRESS: if is_segwit_address(address): txoutputtype.script_type = self.types.PAYTOWITNESS @@ -401,7 +401,8 @@ class KeepKeyPlugin(HW_PluginBase): has_change = False any_output_on_change_branch = is_any_tx_output_on_change_branch(tx) - for _type, address, amount in tx.outputs(): + for o in tx.outputs(): + _type, address, amount = o.type, o.address, o.value use_create_by_derivation = False info = tx.output_info.get(address) diff --git a/electrum/plugins/ledger/ledger.py b/electrum/plugins/ledger/ledger.py index bf7e6d2f..939a7a5a 100644 --- a/electrum/plugins/ledger/ledger.py +++ b/electrum/plugins/ledger/ledger.py @@ -394,9 +394,9 @@ class Ledger_KeyStore(Hardware_KeyStore): self.give_error("Transaction with more than 2 outputs not supported") has_change = False any_output_on_change_branch = is_any_tx_output_on_change_branch(tx) - for _type, address, amount in tx.outputs(): - assert _type == TYPE_ADDRESS - info = tx.output_info.get(address) + for o in tx.outputs(): + assert o.type == TYPE_ADDRESS + info = tx.output_info.get(o.address) if (info is not None) and len(tx.outputs()) > 1 \ and not has_change: index, xpubs, m = info @@ -407,9 +407,9 @@ class Ledger_KeyStore(Hardware_KeyStore): changePath = self.get_derivation()[2:] + "/%d/%d"%index has_change = True else: - output = address + output = o.address else: - output = address + output = o.address self.handler.show_message(_("Confirm Transaction on your Ledger device...")) try: diff --git a/electrum/plugins/safe_t/safe_t.py b/electrum/plugins/safe_t/safe_t.py index ca5d189a..22008977 100644 --- a/electrum/plugins/safe_t/safe_t.py +++ b/electrum/plugins/safe_t/safe_t.py @@ -453,7 +453,7 @@ class SafeTPlugin(HW_PluginBase): txoutputtype.amount = amount if _type == TYPE_SCRIPT: txoutputtype.script_type = self.types.OutputScriptType.PAYTOOPRETURN - txoutputtype.op_return_data = trezor_validate_op_return_output_and_get_data(_type, address, amount) + txoutputtype.op_return_data = trezor_validate_op_return_output_and_get_data(o) elif _type == TYPE_ADDRESS: txoutputtype.script_type = self.types.OutputScriptType.PAYTOADDRESS txoutputtype.address = address @@ -463,7 +463,8 @@ class SafeTPlugin(HW_PluginBase): has_change = False any_output_on_change_branch = is_any_tx_output_on_change_branch(tx) - for _type, address, amount in tx.outputs(): + for o in tx.outputs(): + _type, address, amount = o.type, o.address, o.value use_create_by_derivation = False info = tx.output_info.get(address) diff --git a/electrum/plugins/trezor/trezor.py b/electrum/plugins/trezor/trezor.py index 233bc33e..e9893824 100644 --- a/electrum/plugins/trezor/trezor.py +++ b/electrum/plugins/trezor/trezor.py @@ -464,7 +464,7 @@ class TrezorPlugin(HW_PluginBase): txoutputtype.amount = amount if _type == TYPE_SCRIPT: txoutputtype.script_type = self.types.OutputScriptType.PAYTOOPRETURN - txoutputtype.op_return_data = trezor_validate_op_return_output_and_get_data(_type, address, amount) + txoutputtype.op_return_data = trezor_validate_op_return_output_and_get_data(o) elif _type == TYPE_ADDRESS: txoutputtype.script_type = self.types.OutputScriptType.PAYTOADDRESS txoutputtype.address = address @@ -474,7 +474,8 @@ class TrezorPlugin(HW_PluginBase): has_change = False any_output_on_change_branch = is_any_tx_output_on_change_branch(tx) - for _type, address, amount in tx.outputs(): + for o in tx.outputs(): + _type, address, amount = o.type, o.address, o.value use_create_by_derivation = False info = tx.output_info.get(address) diff --git a/electrum/plugins/trustedcoin/trustedcoin.py b/electrum/plugins/trustedcoin/trustedcoin.py index 9877da97..ccd24e33 100644 --- a/electrum/plugins/trustedcoin/trustedcoin.py +++ b/electrum/plugins/trustedcoin/trustedcoin.py @@ -33,6 +33,7 @@ from urllib.parse import quote from electrum import bitcoin, ecc, constants, keystore, version from electrum.bitcoin import * +from electrum.transaction import TxOutput from electrum.mnemonic import Mnemonic from electrum.wallet import Multisig_Wallet, Deterministic_Wallet from electrum.i18n import _ @@ -273,7 +274,7 @@ class Wallet_2fa(Multisig_Wallet): fee = self.extra_fee(config) if not is_sweep else 0 if fee: address = self.billing_info['billing_address'] - fee_output = (TYPE_ADDRESS, address, fee) + fee_output = TxOutput(TYPE_ADDRESS, address, fee) try: tx = mk_tx(outputs + [fee_output]) except NotEnoughFunds: @@ -395,9 +396,9 @@ class TrustedCoinPlugin(BasePlugin): def get_tx_extra_fee(self, wallet, tx): if type(wallet) != Wallet_2fa: return - for _type, addr, amount in tx.outputs(): - if _type == TYPE_ADDRESS and wallet.is_billing_address(addr): - return addr, amount + for o in tx.outputs(): + if o.type == TYPE_ADDRESS and wallet.is_billing_address(o.address): + return o.address, o.value def finish_requesting(func): def f(self, *args, **kwargs): diff --git a/electrum/tests/test_wallet_vertical.py b/electrum/tests/test_wallet_vertical.py index 1b03983a..1a38c922 100644 --- a/electrum/tests/test_wallet_vertical.py +++ b/electrum/tests/test_wallet_vertical.py @@ -10,6 +10,7 @@ from electrum import SimpleConfig from electrum.address_synchronizer import TX_HEIGHT_UNCONFIRMED, TX_HEIGHT_UNCONF_PARENT from electrum.wallet import sweep, Multisig_Wallet, Standard_Wallet, Imported_Wallet from electrum.util import bfh, bh2u +from electrum.transaction import TxOutput from electrum.plugins.trustedcoin import trustedcoin @@ -532,7 +533,7 @@ class TestWalletSending(TestCaseForTestnet): wallet1.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED) # wallet1 -> wallet2 - outputs = [(bitcoin.TYPE_ADDRESS, wallet2.get_receiving_address(), 250000)] + outputs = [TxOutput(bitcoin.TYPE_ADDRESS, wallet2.get_receiving_address(), 250000)] tx = wallet1.mktx(outputs=outputs, password=None, config=self.config, fee=5000) self.assertTrue(tx.is_complete()) @@ -552,7 +553,7 @@ class TestWalletSending(TestCaseForTestnet): wallet2.receive_tx_callback(tx.txid(), tx, TX_HEIGHT_UNCONFIRMED) # wallet2 -> wallet1 - outputs = [(bitcoin.TYPE_ADDRESS, wallet1.get_receiving_address(), 100000)] + outputs = [TxOutput(bitcoin.TYPE_ADDRESS, wallet1.get_receiving_address(), 100000)] tx = wallet2.mktx(outputs=outputs, password=None, config=self.config, fee=5000) self.assertTrue(tx.is_complete()) @@ -605,7 +606,7 @@ class TestWalletSending(TestCaseForTestnet): wallet1a.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED) # wallet1 -> wallet2 - outputs = [(bitcoin.TYPE_ADDRESS, wallet2.get_receiving_address(), 370000)] + outputs = [TxOutput(bitcoin.TYPE_ADDRESS, wallet2.get_receiving_address(), 370000)] tx = wallet1a.mktx(outputs=outputs, password=None, config=self.config, fee=5000) tx = Transaction(tx.serialize()) # simulates moving partial txn between cosigners self.assertFalse(tx.is_complete()) @@ -628,7 +629,7 @@ class TestWalletSending(TestCaseForTestnet): wallet2.receive_tx_callback(tx.txid(), tx, TX_HEIGHT_UNCONFIRMED) # wallet2 -> wallet1 - outputs = [(bitcoin.TYPE_ADDRESS, wallet1a.get_receiving_address(), 100000)] + outputs = [TxOutput(bitcoin.TYPE_ADDRESS, wallet1a.get_receiving_address(), 100000)] tx = wallet2.mktx(outputs=outputs, password=None, config=self.config, fee=5000) self.assertTrue(tx.is_complete()) @@ -696,7 +697,7 @@ class TestWalletSending(TestCaseForTestnet): wallet1a.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED) # wallet1 -> wallet2 - outputs = [(bitcoin.TYPE_ADDRESS, wallet2a.get_receiving_address(), 165000)] + outputs = [TxOutput(bitcoin.TYPE_ADDRESS, wallet2a.get_receiving_address(), 165000)] tx = wallet1a.mktx(outputs=outputs, password=None, config=self.config, fee=5000) txid = tx.txid() tx = Transaction(tx.serialize()) # simulates moving partial txn between cosigners @@ -722,7 +723,7 @@ class TestWalletSending(TestCaseForTestnet): wallet2a.receive_tx_callback(tx.txid(), tx, TX_HEIGHT_UNCONFIRMED) # wallet2 -> wallet1 - outputs = [(bitcoin.TYPE_ADDRESS, wallet1a.get_receiving_address(), 100000)] + outputs = [TxOutput(bitcoin.TYPE_ADDRESS, wallet1a.get_receiving_address(), 100000)] tx = wallet2a.mktx(outputs=outputs, password=None, config=self.config, fee=5000) txid = tx.txid() tx = Transaction(tx.serialize()) # simulates moving partial txn between cosigners @@ -776,7 +777,7 @@ class TestWalletSending(TestCaseForTestnet): wallet1a.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED) # wallet1 -> wallet2 - outputs = [(bitcoin.TYPE_ADDRESS, wallet2.get_receiving_address(), 1000000)] + outputs = [TxOutput(bitcoin.TYPE_ADDRESS, wallet2.get_receiving_address(), 1000000)] tx = wallet1a.mktx(outputs=outputs, password=None, config=self.config, fee=5000) self.assertTrue(tx.is_complete()) @@ -796,7 +797,7 @@ class TestWalletSending(TestCaseForTestnet): wallet2.receive_tx_callback(tx.txid(), tx, TX_HEIGHT_UNCONFIRMED) # wallet2 -> wallet1 - outputs = [(bitcoin.TYPE_ADDRESS, wallet1a.get_receiving_address(), 300000)] + outputs = [TxOutput(bitcoin.TYPE_ADDRESS, wallet1a.get_receiving_address(), 300000)] tx = wallet2.mktx(outputs=outputs, password=None, config=self.config, fee=5000) self.assertTrue(tx.is_complete()) @@ -832,7 +833,7 @@ class TestWalletSending(TestCaseForTestnet): wallet.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED) # create tx - outputs = [(bitcoin.TYPE_ADDRESS, '2N1VTMMFb91SH9SNRAkT7z8otP5eZEct4KL', 2500000)] + outputs = [TxOutput(bitcoin.TYPE_ADDRESS, '2N1VTMMFb91SH9SNRAkT7z8otP5eZEct4KL', 2500000)] coins = wallet.get_spendable_coins(domain=None, config=self.config) tx = wallet.make_unsigned_transaction(coins, outputs, config=self.config, fixed_fee=5000) tx.set_rbf(True) @@ -918,7 +919,7 @@ class TestWalletSending(TestCaseForTestnet): wallet.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED) # create tx - outputs = [(bitcoin.TYPE_ADDRESS, '2N1VTMMFb91SH9SNRAkT7z8otP5eZEct4KL', 2500000)] + outputs = [TxOutput(bitcoin.TYPE_ADDRESS, '2N1VTMMFb91SH9SNRAkT7z8otP5eZEct4KL', 2500000)] coins = wallet.get_spendable_coins(domain=None, config=self.config) tx = wallet.make_unsigned_transaction(coins, outputs, config=self.config, fixed_fee=5000) tx.set_rbf(True) @@ -1048,7 +1049,7 @@ class TestWalletOfflineSigning(TestCaseForTestnet): wallet_online.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED) # create unsigned tx - outputs = [(bitcoin.TYPE_ADDRESS, 'tb1qp0mv2sxsyxxfj5gl0332f9uyez93su9cf26757', 2500000)] + outputs = [TxOutput(bitcoin.TYPE_ADDRESS, 'tb1qp0mv2sxsyxxfj5gl0332f9uyez93su9cf26757', 2500000)] tx = wallet_online.mktx(outputs=outputs, password=None, config=self.config, fee=5000) tx.set_rbf(True) tx.locktime = 1325340 @@ -1088,7 +1089,7 @@ class TestWalletOfflineSigning(TestCaseForTestnet): wallet_online.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED) # create unsigned tx - outputs = [(bitcoin.TYPE_ADDRESS, 'tb1qp0mv2sxsyxxfj5gl0332f9uyez93su9cf26757', 2500000)] + outputs = [TxOutput(bitcoin.TYPE_ADDRESS, 'tb1qp0mv2sxsyxxfj5gl0332f9uyez93su9cf26757', 2500000)] tx = wallet_online.mktx(outputs=outputs, password=None, config=self.config, fee=5000) tx.set_rbf(True) tx.locktime = 1325341 @@ -1129,7 +1130,7 @@ class TestWalletOfflineSigning(TestCaseForTestnet): wallet_online.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED) # create unsigned tx - outputs = [(bitcoin.TYPE_ADDRESS, 'tb1qp0mv2sxsyxxfj5gl0332f9uyez93su9cf26757', 2500000)] + outputs = [TxOutput(bitcoin.TYPE_ADDRESS, 'tb1qp0mv2sxsyxxfj5gl0332f9uyez93su9cf26757', 2500000)] tx = wallet_online.mktx(outputs=outputs, password=None, config=self.config, fee=5000) tx.set_rbf(True) tx.locktime = 1325341 @@ -1165,7 +1166,7 @@ class TestWalletOfflineSigning(TestCaseForTestnet): wallet_online.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED) # create unsigned tx - outputs = [(bitcoin.TYPE_ADDRESS, 'tb1quk7ahlhr3qmjndy0uvu9y9hxfesrtahtta9ghm', 2500000)] + outputs = [TxOutput(bitcoin.TYPE_ADDRESS, 'tb1quk7ahlhr3qmjndy0uvu9y9hxfesrtahtta9ghm', 2500000)] tx = wallet_online.mktx(outputs=outputs, password=None, config=self.config, fee=5000) tx.set_rbf(True) tx.locktime = 1325340 @@ -1199,7 +1200,7 @@ class TestWalletOfflineSigning(TestCaseForTestnet): wallet_online.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED) # create unsigned tx - outputs = [(bitcoin.TYPE_ADDRESS, 'tb1quk7ahlhr3qmjndy0uvu9y9hxfesrtahtta9ghm', 2500000)] + outputs = [TxOutput(bitcoin.TYPE_ADDRESS, 'tb1quk7ahlhr3qmjndy0uvu9y9hxfesrtahtta9ghm', 2500000)] tx = wallet_online.mktx(outputs=outputs, password=None, config=self.config, fee=5000) tx.set_rbf(True) tx.locktime = 1325340 @@ -1233,7 +1234,7 @@ class TestWalletOfflineSigning(TestCaseForTestnet): wallet_online.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED) # create unsigned tx - outputs = [(bitcoin.TYPE_ADDRESS, 'tb1quk7ahlhr3qmjndy0uvu9y9hxfesrtahtta9ghm', 2500000)] + outputs = [TxOutput(bitcoin.TYPE_ADDRESS, 'tb1quk7ahlhr3qmjndy0uvu9y9hxfesrtahtta9ghm', 2500000)] tx = wallet_online.mktx(outputs=outputs, password=None, config=self.config, fee=5000) tx.set_rbf(True) tx.locktime = 1325340 @@ -1270,7 +1271,7 @@ class TestWalletOfflineSigning(TestCaseForTestnet): wallet_online.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED) # create unsigned tx - outputs = [(bitcoin.TYPE_ADDRESS, 'tb1quk7ahlhr3qmjndy0uvu9y9hxfesrtahtta9ghm', 2500000)] + outputs = [TxOutput(bitcoin.TYPE_ADDRESS, 'tb1quk7ahlhr3qmjndy0uvu9y9hxfesrtahtta9ghm', 2500000)] tx = wallet_online.mktx(outputs=outputs, password=None, config=self.config, fee=5000) tx.set_rbf(True) tx.locktime = 1325340 @@ -1307,7 +1308,7 @@ class TestWalletOfflineSigning(TestCaseForTestnet): wallet_online.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED) # create unsigned tx - outputs = [(bitcoin.TYPE_ADDRESS, 'tb1quk7ahlhr3qmjndy0uvu9y9hxfesrtahtta9ghm', 2500000)] + outputs = [TxOutput(bitcoin.TYPE_ADDRESS, 'tb1quk7ahlhr3qmjndy0uvu9y9hxfesrtahtta9ghm', 2500000)] tx = wallet_online.mktx(outputs=outputs, password=None, config=self.config, fee=5000) tx.set_rbf(True) tx.locktime = 1325340 @@ -1344,7 +1345,7 @@ class TestWalletOfflineSigning(TestCaseForTestnet): wallet_online.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED) # create unsigned tx - outputs = [(bitcoin.TYPE_ADDRESS, 'tb1quk7ahlhr3qmjndy0uvu9y9hxfesrtahtta9ghm', 2500000)] + outputs = [TxOutput(bitcoin.TYPE_ADDRESS, 'tb1quk7ahlhr3qmjndy0uvu9y9hxfesrtahtta9ghm', 2500000)] tx = wallet_online.mktx(outputs=outputs, password=None, config=self.config, fee=5000) tx.set_rbf(True) tx.locktime = 1325340 @@ -1393,7 +1394,7 @@ class TestWalletOfflineSigning(TestCaseForTestnet): wallet_online.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED) # create unsigned tx - outputs = [(bitcoin.TYPE_ADDRESS, '2MuCQQHJNnrXzQzuqfUCfAwAjPqpyEHbgue', 2500000)] + outputs = [TxOutput(bitcoin.TYPE_ADDRESS, '2MuCQQHJNnrXzQzuqfUCfAwAjPqpyEHbgue', 2500000)] tx = wallet_online.mktx(outputs=outputs, password=None, config=self.config, fee=5000) tx.set_rbf(True) tx.locktime = 1325503 @@ -1450,7 +1451,7 @@ class TestWalletOfflineSigning(TestCaseForTestnet): wallet_online.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED) # create unsigned tx - outputs = [(bitcoin.TYPE_ADDRESS, '2N8CtJRwxb2GCaiWWdSHLZHHLoZy53CCyxf', 2500000)] + outputs = [TxOutput(bitcoin.TYPE_ADDRESS, '2N8CtJRwxb2GCaiWWdSHLZHHLoZy53CCyxf', 2500000)] tx = wallet_online.mktx(outputs=outputs, password=None, config=self.config, fee=5000) tx.set_rbf(True) tx.locktime = 1325504 @@ -1509,7 +1510,7 @@ class TestWalletOfflineSigning(TestCaseForTestnet): wallet_online.receive_tx_callback(funding_txid, funding_tx, TX_HEIGHT_UNCONFIRMED) # create unsigned tx - outputs = [(bitcoin.TYPE_ADDRESS, '2MyoZVy8T1t94yLmyKu8DP1SmbWvnxbkwRA', 2500000)] + outputs = [TxOutput(bitcoin.TYPE_ADDRESS, '2MyoZVy8T1t94yLmyKu8DP1SmbWvnxbkwRA', 2500000)] tx = wallet_online.mktx(outputs=outputs, password=None, config=self.config, fee=5000) tx.set_rbf(True) tx.locktime = 1325505 diff --git a/electrum/transaction.py b/electrum/transaction.py index e1a2f87e..8fcce24c 100644 --- a/electrum/transaction.py +++ b/electrum/transaction.py @@ -27,7 +27,7 @@ # Note: The deserialization code originally comes from ABE. -from typing import Sequence, Union +from typing import Sequence, Union, NamedTuple from .util import print_error, profiler @@ -59,6 +59,10 @@ class NotRecognizedRedeemScript(Exception): pass +TxOutput = NamedTuple("TxOutput", [('type', int), ('address', str), ('value', Union[int, str])]) +# ^ value is str when the output is set to max: '!' + + class BCDataStream(object): def __init__(self): self.input = None @@ -721,7 +725,7 @@ class Transaction: return d = deserialize(self.raw, force_full_parse) self._inputs = d['inputs'] - self._outputs = [(x['type'], x['address'], x['value']) for x in d['outputs']] + self._outputs = [TxOutput(x['type'], x['address'], x['value']) for x in d['outputs']] self.locktime = d['lockTime'] self.version = d['version'] self.is_partial_originally = d['partial'] @@ -1180,17 +1184,17 @@ class Transaction: def get_outputs(self): """convert pubkeys to addresses""" - o = [] - for type, x, v in self.outputs(): - if type == TYPE_ADDRESS: - addr = x - elif type == TYPE_PUBKEY: + outputs = [] + for o in self.outputs(): + if o.type == TYPE_ADDRESS: + addr = o.address + elif o.type == TYPE_PUBKEY: # TODO do we really want this conversion? it's not really that address after all - addr = bitcoin.public_key_to_p2pkh(bfh(x)) + addr = bitcoin.public_key_to_p2pkh(bfh(o.address)) else: - addr = 'SCRIPT ' + x - o.append((addr,v)) # consider using yield (addr, v) - return o + addr = 'SCRIPT ' + o.address + outputs.append((addr, o.value)) # consider using yield (addr, v) + return outputs def get_output_addresses(self): return [addr for addr, val in self.get_outputs()] diff --git a/electrum/wallet.py b/electrum/wallet.py index 5410d4ec..3572473c 100644 --- a/electrum/wallet.py +++ b/electrum/wallet.py @@ -51,7 +51,7 @@ from .keystore import load_keystore, Hardware_KeyStore from .storage import multisig_type, STO_EV_PLAINTEXT, STO_EV_USER_PW, STO_EV_XPUB_PW from . import transaction, bitcoin, coinchooser, paymentrequest, contacts -from .transaction import Transaction +from .transaction import Transaction, TxOutput from .plugin import run_hook from .address_synchronizer import (AddressSynchronizer, TX_HEIGHT_LOCAL, TX_HEIGHT_UNCONF_PARENT, TX_HEIGHT_UNCONFIRMED) @@ -133,7 +133,7 @@ def sweep(privkeys, network, config, recipient, fee=None, imax=100): inputs, keypairs = sweep_preparations(privkeys, network, imax) total = sum(i.get('value') for i in inputs) if fee is None: - outputs = [(TYPE_ADDRESS, recipient, total)] + outputs = [TxOutput(TYPE_ADDRESS, recipient, total)] tx = Transaction.from_io(inputs, outputs) fee = config.estimate_fee(tx.estimated_size()) if total - fee < 0: @@ -141,7 +141,7 @@ def sweep(privkeys, network, config, recipient, fee=None, imax=100): if total - fee < dust_threshold(network): raise Exception(_('Not enough funds on address.') + '\nTotal: %d satoshis\nFee: %d\nDust Threshold: %d'%(total, fee, dust_threshold(network))) - outputs = [(TYPE_ADDRESS, recipient, total - fee)] + outputs = [TxOutput(TYPE_ADDRESS, recipient, total - fee)] locktime = network.get_local_height() tx = Transaction.from_io(inputs, outputs, locktime=locktime) @@ -538,11 +538,10 @@ class Abstract_Wallet(AddressSynchronizer): # check outputs i_max = None for i, o in enumerate(outputs): - _type, data, value = o - if _type == TYPE_ADDRESS: - if not is_address(data): - raise Exception("Invalid bitcoin address: {}".format(data)) - if value == '!': + if o.type == TYPE_ADDRESS: + if not is_address(o.address): + raise Exception("Invalid bitcoin address: {}".format(o.address)) + if o.value == '!': if i_max is not None: raise Exception("More than one output set to spend max") i_max = i @@ -593,14 +592,13 @@ class Abstract_Wallet(AddressSynchronizer): else: # FIXME?? this might spend inputs with negative effective value... sendable = sum(map(lambda x:x['value'], inputs)) - _type, data, value = outputs[i_max] - outputs[i_max] = (_type, data, 0) + outputs[i_max] = outputs[i_max]._replace(value=0) tx = Transaction.from_io(inputs, outputs[:]) fee = fee_estimator(tx.estimated_size()) amount = sendable - tx.output_value() - fee if amount < 0: raise NotEnoughFunds() - outputs[i_max] = (_type, data, amount) + outputs[i_max] = outputs[i_max]._replace(value=amount) tx = Transaction.from_io(inputs, outputs[:]) # Sort the inputs and outputs deterministically @@ -694,14 +692,13 @@ class Abstract_Wallet(AddressSynchronizer): s = sorted(s, key=lambda x: x[2]) for o in s: i = outputs.index(o) - otype, address, value = o - if value - delta >= self.dust_threshold(): - outputs[i] = otype, address, value - delta + if o.value - delta >= self.dust_threshold(): + outputs[i] = o._replace(value=o.value-delta) delta = 0 break else: del outputs[i] - delta -= value + delta -= o.value if delta > 0: continue if delta > 0: @@ -714,8 +711,8 @@ class Abstract_Wallet(AddressSynchronizer): def cpfp(self, tx, fee): txid = tx.txid() for i, o in enumerate(tx.outputs()): - otype, address, value = o - if otype == TYPE_ADDRESS and self.is_mine(address): + address, value = o.address, o.value + if o.type == TYPE_ADDRESS and self.is_mine(address): break else: return @@ -725,7 +722,7 @@ class Abstract_Wallet(AddressSynchronizer): return self.add_input_info(item) inputs = [item] - outputs = [(TYPE_ADDRESS, address, value - fee)] + outputs = [TxOutput(TYPE_ADDRESS, address, value - fee)] locktime = self.get_local_height() # note: no need to call tx.BIP_LI01_sort() here - single input/output return Transaction.from_io(inputs, outputs, locktime=locktime) From 6192bfce463fbd05e3ccdc851aab24a994a7258c Mon Sep 17 00:00:00 2001 From: SomberNight Date: Thu, 2 Aug 2018 15:38:01 +0200 Subject: [PATCH 49/56] util.profiler: prepend class name to prints --- electrum/util.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/electrum/util.py b/electrum/util.py index c6471be6..23c6d69c 100644 --- a/electrum/util.py +++ b/electrum/util.py @@ -32,6 +32,7 @@ import urllib import threading import hmac import stat +import inspect from .i18n import _ @@ -310,14 +311,24 @@ def constant_time_compare(val1, val2): # decorator that prints execution time def profiler(func): - def do_profile(func, args, kw_args): - n = func.__name__ + def get_func_name(args): + arg_names_from_sig = inspect.getfullargspec(func).args + # prepend class name if there is one (and if we can find it) + if len(arg_names_from_sig) > 0 and len(args) > 0 \ + and arg_names_from_sig[0] in ('self', 'cls', 'klass'): + classname = args[0].__class__.__name__ + else: + classname = '' + name = '{}.{}'.format(classname, func.__name__) if classname else func.__name__ + return name + def do_profile(args, kw_args): + name = get_func_name(args) t0 = time.time() o = func(*args, **kw_args) t = time.time() - t0 - print_error("[profiler]", n, "%.4f"%t) + print_error("[profiler]", name, "%.4f"%t) return o - return lambda *args, **kw_args: do_profile(func, args, kw_args) + return lambda *args, **kw_args: do_profile(args, kw_args) def android_ext_dir(): From cf14d7b3469e75e7910a2d1e86b567bc50a076af Mon Sep 17 00:00:00 2001 From: SomberNight Date: Fri, 3 Aug 2018 15:23:39 +0200 Subject: [PATCH 50/56] wallet: change meaning of is_used --- electrum/address_synchronizer.py | 5 +---- electrum/gui/kivy/uix/dialogs/addresses.py | 8 ++++---- electrum/gui/qt/address_list.py | 6 +++--- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/electrum/address_synchronizer.py b/electrum/address_synchronizer.py index 7b729f9e..25a4ab82 100644 --- a/electrum/address_synchronizer.py +++ b/electrum/address_synchronizer.py @@ -757,10 +757,7 @@ class AddressSynchronizer(PrintError): def is_used(self, address): h = self.history.get(address,[]) - if len(h) == 0: - return False - c, u, x = self.get_addr_balance(address) - return c + u + x == 0 + return len(h) != 0 def is_empty(self, address): c, u, x = self.get_addr_balance(address) diff --git a/electrum/gui/kivy/uix/dialogs/addresses.py b/electrum/gui/kivy/uix/dialogs/addresses.py index 5b90f2f7..b1fc40cd 100644 --- a/electrum/gui/kivy/uix/dialogs/addresses.py +++ b/electrum/gui/kivy/uix/dialogs/addresses.py @@ -136,14 +136,14 @@ class AddressesDialog(Factory.Popup): for address in _list: label = wallet.labels.get(address, '') balance = sum(wallet.get_addr_balance(address)) - is_used = wallet.is_used(address) - if self.show_used == 1 and (balance or is_used): + is_used_and_empty = wallet.is_used(address) and balance == 0 + if self.show_used == 1 and (balance or is_used_and_empty): continue if self.show_used == 2 and balance == 0: continue - if self.show_used == 3 and not is_used: + if self.show_used == 3 and not is_used_and_empty: continue - card = self.get_card(address, balance, is_used, label) + card = self.get_card(address, balance, is_used_and_empty, label) if search and not self.ext_search(card, search): continue cards.append(card) diff --git a/electrum/gui/qt/address_list.py b/electrum/gui/qt/address_list.py index 7217940f..c3cb553f 100644 --- a/electrum/gui/qt/address_list.py +++ b/electrum/gui/qt/address_list.py @@ -95,15 +95,15 @@ class AddressList(MyTreeWidget): self.clear() for address in addr_list: num = len(self.wallet.get_address_history(address)) - is_used = self.wallet.is_used(address) label = self.wallet.labels.get(address, '') c, u, x = self.wallet.get_addr_balance(address) balance = c + u + x - if self.show_used == 1 and (balance or is_used): + is_used_and_empty = self.wallet.is_used(address) and balance == 0 + if self.show_used == 1 and (balance or is_used_and_empty): continue if self.show_used == 2 and balance == 0: continue - if self.show_used == 3 and not is_used: + if self.show_used == 3 and not is_used_and_empty: continue balance_text = self.parent.format_amount(balance, whitespaces=True) fx = self.parent.fx From 6b42e8448c8b95880b5ccad1d1ecd313a4d8044d Mon Sep 17 00:00:00 2001 From: SomberNight Date: Fri, 3 Aug 2018 14:49:12 +0200 Subject: [PATCH 51/56] address_synchronizer: cache local_height in some cases --- electrum/address_synchronizer.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/electrum/address_synchronizer.py b/electrum/address_synchronizer.py index 25a4ab82..d5112952 100644 --- a/electrum/address_synchronizer.py +++ b/electrum/address_synchronizer.py @@ -72,6 +72,8 @@ class AddressSynchronizer(PrintError): self.unverified_tx = defaultdict(int) # true when synchronized self.up_to_date = False + # thread local storage for caching stuff + self.threadlocal_cache = threading.local() self.load_and_cleanup() @@ -446,6 +448,19 @@ class AddressSynchronizer(PrintError): else: return (1e9+1, 0) + def with_local_height_cached(func): + # get local height only once, as it's relatively expensive. + # take care that nested calls work as expected + def f(self, *args, **kwargs): + orig_val = getattr(self.threadlocal_cache, 'local_height', None) + self.threadlocal_cache.local_height = orig_val or self.get_local_height() + try: + return func(self, *args, **kwargs) + finally: + self.threadlocal_cache.local_height = orig_val + return f + + @with_local_height_cached def get_history(self, domain=None): # get domain if domain is None: @@ -559,6 +574,9 @@ class AddressSynchronizer(PrintError): def get_local_height(self): """ return last known height if we are offline """ + cached_local_height = getattr(self.threadlocal_cache, 'local_height', None) + if cached_local_height is not None: + return cached_local_height return self.network.get_local_height() if self.network else self.storage.get('stored_height', 0) def get_tx_height(self, tx_hash: str) -> TxMinedStatus: @@ -706,8 +724,11 @@ class AddressSynchronizer(PrintError): received, sent = self.get_addr_io(address) return sum([v for height, v, is_cb in received.values()]) - # return the balance of a bitcoin address: confirmed and matured, unconfirmed, unmatured + @with_local_height_cached def get_addr_balance(self, address): + """Return the balance of a bitcoin address: + confirmed and matured, unconfirmed, unmatured + """ received, sent = self.get_addr_io(address) c = u = x = 0 local_height = self.get_local_height() @@ -725,6 +746,7 @@ class AddressSynchronizer(PrintError): u -= v return c, u, x + @with_local_height_cached def get_utxos(self, domain=None, excluded=None, mature=False, confirmed_only=False): coins = [] if domain is None: From 7307c800d7f922f0f9a04eac2822d7014bbab39a Mon Sep 17 00:00:00 2001 From: SomberNight Date: Fri, 3 Aug 2018 16:12:41 +0200 Subject: [PATCH 52/56] small optimisations for history tab refresh (and related) --- electrum/address_synchronizer.py | 3 +++ electrum/gui/qt/address_list.py | 4 ++-- electrum/gui/qt/history_list.py | 11 +++++++---- electrum/plugin.py | 1 - electrum/util.py | 11 +++++++++-- electrum/wallet.py | 17 ++++++++--------- 6 files changed, 29 insertions(+), 18 deletions(-) diff --git a/electrum/address_synchronizer.py b/electrum/address_synchronizer.py index d5112952..2d6a70cc 100644 --- a/electrum/address_synchronizer.py +++ b/electrum/address_synchronizer.py @@ -101,6 +101,9 @@ class AddressSynchronizer(PrintError): h.append((tx_hash, tx_height)) return h + def get_address_history_len(self, addr: str) -> int: + return len(self._history_local.get(addr, ())) + def get_txin_address(self, txi): addr = txi.get('address') if addr and addr != "(pubkey)": diff --git a/electrum/gui/qt/address_list.py b/electrum/gui/qt/address_list.py index c3cb553f..fce1004c 100644 --- a/electrum/gui/qt/address_list.py +++ b/electrum/gui/qt/address_list.py @@ -93,8 +93,9 @@ class AddressList(MyTreeWidget): else: addr_list = self.wallet.get_addresses() self.clear() + fx = self.parent.fx for address in addr_list: - num = len(self.wallet.get_address_history(address)) + num = self.wallet.get_address_history_len(address) label = self.wallet.labels.get(address, '') c, u, x = self.wallet.get_addr_balance(address) balance = c + u + x @@ -106,7 +107,6 @@ class AddressList(MyTreeWidget): if self.show_used == 3 and not is_used_and_empty: continue balance_text = self.parent.format_amount(balance, whitespaces=True) - fx = self.parent.fx # create item if fx and fx.get_fiat_address_config(): rate = fx.exchange_rate() diff --git a/electrum/gui/qt/history_list.py b/electrum/gui/qt/history_list.py index b1d11a46..79fe4d3a 100644 --- a/electrum/gui/qt/history_list.py +++ b/electrum/gui/qt/history_list.py @@ -229,6 +229,9 @@ class HistoryList(MyTreeWidget, AcceptFileDragDrop): current_tx = item.data(0, Qt.UserRole) if item else None self.clear() if fx: fx.history_used_spot = False + blue_brush = QBrush(QColor("#1E1EFF")) + red_brush = QBrush(QColor("#BC1E1E")) + monospace_font = QFont(MONOSPACE_FONT) for tx_item in self.transactions: tx_hash = tx_item['txid'] height = tx_item['height'] @@ -263,12 +266,12 @@ class HistoryList(MyTreeWidget, AcceptFileDragDrop): if i>3: item.setTextAlignment(i, Qt.AlignRight | Qt.AlignVCenter) if i!=2: - item.setFont(i, QFont(MONOSPACE_FONT)) + item.setFont(i, monospace_font) if value and value < 0: - item.setForeground(3, QBrush(QColor("#BC1E1E"))) - item.setForeground(4, QBrush(QColor("#BC1E1E"))) + item.setForeground(3, red_brush) + item.setForeground(4, red_brush) if fiat_value and not tx_item['fiat_default']: - item.setForeground(6, QBrush(QColor("#1E1EFF"))) + item.setForeground(6, blue_brush) if tx_hash: item.setData(0, Qt.UserRole, tx_hash) self.insertTopLevelItem(0, item) diff --git a/electrum/plugin.py b/electrum/plugin.py index 93ba9995..4ea8e53d 100644 --- a/electrum/plugin.py +++ b/electrum/plugin.py @@ -26,7 +26,6 @@ from collections import namedtuple import traceback import sys import os -import imp import pkgutil import time import threading diff --git a/electrum/util.py b/electrum/util.py index 23c6d69c..123dc212 100644 --- a/electrum/util.py +++ b/electrum/util.py @@ -33,6 +33,7 @@ import threading import hmac import stat import inspect +from locale import localeconv from .i18n import _ @@ -120,6 +121,8 @@ class UserCancelled(Exception): pass class Satoshis(object): + __slots__ = ('value',) + def __new__(cls, value): self = super(Satoshis, cls).__new__(cls) self.value = value @@ -132,6 +135,8 @@ class Satoshis(object): return format_satoshis(self.value) + " BTC" class Fiat(object): + __slots__ = ('value', 'ccy') + def __new__(cls, value, ccy): self = super(Fiat, cls).__new__(cls) self.ccy = ccy @@ -477,8 +482,10 @@ def format_satoshis_plain(x, decimal_point = 8): return "{:.8f}".format(Decimal(x) / scale_factor).rstrip('0').rstrip('.') +DECIMAL_POINT = localeconv()['decimal_point'] + + def format_satoshis(x, num_zeros=0, decimal_point=8, precision=None, is_diff=False, whitespaces=False): - from locale import localeconv if x is None: return 'unknown' if precision is None: @@ -488,7 +495,7 @@ def format_satoshis(x, num_zeros=0, decimal_point=8, precision=None, is_diff=Fal decimal_format = '+' + decimal_format result = ("{:" + decimal_format + "f}").format(x / pow (10, decimal_point)).rstrip('0') integer_part, fract_part = result.split(".") - dp = localeconv()['decimal_point'] + dp = DECIMAL_POINT if len(fract_part) < num_zeros: fract_part += "0" * (num_zeros - len(fract_part)) result = integer_part + dp + fract_part diff --git a/electrum/wallet.py b/electrum/wallet.py index 3572473c..3fac9aed 100644 --- a/electrum/wallet.py +++ b/electrum/wallet.py @@ -43,7 +43,7 @@ from .i18n import _ from .util import (NotEnoughFunds, PrintError, UserCancelled, profiler, format_satoshis, format_fee_satoshis, NoDynamicFeeEstimates, TimeoutException, WalletFileException, BitcoinException, - InvalidPassword) + InvalidPassword, format_time) from .bitcoin import * from .version import * @@ -386,11 +386,12 @@ class Abstract_Wallet(AddressSynchronizer): fiat_income = Decimal(0) fiat_expenditures = Decimal(0) h = self.get_history(domain) + now = time.time() for tx_hash, tx_mined_status, value, balance in h: timestamp = tx_mined_status.timestamp - if from_timestamp and (timestamp or time.time()) < from_timestamp: + if from_timestamp and (timestamp or now) < from_timestamp: continue - if to_timestamp and (timestamp or time.time()) >= to_timestamp: + if to_timestamp and (timestamp or now) >= to_timestamp: continue item = { 'txid': tx_hash, @@ -398,10 +399,10 @@ class Abstract_Wallet(AddressSynchronizer): 'confirmations': tx_mined_status.conf, 'timestamp': timestamp, 'value': Satoshis(value), - 'balance': Satoshis(balance) + 'balance': Satoshis(balance), + 'date': timestamp_to_datetime(timestamp), + 'label': self.get_label(tx_hash), } - item['date'] = timestamp_to_datetime(timestamp) - item['label'] = self.get_label(tx_hash) if show_addresses: tx = self.transactions.get(tx_hash) item['inputs'] = list(map(lambda x: dict((k, x[k]) for k in ('prevout_hash', 'prevout_n')), tx.inputs())) @@ -416,10 +417,9 @@ class Abstract_Wallet(AddressSynchronizer): income += value # fiat computations if fx and fx.is_enabled(): - date = timestamp_to_datetime(timestamp) fiat_value = self.get_fiat_value(tx_hash, fx.ccy) fiat_default = fiat_value is None - fiat_value = fiat_value if fiat_value is not None else value / Decimal(COIN) * self.price_at_timestamp(tx_hash, fx.timestamp_rate) + fiat_value = fiat_value if fiat_value is not None else value / Decimal(COIN) * self.price_at_timestamp(tx_hash, fx.timestamp_rate) # item['fiat_value'] = Fiat(fiat_value, fx.ccy) item['fiat_default'] = fiat_default if value < 0: @@ -487,7 +487,6 @@ class Abstract_Wallet(AddressSynchronizer): return '' def get_tx_status(self, tx_hash, tx_mined_status): - from .util import format_time extra = [] height = tx_mined_status.height conf = tx_mined_status.conf From 531cdeffa9abbbbfdc4f2326dd6af252ace1b0d9 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Fri, 3 Aug 2018 18:25:53 +0200 Subject: [PATCH 53/56] blockchain.py: rename 'checkpoint' to 'forkpoint' --- electrum/blockchain.py | 61 ++++++++++----------- electrum/gui/kivy/main_window.py | 4 +- electrum/gui/kivy/uix/ui_screens/network.kv | 2 +- electrum/gui/qt/network_dialog.py | 8 +-- electrum/network.py | 6 +- electrum/verifier.py | 2 +- 6 files changed, 41 insertions(+), 42 deletions(-) diff --git a/electrum/blockchain.py b/electrum/blockchain.py index 3aaa8067..4adf5856 100644 --- a/electrum/blockchain.py +++ b/electrum/blockchain.py @@ -79,12 +79,12 @@ def read_blockchains(config): l = filter(lambda x: x.startswith('fork_'), os.listdir(fdir)) l = sorted(l, key = lambda x: int(x.split('_')[1])) for filename in l: - checkpoint = int(filename.split('_')[2]) + forkpoint = int(filename.split('_')[2]) parent_id = int(filename.split('_')[1]) - b = Blockchain(config, checkpoint, parent_id) - h = b.read_header(b.checkpoint) + b = Blockchain(config, forkpoint, parent_id) + h = b.read_header(b.forkpoint) if b.parent().can_connect(h, check_height=False): - blockchains[b.checkpoint] = b + blockchains[b.forkpoint] = b else: util.print_error("cannot connect", filename) return blockchains @@ -109,10 +109,10 @@ class Blockchain(util.PrintError): Manages blockchain headers and their verification """ - def __init__(self, config, checkpoint, parent_id): + def __init__(self, config, forkpoint, parent_id): self.config = config - self.catch_up = None # interface catching up - self.checkpoint = checkpoint + self.catch_up = None # interface catching up + self.forkpoint = forkpoint self.checkpoints = constants.net.CHECKPOINTS self.parent_id = parent_id self.lock = threading.Lock() @@ -123,18 +123,18 @@ class Blockchain(util.PrintError): return blockchains[self.parent_id] def get_max_child(self): - children = list(filter(lambda y: y.parent_id==self.checkpoint, blockchains.values())) - return max([x.checkpoint for x in children]) if children else None + children = list(filter(lambda y: y.parent_id==self.forkpoint, blockchains.values())) + return max([x.forkpoint for x in children]) if children else None - def get_checkpoint(self): + def get_forkpoint(self): mc = self.get_max_child() - return mc if mc is not None else self.checkpoint + return mc if mc is not None else self.forkpoint def get_branch_size(self): - return self.height() - self.get_checkpoint() + 1 + return self.height() - self.get_forkpoint() + 1 def get_name(self): - return self.get_hash(self.get_checkpoint()).lstrip('00')[0:10] + return self.get_hash(self.get_forkpoint()).lstrip('00')[0:10] def check_header(self, header): header_hash = hash_header(header) @@ -142,14 +142,14 @@ class Blockchain(util.PrintError): return header_hash == self.get_hash(height) def fork(parent, header): - checkpoint = header.get('block_height') - self = Blockchain(parent.config, checkpoint, parent.checkpoint) + forkpoint = header.get('block_height') + self = Blockchain(parent.config, forkpoint, parent.forkpoint) open(self.path(), 'w+').close() self.save_header(header) return self def height(self): - return self.checkpoint + self.size() - 1 + return self.forkpoint + self.size() - 1 def size(self): with self.lock: @@ -183,12 +183,11 @@ class Blockchain(util.PrintError): def path(self): d = util.get_headers_dir(self.config) - filename = 'blockchain_headers' if self.parent_id is None else os.path.join('forks', 'fork_%d_%d'%(self.parent_id, self.checkpoint)) + filename = 'blockchain_headers' if self.parent_id is None else os.path.join('forks', 'fork_%d_%d'%(self.parent_id, self.forkpoint)) return os.path.join(d, filename) def save_chunk(self, index, chunk): - filename = self.path() - d = (index * 2016 - self.checkpoint) * 80 + d = (index * 2016 - self.forkpoint) * 80 if d < 0: chunk = chunk[-d:] d = 0 @@ -199,28 +198,28 @@ class Blockchain(util.PrintError): def swap_with_parent(self): if self.parent_id is None: return - parent_branch_size = self.parent().height() - self.checkpoint + 1 + parent_branch_size = self.parent().height() - self.forkpoint + 1 if parent_branch_size >= self.size(): return - self.print_error("swap", self.checkpoint, self.parent_id) + self.print_error("swap", self.forkpoint, self.parent_id) parent_id = self.parent_id - checkpoint = self.checkpoint + forkpoint = self.forkpoint parent = self.parent() self.assert_headers_file_available(self.path()) with open(self.path(), 'rb') as f: my_data = f.read() self.assert_headers_file_available(parent.path()) with open(parent.path(), 'rb') as f: - f.seek((checkpoint - parent.checkpoint)*80) + f.seek((forkpoint - parent.forkpoint)*80) parent_data = f.read(parent_branch_size*80) self.write(parent_data, 0) - parent.write(my_data, (checkpoint - parent.checkpoint)*80) + parent.write(my_data, (forkpoint - parent.forkpoint)*80) # store file path for b in blockchains.values(): b.old_path = b.path() # swap parameters self.parent_id = parent.parent_id; parent.parent_id = parent_id - self.checkpoint = parent.checkpoint; parent.checkpoint = checkpoint + self.forkpoint = parent.forkpoint; parent.forkpoint = forkpoint self._size = parent._size; parent._size = parent_branch_size # move files for b in blockchains.values(): @@ -229,8 +228,8 @@ class Blockchain(util.PrintError): self.print_error("renaming", b.old_path, b.path()) os.rename(b.old_path, b.path()) # update pointers - blockchains[self.checkpoint] = self - blockchains[parent.checkpoint] = parent + blockchains[self.forkpoint] = self + blockchains[parent.forkpoint] = parent def assert_headers_file_available(self, path): if os.path.exists(path): @@ -255,7 +254,7 @@ class Blockchain(util.PrintError): self.update_size() def save_header(self, header): - delta = header.get('block_height') - self.checkpoint + delta = header.get('block_height') - self.forkpoint data = bfh(serialize_header(header)) assert delta == self.size() assert len(data) == 80 @@ -263,14 +262,14 @@ class Blockchain(util.PrintError): self.swap_with_parent() def read_header(self, height): - assert self.parent_id != self.checkpoint + assert self.parent_id != self.forkpoint if height < 0: return - if height < self.checkpoint: + if height < self.forkpoint: return self.parent().read_header(height) if height > self.height(): return - delta = height - self.checkpoint + delta = height - self.forkpoint name = self.path() self.assert_headers_file_available(name) with open(name, 'rb') as f: diff --git a/electrum/gui/kivy/main_window.py b/electrum/gui/kivy/main_window.py index d0db73a2..ba8e7da2 100644 --- a/electrum/gui/kivy/main_window.py +++ b/electrum/gui/kivy/main_window.py @@ -88,7 +88,7 @@ class ElectrumWindow(App): balance = StringProperty('') fiat_balance = StringProperty('') is_fiat = BooleanProperty(False) - blockchain_checkpoint = NumericProperty(0) + blockchain_forkpoint = NumericProperty(0) auto_connect = BooleanProperty(False) def on_auto_connect(self, instance, x): @@ -654,7 +654,7 @@ class ElectrumWindow(App): self.num_nodes = len(self.network.get_interfaces()) self.num_chains = len(self.network.get_blockchains()) chain = self.network.blockchain() - self.blockchain_checkpoint = chain.get_checkpoint() + self.blockchain_forkpoint = chain.get_forkpoint() self.blockchain_name = chain.get_name() interface = self.network.interface if interface: diff --git a/electrum/gui/kivy/uix/ui_screens/network.kv b/electrum/gui/kivy/uix/ui_screens/network.kv index f499618a..99fb8366 100644 --- a/electrum/gui/kivy/uix/ui_screens/network.kv +++ b/electrum/gui/kivy/uix/ui_screens/network.kv @@ -46,7 +46,7 @@ Popup: CardSeparator SettingsItem: - title: _('Fork detected at block {}').format(app.blockchain_checkpoint) if app.num_chains>1 else _('No fork detected') + title: _('Fork detected at block {}').format(app.blockchain_forkpoint) if app.num_chains>1 else _('No fork detected') fork_description: (_('You are following branch') if app.auto_connect else _("Your server is on branch")) + ' ' + app.blockchain_name description: self.fork_description if app.num_chains>1 else _('Connected nodes are on the same chain') action: app.choose_blockchain_dialog diff --git a/electrum/gui/qt/network_dialog.py b/electrum/gui/qt/network_dialog.py index f9a1f7da..ecc7695b 100644 --- a/electrum/gui/qt/network_dialog.py +++ b/electrum/gui/qt/network_dialog.py @@ -106,9 +106,9 @@ class NodesListWidget(QTreeWidget): b = network.blockchains[k] name = b.get_name() if n_chains >1: - x = QTreeWidgetItem([name + '@%d'%b.get_checkpoint(), '%d'%b.height()]) + x = QTreeWidgetItem([name + '@%d'%b.get_forkpoint(), '%d'%b.height()]) x.setData(0, Qt.UserRole, 1) - x.setData(1, Qt.UserRole, b.checkpoint) + x.setData(1, Qt.UserRole, b.forkpoint) else: x = self for i in items: @@ -357,9 +357,9 @@ class NetworkChoiceLayout(object): chains = self.network.get_blockchains() if len(chains)>1: chain = self.network.blockchain() - checkpoint = chain.get_checkpoint() + forkpoint = chain.get_forkpoint() name = chain.get_name() - msg = _('Chain split detected at block {0}').format(checkpoint) + '\n' + msg = _('Chain split detected at block {0}').format(forkpoint) + '\n' msg += (_('You are following branch') if auto_connect else _('Your server is on branch'))+ ' ' + name msg += ' (%d %s)' % (chain.get_branch_size(), _('blocks')) else: diff --git a/electrum/network.py b/electrum/network.py index d31569d6..344d60f0 100644 --- a/electrum/network.py +++ b/electrum/network.py @@ -964,7 +964,7 @@ class Network(util.DaemonThread): interface.blockchain = branch.parent() next_height = interface.bad else: - interface.print_error('checkpoint conflicts with existing fork', branch.path()) + interface.print_error('forkpoint conflicts with existing fork', branch.path()) branch.write(b'', 0) branch.save_header(interface.bad_header) interface.mode = 'catch_up' @@ -980,7 +980,7 @@ class Network(util.DaemonThread): with self.blockchains_lock: self.blockchains[interface.bad] = b interface.blockchain = b - interface.print_error("new chain", b.checkpoint) + interface.print_error("new chain", b.forkpoint) interface.mode = 'catch_up' maybe_next_height = interface.bad + 1 if maybe_next_height <= interface.tip: @@ -1143,7 +1143,7 @@ class Network(util.DaemonThread): @with_interface_lock def blockchain(self): if self.interface and self.interface.blockchain is not None: - self.blockchain_index = self.interface.blockchain.checkpoint + self.blockchain_index = self.interface.blockchain.forkpoint return self.blockchains[self.blockchain_index] @with_interface_lock diff --git a/electrum/verifier.py b/electrum/verifier.py index c85cbc3a..4a0d82ec 100644 --- a/electrum/verifier.py +++ b/electrum/verifier.py @@ -145,7 +145,7 @@ class SPV(ThreadJob): raise InnerNodeOfSpvProofIsValidTx() def undo_verifications(self): - height = self.blockchain.get_checkpoint() + height = self.blockchain.get_forkpoint() tx_hashes = self.wallet.undo_verifications(self.blockchain, height) for tx_hash in tx_hashes: self.print_error("redoing", tx_hash) From 2a9f5db5769e52958bc3c4f94ca53d58a8baf9d7 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Fri, 3 Aug 2018 19:06:23 +0200 Subject: [PATCH 54/56] blockchain.py: fix: chunks in checkpoint region were not getting saved if we were on a fork --- electrum/blockchain.py | 35 ++++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/electrum/blockchain.py b/electrum/blockchain.py index 4adf5856..c4c55782 100644 --- a/electrum/blockchain.py +++ b/electrum/blockchain.py @@ -115,10 +115,17 @@ class Blockchain(util.PrintError): self.forkpoint = forkpoint self.checkpoints = constants.net.CHECKPOINTS self.parent_id = parent_id - self.lock = threading.Lock() + assert parent_id != forkpoint + self.lock = threading.RLock() with self.lock: self.update_size() + def with_lock(func): + def func_wrapper(self, *args, **kwargs): + with self.lock: + return func(self, *args, **kwargs) + return func_wrapper + def parent(self): return blockchains[self.parent_id] @@ -186,15 +193,27 @@ class Blockchain(util.PrintError): filename = 'blockchain_headers' if self.parent_id is None else os.path.join('forks', 'fork_%d_%d'%(self.parent_id, self.forkpoint)) return os.path.join(d, filename) + @with_lock def save_chunk(self, index, chunk): - d = (index * 2016 - self.forkpoint) * 80 - if d < 0: - chunk = chunk[-d:] - d = 0 - truncate = index >= len(self.checkpoints) - self.write(chunk, d, truncate) + chunk_within_checkpoint_region = index < len(self.checkpoints) + # chunks in checkpoint region are the responsibility of the 'main chain' + if chunk_within_checkpoint_region and self.parent_id is not None: + main_chain = blockchains[0] + main_chain.save_chunk(index, chunk) + return + + delta_height = (index * 2016 - self.forkpoint) + delta_bytes = delta_height * 80 + # if this chunk contains our forkpoint, only save the part after forkpoint + # (the part before is the responsibility of the parent) + if delta_bytes < 0: + chunk = chunk[-delta_bytes:] + delta_bytes = 0 + truncate = not chunk_within_checkpoint_region + self.write(chunk, delta_bytes, truncate) self.swap_with_parent() + @with_lock def swap_with_parent(self): if self.parent_id is None: return @@ -253,9 +272,11 @@ class Blockchain(util.PrintError): os.fsync(f.fileno()) self.update_size() + @with_lock def save_header(self, header): delta = header.get('block_height') - self.forkpoint data = bfh(serialize_header(header)) + # headers are only _appended_ to the end: assert delta == self.size() assert len(data) == 80 self.write(data, delta*80) From 9228cb5b8ed06317fe51a3e21708c0ba95e0e115 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Fri, 3 Aug 2018 19:56:35 +0200 Subject: [PATCH 55/56] wallet: override get_addresses in Imported_Wallet so that clear_history works --- electrum/wallet.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/electrum/wallet.py b/electrum/wallet.py index 3fac9aed..c9fef88b 100644 --- a/electrum/wallet.py +++ b/electrum/wallet.py @@ -1219,6 +1219,10 @@ class Imported_Wallet(Simple_Wallet): def get_fingerprint(self): return '' + def get_addresses(self): + # note: overridden so that the history can be cleared + return sorted(self.addresses.keys()) + def get_receiving_addresses(self): return self.get_addresses() @@ -1351,7 +1355,8 @@ class Deterministic_Wallet(Abstract_Wallet): return self.keystore.has_seed() def get_addresses(self): - # overloaded so that addresses are ordered based on derivation + # note: overridden so that the history can be cleared. + # addresses are ordered based on derivation out = [] out += self.get_receiving_addresses() out += self.get_change_addresses() From a7cfa56621b5a98279859cdc4874512be7b77999 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Fri, 3 Aug 2018 20:53:56 +0200 Subject: [PATCH 56/56] cosigner pool: don't block gui --- electrum/plugins/cosigner_pool/qt.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/electrum/plugins/cosigner_pool/qt.py b/electrum/plugins/cosigner_pool/qt.py index 6efe73b5..3db937ab 100644 --- a/electrum/plugins/cosigner_pool/qt.py +++ b/electrum/plugins/cosigner_pool/qt.py @@ -38,6 +38,7 @@ from electrum.wallet import Multisig_Wallet from electrum.util import bh2u, bfh from electrum.gui.qt.transaction_dialog import show_transaction +from electrum.gui.qt.util import WaitingDialog import sys import traceback @@ -170,20 +171,26 @@ class Plugin(BasePlugin): return cosigner_xpub in xpub_set def do_send(self, tx): + def on_success(result): + window.show_message(_("Your transaction was sent to the cosigning pool.") + '\n' + + _("Open your cosigner wallet to retrieve it.")) + def on_failure(exc_info): + e = exc_info[1] + try: traceback.print_exception(*exc_info) + except OSError: pass + window.show_error(_("Failed to send transaction to cosigning pool") + ':\n' + str(e)) + for window, xpub, K, _hash in self.cosigner_list: if not self.cosigner_can_sign(tx, xpub): continue + # construct message raw_tx_bytes = bfh(str(tx)) public_key = ecc.ECPubkey(K) message = public_key.encrypt_message(raw_tx_bytes).decode('ascii') - try: - server.put(_hash, message) - except Exception as e: - traceback.print_exc(file=sys.stdout) - window.show_error(_("Failed to send transaction to cosigning pool") + ':\n' + str(e)) - return - window.show_message(_("Your transaction was sent to the cosigning pool.") + '\n' + - _("Open your cosigner wallet to retrieve it.")) + # send message + task = lambda: server.put(_hash, message) + msg = _('Sending transaction to cosigning pool...') + WaitingDialog(window, msg, task, on_success, on_failure) def on_receive(self, keyhash, message): self.print_error("signal arrived for", keyhash)