From 7cc628dc7909c3ff44a5e5dfc0de60bc8e438ce1 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Mon, 24 Sep 2018 17:37:09 +0200 Subject: [PATCH 01/12] synchronizer: fix adding duplicate addresses race --- electrum/synchronizer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/electrum/synchronizer.py b/electrum/synchronizer.py index d74d80d8..568d2f84 100644 --- a/electrum/synchronizer.py +++ b/electrum/synchronizer.py @@ -73,6 +73,7 @@ class Synchronizer(PrintError): asyncio.run_coroutine_threadsafe(self._add(addr), self.asyncio_loop) async def _add(self, addr): + if addr in self.requested_addrs: return self.requested_addrs.add(addr) await self.add_queue.put(addr) From 952e9b87e14b93dd82b603ffdd749a1d8a48212b Mon Sep 17 00:00:00 2001 From: SomberNight Date: Tue, 25 Sep 2018 16:38:26 +0200 Subject: [PATCH 02/12] network: clean-up. make external API clear. rm interface_lock (mostly). --- electrum/commands.py | 2 +- electrum/daemon.py | 7 +- electrum/gui/kivy/main_window.py | 15 +- electrum/gui/kivy/uix/dialogs/settings.py | 5 +- electrum/gui/kivy/uix/ui_screens/proxy.kv | 2 +- electrum/gui/kivy/uix/ui_screens/server.kv | 2 +- electrum/gui/qt/main_window.py | 3 +- electrum/gui/qt/network_dialog.py | 16 +- electrum/gui/stdio.py | 3 +- electrum/gui/text.py | 6 +- electrum/interface.py | 44 +-- electrum/network.py | 419 ++++++++++----------- electrum/plugin.py | 1 + electrum/verifier.py | 11 +- 14 files changed, 254 insertions(+), 282 deletions(-) diff --git a/electrum/commands.py b/electrum/commands.py index 7efe44fb..19f45d6e 100644 --- a/electrum/commands.py +++ b/electrum/commands.py @@ -255,7 +255,7 @@ class Commands: def broadcast(self, tx): """Broadcast a transaction to the network. """ tx = Transaction(tx) - return self.network.broadcast_transaction_from_non_network_thread(tx) + return self.network.run_from_another_thread(self.network.broadcast_transaction(tx)) @command('') def createmultisig(self, num, pubkeys): diff --git a/electrum/daemon.py b/electrum/daemon.py index e867882b..2ce648bf 100644 --- a/electrum/daemon.py +++ b/electrum/daemon.py @@ -28,11 +28,11 @@ import os import time import traceback import sys +import threading -# from jsonrpc import JSONRPCResponseManager import jsonrpclib -from .jsonrpc import VerifyingJSONRPCServer +from .jsonrpc import VerifyingJSONRPCServer from .version import ELECTRUM_VERSION from .network import Network from .util import json_decode, DaemonThread @@ -129,7 +129,7 @@ class Daemon(DaemonThread): self.network = Network(config) self.fx = FxThread(config, self.network) if self.network: - self.network.start(self.fx.run()) + self.network.start([self.fx.run]) self.gui = None self.wallets = {} # Setup JSONRPC server @@ -308,6 +308,7 @@ class Daemon(DaemonThread): gui_name = 'qt' gui = __import__('electrum.gui.' + gui_name, fromlist=['electrum']) self.gui = gui.ElectrumGui(config, self, plugins) + threading.current_thread().setName('GUI') try: self.gui.main() except BaseException as e: diff --git a/electrum/gui/kivy/main_window.py b/electrum/gui/kivy/main_window.py index ec3320f7..27eab521 100644 --- a/electrum/gui/kivy/main_window.py +++ b/electrum/gui/kivy/main_window.py @@ -16,6 +16,7 @@ from electrum.plugin import run_hook from electrum.util import format_satoshis, format_satoshis_plain from electrum.paymentrequest import PR_UNPAID, PR_PAID, PR_UNKNOWN, PR_EXPIRED from electrum import blockchain +from electrum.network import Network from .i18n import _ from kivy.app import App @@ -96,7 +97,7 @@ class ElectrumWindow(App): def on_auto_connect(self, instance, x): net_params = self.network.get_parameters() net_params = net_params._replace(auto_connect=self.auto_connect) - self.network.set_parameters(net_params) + self.network.run_from_another_thread(self.network.set_parameters(net_params)) def toggle_auto_connect(self, x): self.auto_connect = not self.auto_connect @@ -116,9 +117,10 @@ class ElectrumWindow(App): from .uix.dialogs.choice_dialog import ChoiceDialog chains = self.network.get_blockchains() def cb(name): - for index, b in blockchain.blockchains.items(): + with blockchain.blockchains_lock: blockchain_items = list(blockchain.blockchains.items()) + for index, b in blockchain_items: if name == b.get_name(): - self.network.follow_chain(index) + self.network.run_from_another_thread(self.network.follow_chain(index)) names = [blockchain.blockchains[b].get_name() for b in chains] if len(names) > 1: cur_chain = self.network.blockchain().get_name() @@ -265,7 +267,7 @@ class ElectrumWindow(App): title = _('Electrum App') self.electrum_config = config = kwargs.get('config', None) self.language = config.get('language', 'en') - self.network = network = kwargs.get('network', None) + self.network = network = kwargs.get('network', None) # type: Network if self.network: self.num_blocks = self.network.get_local_height() self.num_nodes = len(self.network.get_interfaces()) @@ -708,7 +710,7 @@ class ElectrumWindow(App): status = _("Offline") elif self.network.is_connected(): server_height = self.network.get_server_height() - server_lag = self.network.get_local_height() - server_height + server_lag = self.num_blocks - server_height if not self.wallet.up_to_date or server_height == 0: status = _("Synchronizing...") elif server_lag > 1: @@ -885,7 +887,8 @@ class ElectrumWindow(App): Clock.schedule_once(lambda dt: on_success(tx)) def _broadcast_thread(self, tx, on_complete): - ok, txid = self.network.broadcast_transaction_from_non_network_thread(tx) + ok, txid = self.network.run_from_another_thread( + self.network.broadcast_transaction(tx)) Clock.schedule_once(lambda dt: on_complete(ok, txid)) def broadcast(self, tx, pr=None): diff --git a/electrum/gui/kivy/uix/dialogs/settings.py b/electrum/gui/kivy/uix/dialogs/settings.py index 07530a0a..dddd501b 100644 --- a/electrum/gui/kivy/uix/dialogs/settings.py +++ b/electrum/gui/kivy/uix/dialogs/settings.py @@ -159,8 +159,9 @@ class SettingsDialog(Factory.Popup): return proxy.get('host') +':' + proxy.get('port') if proxy else _('None') def proxy_dialog(self, item, dt): + network = self.app.network if self._proxy_dialog is None: - net_params = self.app.network.get_parameters() + net_params = network.get_parameters() proxy = net_params.proxy def callback(popup): nonlocal net_params @@ -175,7 +176,7 @@ class SettingsDialog(Factory.Popup): else: proxy = None net_params = net_params._replace(proxy=proxy) - self.app.network.set_parameters(net_params) + network.run_from_another_thread(network.set_parameters(net_params)) item.status = self.proxy_status() popup = Builder.load_file('electrum/gui/kivy/uix/ui_screens/proxy.kv') popup.ids.mode.text = proxy.get('mode') if proxy else 'None' diff --git a/electrum/gui/kivy/uix/ui_screens/proxy.kv b/electrum/gui/kivy/uix/ui_screens/proxy.kv index 087254ad..538caa05 100644 --- a/electrum/gui/kivy/uix/ui_screens/proxy.kv +++ b/electrum/gui/kivy/uix/ui_screens/proxy.kv @@ -72,6 +72,6 @@ Popup: proxy['password']=str(root.ids.password.text) if proxy['mode']=='none': proxy = None net_params = net_params._replace(proxy=proxy) - app.network.set_parameters(net_params) + app.network.run_from_another_thread(app.network.set_parameters(net_params)) app.proxy_config = proxy if proxy else {} nd.dismiss() diff --git a/electrum/gui/kivy/uix/ui_screens/server.kv b/electrum/gui/kivy/uix/ui_screens/server.kv index ee959cba..1864e0fb 100644 --- a/electrum/gui/kivy/uix/ui_screens/server.kv +++ b/electrum/gui/kivy/uix/ui_screens/server.kv @@ -58,5 +58,5 @@ Popup: on_release: net_params = app.network.get_parameters() net_params = net_params._replace(host=str(root.ids.host.text), port=str(root.ids.port.text)) - app.network.set_parameters(net_params) + app.network.run_from_another_thread(app.network.set_parameters(net_params)) nd.dismiss() diff --git a/electrum/gui/qt/main_window.py b/electrum/gui/qt/main_window.py index 14ef70ff..d903bad7 100644 --- a/electrum/gui/qt/main_window.py +++ b/electrum/gui/qt/main_window.py @@ -1635,7 +1635,8 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError): if pr and pr.has_expired(): self.payment_request = None return False, _("Payment request has expired") - status, msg = self.network.broadcast_transaction_from_non_network_thread(tx) + status, msg = self.network.run_from_another_thread( + self.network.broadcast_transaction(tx)) if pr and status is True: self.invoices.set_paid(pr, tx.txid()) self.invoices.save() diff --git a/electrum/gui/qt/network_dialog.py b/electrum/gui/qt/network_dialog.py index e58148ac..afb288d8 100644 --- a/electrum/gui/qt/network_dialog.py +++ b/electrum/gui/qt/network_dialog.py @@ -34,6 +34,7 @@ from electrum.i18n import _ from electrum import constants, blockchain from electrum.util import print_error from electrum.interface import serialize_server, deserialize_server +from electrum.network import Network from .util import * @@ -97,7 +98,7 @@ class NodesListWidget(QTreeWidget): pt.setX(50) self.customContextMenuRequested.emit(pt) - def update(self, network): + def update(self, network: Network): self.clear() self.addChild = self.addTopLevelItem chains = network.get_blockchains() @@ -187,7 +188,7 @@ class ServerListWidget(QTreeWidget): class NetworkChoiceLayout(object): - def __init__(self, network, config, wizard=False): + def __init__(self, network: Network, config, wizard=False): self.network = network self.config = config self.protocol = None @@ -361,7 +362,7 @@ class NetworkChoiceLayout(object): status = _("Connected to {0} nodes.").format(n) if n else _("Not connected") self.status_label.setText(status) chains = self.network.get_blockchains() - if len(chains)>1: + if len(chains) > 1: chain = self.network.blockchain() forkpoint = chain.get_forkpoint() name = chain.get_name() @@ -410,15 +411,14 @@ class NetworkChoiceLayout(object): self.set_server() def follow_branch(self, index): - self.network.follow_chain(index) + self.network.run_from_another_thread(self.network.follow_chain(index)) self.update() def follow_server(self, server): - self.network.switch_to_interface(server) net_params = self.network.get_parameters() host, port, protocol = deserialize_server(server) net_params = net_params._replace(host=host, port=port, protocol=protocol) - self.network.set_parameters(net_params) + self.network.run_from_another_thread(self.network.set_parameters(net_params)) self.update() def server_changed(self, x): @@ -451,7 +451,7 @@ class NetworkChoiceLayout(object): net_params = net_params._replace(host=str(self.server_host.text()), port=str(self.server_port.text()), auto_connect=self.autoconnect_cb.isChecked()) - self.network.set_parameters(net_params) + self.network.run_from_another_thread(self.network.set_parameters(net_params)) def set_proxy(self): net_params = self.network.get_parameters() @@ -465,7 +465,7 @@ class NetworkChoiceLayout(object): proxy = None self.tor_cb.setChecked(False) net_params = net_params._replace(proxy=proxy) - self.network.set_parameters(net_params) + self.network.run_from_another_thread(self.network.set_parameters(net_params)) def suggest_proxy(self, found_proxy): self.tor_proxy = found_proxy diff --git a/electrum/gui/stdio.py b/electrum/gui/stdio.py index dc547765..aeb43408 100644 --- a/electrum/gui/stdio.py +++ b/electrum/gui/stdio.py @@ -200,7 +200,8 @@ class ElectrumGui: self.wallet.labels[tx.txid()] = self.str_description print(_("Please wait...")) - status, msg = self.network.broadcast_transaction_from_non_network_thread(tx) + status, msg = self.network.run_from_another_thread( + self.network.broadcast_transaction(tx)) if status: print(_('Payment sent.')) diff --git a/electrum/gui/text.py b/electrum/gui/text.py index 1bfcc4ef..af1db4d1 100644 --- a/electrum/gui/text.py +++ b/electrum/gui/text.py @@ -365,7 +365,8 @@ class ElectrumGui: self.wallet.labels[tx.txid()] = self.str_description self.show_message(_("Please wait..."), getchar=False) - status, msg = self.network.broadcast_transaction_from_non_network_thread(tx) + status, msg = self.network.run_from_another_thread( + self.network.broadcast_transaction(tx)) if status: self.show_message(_('Payment sent.')) @@ -410,7 +411,8 @@ class ElectrumGui: return False if out.get('server') or out.get('proxy'): proxy = electrum.network.deserialize_proxy(out.get('proxy')) if out.get('proxy') else proxy_config - self.network.set_parameters(NetworkParameters(host, port, protocol, proxy, auto_connect)) + net_params = NetworkParameters(host, port, protocol, proxy, auto_connect) + self.network.run_from_another_thread(self.network.set_parameters(net_params)) def settings_dialog(self): fee = str(Decimal(self.config.fee_per_kb()) / COIN) diff --git a/electrum/interface.py b/electrum/interface.py index aa2c2ce7..ddad78e7 100644 --- a/electrum/interface.py +++ b/electrum/interface.py @@ -107,11 +107,7 @@ class NotificationSession(ClientSession): class GracefulDisconnect(Exception): pass - - class ErrorParsingSSLCert(Exception): pass - - class ErrorGettingSSLCertFromServer(Exception): pass @@ -150,8 +146,11 @@ class Interface(PrintError): self.tip_header = None self.tip = 0 - # TODO combine? - self.fut = asyncio.get_event_loop().create_task(self.run()) + # note that an interface dying MUST NOT kill the whole network, + # hence exceptions raised by "run" need to be caught not to kill + # main_taskgroup! the aiosafe decorator does this. + asyncio.run_coroutine_threadsafe( + self.network.main_taskgroup.spawn(self.run()), self.network.asyncio_loop) self.group = SilentTaskGroup() def diagnostic_name(self): @@ -239,31 +238,29 @@ class Interface(PrintError): sslc.check_hostname = 0 return sslc - def handle_graceful_disconnect(func): + def handle_disconnect(func): async def wrapper_func(self, *args, **kwargs): try: return await func(self, *args, **kwargs) except GracefulDisconnect as e: self.print_error("disconnecting gracefully. {}".format(e)) - self.exception = e + finally: + await self.network.connection_down(self.server) return wrapper_func @aiosafe - @handle_graceful_disconnect + @handle_disconnect async def run(self): try: ssl_context = await self._get_ssl_context() except (ErrorParsingSSLCert, ErrorGettingSSLCertFromServer) as e: - self.exception = e + self.print_error('disconnecting due to: {} {}'.format(e, type(e))) return try: await self.open_session(ssl_context, exit_early=False) except (asyncio.CancelledError, OSError, aiorpcx.socks.SOCKSFailure) as e: self.print_error('disconnecting due to: {} {}'.format(e, type(e))) - self.exception = e return - # should never get here (can only exit via exception) - assert False def mark_ready(self): if self.ready.cancelled(): @@ -352,9 +349,9 @@ class Interface(PrintError): self.print_error("connection established. version: {}".format(ver)) async with self.group as group: - await group.spawn(self.ping()) - await group.spawn(self.run_fetch_blocks()) - await group.spawn(self.monitor_connection()) + await group.spawn(self.ping) + await group.spawn(self.run_fetch_blocks) + await group.spawn(self.monitor_connection) # NOTE: group.__aexit__ will be called here; this is needed to notice exceptions in the group! async def monitor_connection(self): @@ -368,11 +365,8 @@ class Interface(PrintError): await asyncio.sleep(300) await self.session.send_request('server.ping') - def close(self): - async def job(): - self.fut.cancel() - await self.group.cancel_remaining() - asyncio.run_coroutine_threadsafe(job(), self.network.asyncio_loop) + async def close(self): + await self.group.cancel_remaining() async def run_fetch_blocks(self): header_queue = asyncio.Queue() @@ -389,7 +383,7 @@ class Interface(PrintError): self.mark_ready() await self._process_header_at_tip() self.network.trigger_callback('network_updated') - self.network.switch_lagging_interface() + await self.network.switch_lagging_interface() async def _process_header_at_tip(self): height, header = self.tip, self.tip_header @@ -517,7 +511,7 @@ class Interface(PrintError): return 'fork_conflict', height self.print_error('forkpoint conflicts with existing fork', branch.path()) self._raise_if_fork_conflicts_with_default_server(branch) - self._disconnect_from_interfaces_on_conflicting_blockchain(branch) + await self._disconnect_from_interfaces_on_conflicting_blockchain(branch) branch.write(b'', 0) branch.save_header(bad_header) self.blockchain = branch @@ -543,8 +537,8 @@ class Interface(PrintError): if chain_to_delete == chain_of_default_server: raise GracefulDisconnect('refusing to overwrite blockchain of default server') - def _disconnect_from_interfaces_on_conflicting_blockchain(self, chain: Blockchain) -> None: - ifaces = self.network.disconnect_from_interfaces_on_given_blockchain(chain) + async def _disconnect_from_interfaces_on_conflicting_blockchain(self, chain: Blockchain) -> None: + ifaces = await self.network.disconnect_from_interfaces_on_given_blockchain(chain) if not ifaces: return servers = [interface.server for interface in ifaces] self.print_error("forcing disconnect of other interfaces: {}".format(servers)) diff --git a/electrum/network.py b/electrum/network.py index 687b7322..f54c9e06 100644 --- a/electrum/network.py +++ b/electrum/network.py @@ -32,18 +32,19 @@ import json import sys import ipaddress import asyncio -from typing import NamedTuple, Optional, Sequence +from typing import NamedTuple, Optional, Sequence, List +import traceback import dns import dns.resolver from aiorpcx import TaskGroup from . import util -from .util import PrintError, print_error, aiosafe, bfh +from .util import PrintError, print_error, aiosafe, bfh, SilentTaskGroup from .bitcoin import COIN from . import constants from . import blockchain -from .blockchain import Blockchain +from .blockchain import Blockchain, HEADER_SIZE from .interface import Interface, serialize_server, deserialize_server from .version import PROTOCOL_VERSION from .simple_config import SimpleConfig @@ -160,14 +161,6 @@ INSTANCE = None class Network(PrintError): """The Network class manages a set of connections to remote electrum servers, each connected socket is handled by an Interface() object. - Connections are initiated by a Connection() thread which stops once - the connection succeeds or fails. - - Our external API: - - - Member functions get_header(), get_interfaces(), get_local_height(), - get_parameters(), get_server_height(), get_status_value(), - is_connected(), set_parameters(), stop() """ verbosity_filter = 'n' @@ -195,14 +188,18 @@ class Network(PrintError): if not self.default_server: self.default_server = pick_random_server() - # locks: if you need to take multiple ones, acquire them in the order they are defined here! + self.main_taskgroup = None + self._jobs = [] + + # locks + self.restart_lock = asyncio.Lock() self.bhi_lock = asyncio.Lock() - self.interface_lock = threading.RLock() # <- re-entrant self.callback_lock = threading.Lock() self.recent_servers_lock = threading.RLock() # <- re-entrant + self.interfaces_lock = threading.Lock() # for mutating/iterating self.interfaces self.server_peers = {} # returned by interface (servers that the main interface knows about) - self.recent_servers = self.read_recent_servers() # note: needs self.recent_servers_lock + self.recent_servers = self._read_recent_servers() # note: needs self.recent_servers_lock self.banner = '' self.donation_address = '' @@ -219,26 +216,30 @@ class Network(PrintError): # kick off the network. interface is the main server we are currently # communicating with. interfaces is the set of servers we are connecting # to or have an ongoing connection with - self.interface = None # note: needs self.interface_lock - self.interfaces = {} # note: needs self.interface_lock + self.interface = None + self.interfaces = {} self.auto_connect = self.config.get('auto_connect', True) self.connecting = set() self.server_queue = None - self.server_queue_group = None + self.proxy = None + self.asyncio_loop = asyncio.get_event_loop() - self.start_network(deserialize_server(self.default_server)[2], - deserialize_proxy(self.config.get('proxy'))) + #self.asyncio_loop.set_debug(1) + self._run_forever = asyncio.Future() + self._thread = threading.Thread(target=self.asyncio_loop.run_until_complete, + args=(self._run_forever,), + name='Network') + self._thread.start() + + def run_from_another_thread(self, coro): + assert self._thread != threading.current_thread(), 'must not be called from network thread' + fut = asyncio.run_coroutine_threadsafe(coro, self.asyncio_loop) + return fut.result() @staticmethod def get_instance(): return INSTANCE - def with_interface_lock(func): - def func_wrapper(self, *args, **kwargs): - with self.interface_lock: - return func(self, *args, **kwargs) - return func_wrapper - def with_recent_servers_lock(func): def func_wrapper(self, *args, **kwargs): with self.recent_servers_lock: @@ -266,7 +267,7 @@ class Network(PrintError): else: self.asyncio_loop.call_soon_threadsafe(callback, event, *args) - def read_recent_servers(self): + def _read_recent_servers(self): if not self.config.path: return [] path = os.path.join(self.config.path, "recent_servers") @@ -278,7 +279,7 @@ class Network(PrintError): return [] @with_recent_servers_lock - def save_recent_servers(self): + def _save_recent_servers(self): if not self.config.path: return path = os.path.join(self.config.path, "recent_servers") @@ -289,11 +290,11 @@ class Network(PrintError): except: pass - @with_interface_lock def get_server_height(self): - return self.interface.tip if self.interface else 0 + interface = self.interface + return interface.tip if interface else 0 - def server_is_lagging(self): + async def _server_is_lagging(self): sh = self.get_server_height() if not sh: self.print_error('no height for main interface') @@ -304,7 +305,7 @@ class Network(PrintError): self.print_error('%s is lagging (%d vs %d)' % (self.default_server, sh, lh)) return result - def set_status(self, status): + def _set_status(self, status): self.connection_status = status self.notify('status') @@ -315,7 +316,7 @@ class Network(PrintError): def is_connecting(self): return self.connection_status == 'connecting' - async def request_server_info(self, interface): + async def _request_server_info(self, interface): await interface.ready session = interface.session @@ -340,9 +341,9 @@ class Network(PrintError): await group.spawn(get_donation_address) await group.spawn(get_server_peers) await group.spawn(get_relay_fee) - await group.spawn(self.request_fee_estimates(interface)) + await group.spawn(self._request_fee_estimates(interface)) - async def request_fee_estimates(self, interface): + async def _request_fee_estimates(self, interface): session = interface.session from .simple_config import FEE_ETA_TARGETS self.config.requested_fee_estimates() @@ -389,10 +390,10 @@ class Network(PrintError): if self.is_connected(): return self.donation_address - @with_interface_lock - def get_interfaces(self): - '''The interfaces that are in connected state''' - return list(self.interfaces.keys()) + def get_interfaces(self) -> List[str]: + """The list of servers for the connected interfaces.""" + with self.interfaces_lock: + return list(self.interfaces) @with_recent_servers_lock def get_servers(self): @@ -407,31 +408,31 @@ class Network(PrintError): if host not in out: out[host] = {protocol: port} # add servers received from main interface - if self.server_peers: - out.update(filter_version(self.server_peers.copy())) + server_peers = self.server_peers + if server_peers: + out.update(filter_version(server_peers.copy())) # potentially filter out some if self.config.get('noonion'): out = filter_noonion(out) return out - @with_interface_lock - def start_interface(self, server): + def _start_interface(self, server): if server not in self.interfaces and server not in self.connecting: if server == self.default_server: self.print_error("connecting to %s as new interface" % server) - self.set_status('connecting') + self._set_status('connecting') self.connecting.add(server) self.server_queue.put(server) - def start_random_interface(self): - with self.interface_lock: + def _start_random_interface(self): + with self.interfaces_lock: exclude_set = self.disconnected_servers | set(self.interfaces) | self.connecting server = pick_random_server(self.get_servers(), self.protocol, exclude_set) if server: - self.start_interface(server) + self._start_interface(server) return server - def set_proxy(self, proxy: Optional[dict]): + def _set_proxy(self, proxy: Optional[dict]): self.proxy = proxy # Store these somewhere so we can un-monkey-patch if not hasattr(socket, "_getaddrinfo"): @@ -467,10 +468,10 @@ class Network(PrintError): addr = str(answers[0]) else: addr = host - except dns.exception.DNSException: + except dns.exception.DNSException as e: # dns failed for some reason, e.g. dns.resolver.NXDOMAIN # this is normal. Simply report back failure: - raise socket.gaierror(11001, 'getaddrinfo failed') + raise socket.gaierror(11001, 'getaddrinfo failed') from e except BaseException as e: # Possibly internal error in dnspython :( see #4483 # Fall back to original socket.getaddrinfo to resolve dns. @@ -478,48 +479,8 @@ class Network(PrintError): addr = host return socket._getaddrinfo(addr, *args, **kwargs) - @with_interface_lock - def start_network(self, protocol: str, proxy: Optional[dict]): - assert not self.interface and not self.interfaces - assert not self.connecting and not self.server_queue - assert not self.server_queue_group - self.print_error('starting network') - self.disconnected_servers = set([]) # note: needs self.interface_lock - self.protocol = protocol - self._init_server_queue() - self.set_proxy(proxy) - self.start_interface(self.default_server) - self.trigger_callback('network_updated') - - def _init_server_queue(self): - self.server_queue = queue.Queue() - self.server_queue_group = server_queue_group = TaskGroup() - async def job(): - forever = asyncio.Event() - async with server_queue_group as group: - await group.spawn(forever.wait()) - asyncio.run_coroutine_threadsafe(job(), self.asyncio_loop) - - @with_interface_lock - def stop_network(self): - self.print_error("stopping network") - for interface in list(self.interfaces.values()): - self.close_interface(interface) - if self.interface: - self.close_interface(self.interface) - assert self.interface is None - assert not self.interfaces - self.connecting.clear() - self._stop_server_queue() - self.trigger_callback('network_updated') - - def _stop_server_queue(self): - # Get a new queue - no old pending connections thanks! - self.server_queue = None - asyncio.run_coroutine_threadsafe(self.server_queue_group.cancel_remaining(), self.asyncio_loop) - self.server_queue_group = None - - def set_parameters(self, net_params: NetworkParameters): + @aiosafe + async def set_parameters(self, net_params: NetworkParameters): proxy = net_params.proxy proxy_str = serialize_proxy(proxy) host, port, protocol = net_params.host, net_params.port, net_params.protocol @@ -538,30 +499,30 @@ class Network(PrintError): # abort if changes were not allowed by config if self.config.get('server') != server_str or self.config.get('proxy') != proxy_str: return - self.auto_connect = net_params.auto_connect - if self.proxy != proxy or self.protocol != protocol: - # Restart the network defaulting to the given server - with self.interface_lock: - self.stop_network() - self.default_server = server_str - self.start_network(protocol, proxy) - elif self.default_server != server_str: - self.switch_to_interface(server_str) - else: - self.switch_lagging_interface() - def switch_to_random_interface(self): + async with self.restart_lock: + self.auto_connect = net_params.auto_connect + if self.proxy != proxy or self.protocol != protocol: + # Restart the network defaulting to the given server + await self._stop() + self.default_server = server_str + await self._start() + elif self.default_server != server_str: + await self.switch_to_interface(server_str) + else: + await self.switch_lagging_interface() + + async def _switch_to_random_interface(self): '''Switch to a random connected server other than the current one''' servers = self.get_interfaces() # Those in connected state if self.default_server in servers: servers.remove(self.default_server) if servers: - self.switch_to_interface(random.choice(servers)) + await self.switch_to_interface(random.choice(servers)) - @with_interface_lock - def switch_lagging_interface(self): + async def switch_lagging_interface(self): '''If auto_connect and lagging, switch interface''' - if self.server_is_lagging() and self.auto_connect: + if await self._server_is_lagging() and self.auto_connect: # switch to one that has the correct header (not height) header = self.blockchain().read_header(self.get_local_height()) def filt(x): @@ -569,111 +530,105 @@ class Network(PrintError): b = header assert type(a) is type(b) return a == b - filtered = list(map(lambda x: x[0], filter(filt, self.interfaces.items()))) + + with self.interfaces_lock: interfaces_items = list(self.interfaces.items()) + filtered = list(map(lambda x: x[0], filter(filt, interfaces_items))) if filtered: choice = random.choice(filtered) - self.switch_to_interface(choice) + await self.switch_to_interface(choice) - @with_interface_lock - def switch_to_interface(self, server): - '''Switch to server as our interface. If no connection exists nor - being opened, start a thread to connect. The actual switch will - happen on receipt of the connection notification. Do nothing - if server already is our interface.''' + async def switch_to_interface(self, server: str): + """Switch to server as our main interface. If no connection exists, + queue interface to be started. The actual switch will + happen when the interface becomes ready. + """ self.default_server = server + old_interface = self.interface + old_server = old_interface.server if old_interface else None + + # Stop any current interface in order to terminate subscriptions, + # and to cancel tasks in interface.group. + # However, for headers sub, give preference to this interface + # over unknown ones, i.e. start it again right away. + if old_server and old_server != server: + await self._close_interface(old_interface) + if len(self.interfaces) <= self.num_server: + self._start_interface(old_server) + if server not in self.interfaces: self.interface = None - self.start_interface(server) + self._start_interface(server) return i = self.interfaces[server] - if self.interface != i: + if old_interface != i: self.print_error("switching to", server) - blockchain_updated = False - if self.interface is not None: - blockchain_updated = i.blockchain != self.interface.blockchain - # Stop any current interface in order to terminate subscriptions, - # and to cancel tasks in interface.group. - # However, for headers sub, give preference to this interface - # over unknown ones, i.e. start it again right away. - old_server = self.interface.server - self.close_interface(self.interface) - if old_server != server and len(self.interfaces) <= self.num_server: - self.start_interface(old_server) - + blockchain_updated = i.blockchain != self.blockchain() self.interface = i - asyncio.run_coroutine_threadsafe( - i.group.spawn(self.request_server_info(i)), self.asyncio_loop) + await i.group.spawn(self._request_server_info(i)) self.trigger_callback('default_server_changed') - self.set_status('connected') + self._set_status('connected') self.trigger_callback('network_updated') if blockchain_updated: self.trigger_callback('blockchain_updated') - @with_interface_lock - def close_interface(self, interface): + async def _close_interface(self, interface): if interface: - if interface.server in self.interfaces: - self.interfaces.pop(interface.server) + with self.interfaces_lock: + if self.interfaces.get(interface.server) == interface: + self.interfaces.pop(interface.server) if interface.server == self.default_server: self.interface = None - interface.close() + await interface.close() @with_recent_servers_lock - def add_recent_server(self, server): + def _add_recent_server(self, server): # list is ordered if server in self.recent_servers: self.recent_servers.remove(server) self.recent_servers.insert(0, server) self.recent_servers = self.recent_servers[0:20] - self.save_recent_servers() + self._save_recent_servers() - @with_interface_lock - def connection_down(self, server): + async def connection_down(self, server): '''A connection to server either went down, or was never made. We distinguish by whether it is in self.interfaces.''' self.disconnected_servers.add(server) if server == self.default_server: - self.set_status('disconnected') - if server in self.interfaces: - self.close_interface(self.interfaces[server]) + self._set_status('disconnected') + interface = self.interfaces.get(server, None) + if interface: + await self._close_interface(interface) self.trigger_callback('network_updated') @aiosafe - async def new_interface(self, server): + async def _run_new_interface(self, server): interface = Interface(self, server, self.config.path, self.proxy) timeout = 10 if not self.proxy else 20 try: await asyncio.wait_for(interface.ready, timeout) except BaseException as e: - #import traceback #traceback.print_exc() self.print_error(server, "couldn't launch because", str(e), str(type(e))) - # note: connection_down will not call interface.close() as - # interface is not yet in self.interfaces. OTOH, calling - # interface.close() here will sometimes raise deep inside the - # asyncio internal select.select... instead, interface will close - # itself when it detects the cancellation of interface.ready; - # however this might take several seconds... - self.connection_down(server) + await interface.close() return else: - with self.interface_lock: + with self.interfaces_lock: + assert server not in self.interfaces self.interfaces[server] = interface finally: - with self.interface_lock: - try: self.connecting.remove(server) - except KeyError: pass + try: self.connecting.remove(server) + except KeyError: pass if server == self.default_server: - self.switch_to_interface(server) + await self.switch_to_interface(server) - self.add_recent_server(server) + self._add_recent_server(server) self.trigger_callback('network_updated') - def init_headers_file(self): + async def _init_headers_file(self): b = blockchain.blockchains[0] filename = b.path() - length = 80 * len(constants.net.CHECKPOINTS) * 2016 + length = HEADER_SIZE * len(constants.net.CHECKPOINTS) * 2016 if not os.path.exists(filename) or os.path.getsize(filename) < length: with open(filename, 'wb') as f: if length > 0: @@ -686,11 +641,6 @@ class Network(PrintError): async def get_merkle_for_transaction(self, tx_hash, tx_height): return await self.interface.session.send_request('blockchain.transaction.get_merkle', [tx_hash, tx_height]) - def broadcast_transaction_from_non_network_thread(self, tx, timeout=10): - # note: calling this from the network thread will deadlock it - fut = asyncio.run_coroutine_threadsafe(self.broadcast_transaction(tx, timeout=timeout), self.asyncio_loop) - return fut.result() - async def broadcast_transaction(self, tx, timeout=10): try: out = await self.interface.session.send_request('blockchain.transaction.broadcast', [str(tx)], timeout=timeout) @@ -706,101 +656,124 @@ class Network(PrintError): async def request_chunk(self, height, tip=None, *, can_return_early=False): return await self.interface.request_chunk(height, tip=tip, can_return_early=can_return_early) - @with_interface_lock def blockchain(self): - if self.interface and self.interface.blockchain is not None: - self.blockchain_index = self.interface.blockchain.forkpoint + interface = self.interface + if interface and interface.blockchain is not None: + self.blockchain_index = interface.blockchain.forkpoint return blockchain.blockchains[self.blockchain_index] - @with_interface_lock def get_blockchains(self): out = {} # blockchain_id -> list(interfaces) with blockchain.blockchains_lock: blockchain_items = list(blockchain.blockchains.items()) + with self.interfaces_lock: interfaces_values = list(self.interfaces.values()) for chain_id, bc in blockchain_items: - r = list(filter(lambda i: i.blockchain==bc, list(self.interfaces.values()))) + r = list(filter(lambda i: i.blockchain==bc, interfaces_values)) if r: out[chain_id] = r return out - @with_interface_lock - def disconnect_from_interfaces_on_given_blockchain(self, chain: Blockchain) -> Sequence[Interface]: + async def disconnect_from_interfaces_on_given_blockchain(self, chain: Blockchain) -> Sequence[Interface]: chain_id = chain.forkpoint ifaces = self.get_blockchains().get(chain_id) or [] for interface in ifaces: - self.connection_down(interface.server) + await self.connection_down(interface.server) return ifaces - def follow_chain(self, index): - bc = blockchain.blockchains.get(index) + async def follow_chain(self, chain_id): + bc = blockchain.blockchains.get(chain_id) if bc: - self.blockchain_index = index - self.config.set_key('blockchain_index', index) - with self.interface_lock: - interfaces = list(self.interfaces.values()) - for i in interfaces: - if i.blockchain == bc: - self.switch_to_interface(i.server) + self.blockchain_index = chain_id + self.config.set_key('blockchain_index', chain_id) + with self.interfaces_lock: interfaces_values = list(self.interfaces.values()) + for iface in interfaces_values: + if iface.blockchain == bc: + await self.switch_to_interface(iface.server) break else: - raise Exception('blockchain not found', index) + raise Exception('blockchain not found', chain_id) - with self.interface_lock: - if self.interface: - net_params = self.get_parameters() - host, port, protocol = deserialize_server(self.interface.server) - net_params = net_params._replace(host=host, port=port, protocol=protocol) - self.set_parameters(net_params) + if self.interface: + net_params = self.get_parameters() + host, port, protocol = deserialize_server(self.interface.server) + net_params = net_params._replace(host=host, port=port, protocol=protocol) + await self.set_parameters(net_params) def get_local_height(self): return self.blockchain().height() def export_checkpoints(self, path): - # run manually from the console to generate checkpoints + """Run manually to generate blockchain checkpoints. + Kept for console use only. + """ cp = self.blockchain().get_checkpoints() with open(path, 'w', encoding='utf-8') as f: f.write(json.dumps(cp, indent=4)) - def start(self, fx=None): - self.main_taskgroup = TaskGroup() + async def _start(self, jobs=None): + if jobs is None: jobs = self._jobs + self._jobs = jobs + assert not self.main_taskgroup + self.main_taskgroup = SilentTaskGroup() + async def main(): - self.init_headers_file() - async with self.main_taskgroup as group: - await group.spawn(self.maintain_sessions()) - if fx: await group.spawn(fx) - self._wrapper_thread = threading.Thread(target=self.asyncio_loop.run_until_complete, args=(main(),)) - self._wrapper_thread.start() + try: + await self._init_headers_file() + async with self.main_taskgroup as group: + await group.spawn(self._maintain_sessions()) + [await group.spawn(job) for job in jobs] + except Exception as e: + traceback.print_exc(file=sys.stderr) + raise e + asyncio.run_coroutine_threadsafe(main(), self.asyncio_loop) + + assert not self.interface and not self.interfaces + assert not self.connecting and not self.server_queue + self.print_error('starting network') + self.disconnected_servers = set([]) + self.protocol = deserialize_server(self.default_server)[2] + self.server_queue = queue.Queue() + self._set_proxy(deserialize_proxy(self.config.get('proxy'))) + self._start_interface(self.default_server) + self.trigger_callback('network_updated') + + def start(self, jobs=None): + asyncio.run_coroutine_threadsafe(self._start(jobs=jobs), self.asyncio_loop) + + async def _stop(self, full_shutdown=False): + self.print_error("stopping network") + try: + asyncio.wait_for(await self.main_taskgroup.cancel_remaining(), timeout=2) + except asyncio.TimeoutError: pass + self.main_taskgroup = None + + assert self.interface is None + assert not self.interfaces + self.connecting.clear() + self.server_queue = None + self.trigger_callback('network_updated') + + if full_shutdown: + self._run_forever.set_result(1) def stop(self): - asyncio.run_coroutine_threadsafe(self.main_taskgroup.cancel_remaining(), self.asyncio_loop) + assert self._thread != threading.current_thread(), 'must not be called from network thread' + fut = asyncio.run_coroutine_threadsafe(self._stop(full_shutdown=True), self.asyncio_loop) + fut.result() def join(self): - self._wrapper_thread.join(1) + self._thread.join(1) - async def maintain_sessions(self): + async def _maintain_sessions(self): while True: + # launch already queued up new interfaces while self.server_queue.qsize() > 0: server = self.server_queue.get() - await self.server_queue_group.spawn(self.new_interface(server)) - remove = [] - for k, i in self.interfaces.items(): - if i.fut.done() and not i.exception: - assert False, "interface future should not finish without exception" - if i.exception: - if not i.fut.done(): - try: i.fut.cancel() - except Exception as e: self.print_error('exception while cancelling fut', e) - try: - raise i.exception - except BaseException as e: - self.print_error(i.server, "errored because:", str(e), str(type(e))) - remove.append(k) - for k in remove: - self.connection_down(k) + await self.main_taskgroup.spawn(self._run_new_interface(server)) - # nodes + # maybe queue new interfaces to be launched later now = time.time() for i in range(self.num_server - len(self.interfaces) - len(self.connecting)): - self.start_random_interface() + self._start_random_interface() if now - self.nodes_retry_time > NODES_RETRY_INTERVAL: self.print_error('network: retrying connections') self.disconnected_servers = set([]) @@ -810,16 +783,16 @@ class Network(PrintError): if not self.is_connected(): if self.auto_connect: if not self.is_connecting(): - self.switch_to_random_interface() + await self._switch_to_random_interface() else: if self.default_server in self.disconnected_servers: if now - self.server_retry_time > SERVER_RETRY_INTERVAL: self.disconnected_servers.remove(self.default_server) self.server_retry_time = now else: - self.switch_to_interface(self.default_server) + await self.switch_to_interface(self.default_server) else: if self.config.is_fee_estimates_update_required(): - await self.interface.group.spawn(self.request_fee_estimates, self.interface) + await self.interface.group.spawn(self._request_fee_estimates, self.interface) await asyncio.sleep(0.1) diff --git a/electrum/plugin.py b/electrum/plugin.py index f269fb89..a13f7a72 100644 --- a/electrum/plugin.py +++ b/electrum/plugin.py @@ -47,6 +47,7 @@ class Plugins(DaemonThread): @profiler def __init__(self, config, is_local, gui_name): DaemonThread.__init__(self) + self.setName('Plugins') self.pkgpath = os.path.dirname(plugins.__file__) self.config = config self.hw_wallets = {} diff --git a/electrum/verifier.py b/electrum/verifier.py index eada0eee..cb6bdfa6 100644 --- a/electrum/verifier.py +++ b/electrum/verifier.py @@ -47,7 +47,6 @@ class SPV(PrintError): def __init__(self, network, wallet): self.wallet = wallet self.network = network - self.blockchain = network.blockchain() self.merkle_roots = {} # txid -> merkle root (once it has been verified) self.requested_merkle = set() # txid set of pending requests @@ -55,18 +54,14 @@ class SPV(PrintError): return '{}:{}'.format(self.__class__.__name__, self.wallet.diagnostic_name()) async def main(self, group: TaskGroup): + self.blockchain = self.network.blockchain() while True: await self._maybe_undo_verifications() await self._request_proofs(group) await asyncio.sleep(0.1) async def _request_proofs(self, group: TaskGroup): - blockchain = self.network.blockchain() - if not blockchain: - self.print_error("no blockchain") - return - - local_height = self.network.get_local_height() + local_height = self.blockchain.height() unverified = self.wallet.get_unverified_txs() for tx_hash, tx_height in unverified.items(): @@ -77,7 +72,7 @@ class SPV(PrintError): if tx_height <= 0 or tx_height > local_height: continue # if it's in the checkpoint region, we still might not have the header - header = blockchain.read_header(tx_height) + header = self.blockchain.read_header(tx_height) if header is None: if tx_height < constants.net.max_checkpoint(): await group.spawn(self.network.request_chunk(tx_height, None, can_return_early=True)) From 9d7cf12244a0a519e7b65398e4783ab9ea64ba05 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Tue, 25 Sep 2018 17:00:43 +0200 Subject: [PATCH 03/12] follow-up prev: fix tests --- electrum/tests/test_network.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/electrum/tests/test_network.py b/electrum/tests/test_network.py index dc5ba440..c69375bd 100644 --- a/electrum/tests/test_network.py +++ b/electrum/tests/test_network.py @@ -7,10 +7,17 @@ from electrum.simple_config import SimpleConfig from electrum import blockchain from electrum.interface import Interface +class MockTaskGroup: + async def spawn(self, x): return + +class MockNetwork: + main_taskgroup = MockTaskGroup() + asyncio_loop = asyncio.get_event_loop() + class MockInterface(Interface): def __init__(self, config): self.config = config - super().__init__(None, 'mock-server:50000:t', self.config.electrum_path(), None) + super().__init__(MockNetwork(), 'mock-server:50000:t', self.config.electrum_path(), None) self.q = asyncio.Queue() self.blockchain = blockchain.Blockchain(self.config, 2002, None) self.tip = 12 From 33d14e4238d08f69934c8fcb6f77926a8f39f6d1 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Tue, 25 Sep 2018 18:15:28 +0200 Subject: [PATCH 04/12] some import clean-up in qt --- electrum/gui/__init__.py | 2 +- electrum/gui/qt/__init__.py | 4 ---- electrum/gui/qt/completion_text_edit.py | 2 ++ electrum/gui/qt/console.py | 9 +++++++-- electrum/gui/qt/contact_list.py | 10 ++++++---- electrum/gui/qt/fee_slider.py | 6 ++++-- electrum/gui/qt/history_list.py | 3 ++- electrum/gui/qt/main_window.py | 12 +++++++----- electrum/gui/qt/password_dialog.py | 13 ++++++++----- electrum/gui/qt/paytoedit.py | 4 +++- electrum/gui/qt/qrcodewidget.py | 5 ++--- electrum/gui/qt/qrtextedit.py | 6 +++--- electrum/gui/qt/qrwindow.py | 1 + electrum/gui/qt/request_list.py | 8 +++++--- electrum/gui/qt/transaction_dialog.py | 1 - electrum/gui/qt/utxo_list.py | 4 +++- electrum/gui/stdio.py | 9 ++++++--- electrum/gui/text.py | 8 +++++--- 18 files changed, 65 insertions(+), 42 deletions(-) diff --git a/electrum/gui/__init__.py b/electrum/gui/__init__.py index 9974520a..02fe271c 100644 --- a/electrum/gui/__init__.py +++ b/electrum/gui/__init__.py @@ -1,5 +1,5 @@ # To create a new GUI, please add its code to this directory. # Three objects are passed to the ElectrumGui: config, daemon and plugins -# The Wallet object is instanciated by the GUI +# The Wallet object is instantiated by the GUI # Notifications about network events are sent to the GUI by using network.register_callback() diff --git a/electrum/gui/qt/__init__.py b/electrum/gui/qt/__init__.py index 80adeaa9..767de6e9 100644 --- a/electrum/gui/qt/__init__.py +++ b/electrum/gui/qt/__init__.py @@ -42,12 +42,8 @@ from electrum.i18n import _, set_language from electrum.plugin import run_hook from electrum.storage import WalletStorage from electrum.base_wizard import GoBack -# from electrum.synchronizer import Synchronizer -# from electrum.verifier import SPV -# from electrum.util import DebugMem from electrum.util import (UserCancelled, PrintError, WalletFileException, BitcoinException) -# from electrum.wallet import Abstract_Wallet from .installwizard import InstallWizard diff --git a/electrum/gui/qt/completion_text_edit.py b/electrum/gui/qt/completion_text_edit.py index 4709b03d..ca6e6832 100644 --- a/electrum/gui/qt/completion_text_edit.py +++ b/electrum/gui/qt/completion_text_edit.py @@ -26,8 +26,10 @@ from PyQt5.QtGui import * from PyQt5.QtCore import * from PyQt5.QtWidgets import * + from .util import ButtonsTextEdit + class CompletionTextEdit(ButtonsTextEdit): def __init__(self, parent=None): diff --git a/electrum/gui/qt/console.py b/electrum/gui/qt/console.py index d0b10a30..f378e580 100644 --- a/electrum/gui/qt/console.py +++ b/electrum/gui/qt/console.py @@ -1,11 +1,16 @@ # source: http://stackoverflow.com/questions/2758159/how-to-embed-a-python-interpreter-in-a-pyqt-widget -import sys, os, re -import traceback, platform +import sys +import os +import re +import traceback +import platform + from PyQt5 import QtCore from PyQt5 import QtGui from PyQt5 import QtWidgets + from electrum import util from electrum.i18n import _ diff --git a/electrum/gui/qt/contact_list.py b/electrum/gui/qt/contact_list.py index b13ee9ec..d85c6df5 100644 --- a/electrum/gui/qt/contact_list.py +++ b/electrum/gui/qt/contact_list.py @@ -24,14 +24,16 @@ # SOFTWARE. import webbrowser -from electrum.i18n import _ -from electrum.bitcoin import is_address -from electrum.util import block_explorer_URL -from electrum.plugin import run_hook from PyQt5.QtGui import * from PyQt5.QtCore import * from PyQt5.QtWidgets import ( QAbstractItemView, QFileDialog, QMenu, QTreeWidgetItem) + +from electrum.i18n import _ +from electrum.bitcoin import is_address +from electrum.util import block_explorer_URL +from electrum.plugin import run_hook + from .util import MyTreeWidget, import_meta_gui, export_meta_gui diff --git a/electrum/gui/qt/fee_slider.py b/electrum/gui/qt/fee_slider.py index 04911d87..019412b6 100644 --- a/electrum/gui/qt/fee_slider.py +++ b/electrum/gui/qt/fee_slider.py @@ -1,9 +1,11 @@ -from electrum.i18n import _ +import threading + from PyQt5.QtGui import * from PyQt5.QtCore import * from PyQt5.QtWidgets import QSlider, QToolTip -import threading +from electrum.i18n import _ + class FeeSlider(QSlider): diff --git a/electrum/gui/qt/history_list.py b/electrum/gui/qt/history_list.py index bb865580..d12903c7 100644 --- a/electrum/gui/qt/history_list.py +++ b/electrum/gui/qt/history_list.py @@ -28,10 +28,11 @@ import datetime from datetime import date from electrum.address_synchronizer import TX_HEIGHT_LOCAL -from .util import * from electrum.i18n import _ from electrum.util import block_explorer_URL, profiler, print_error, TxMinedStatus +from .util import * + try: from electrum.plot import plot_history, NothingToPlotException except: diff --git a/electrum/gui/qt/main_window.py b/electrum/gui/qt/main_window.py index d903bad7..f89b9e7f 100644 --- a/electrum/gui/qt/main_window.py +++ b/electrum/gui/qt/main_window.py @@ -22,8 +22,12 @@ # ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -import sys, time, threading -import os, json, traceback +import sys +import time +import threading +import os +import traceback +import json import shutil import weakref import webbrowser @@ -36,8 +40,6 @@ import queue from PyQt5.QtGui import * from PyQt5.QtCore import * import PyQt5.QtCore as QtCore - -from .exception_window import Exception_Hook from PyQt5.QtWidgets import * from electrum import (keystore, simple_config, ecc, constants, util, bitcoin, commands, @@ -56,6 +58,7 @@ from electrum.transaction import Transaction, TxOutput from electrum.address_synchronizer import AddTransactionException from electrum.wallet import Multisig_Wallet, CannotBumpFee +from .exception_window import Exception_Hook from .amountedit import AmountEdit, BTCAmountEdit, MyLineEdit, FeerateEdit from .qrcodewidget import QRCodeWidget, QRDialog from .qrtextedit import ShowQRTextEdit, ScanQRTextEdit @@ -2504,7 +2507,6 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError): for addr, pk in pklist.items(): transaction.writerow(["%34s"%addr,pk]) else: - import json f.write(json.dumps(pklist, indent = 4)) def do_import_labels(self): diff --git a/electrum/gui/qt/password_dialog.py b/electrum/gui/qt/password_dialog.py index ecaf781c..66b3f51b 100644 --- a/electrum/gui/qt/password_dialog.py +++ b/electrum/gui/qt/password_dialog.py @@ -23,16 +23,19 @@ # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -from PyQt5.QtCore import Qt -from PyQt5.QtGui import * -from PyQt5.QtWidgets import * -from electrum.i18n import _ -from .util import * import re import math +from PyQt5.QtCore import Qt +from PyQt5.QtGui import * +from PyQt5.QtWidgets import * + +from electrum.i18n import _ from electrum.plugin import run_hook +from .util import * + + def check_password_strength(password): ''' diff --git a/electrum/gui/qt/paytoedit.py b/electrum/gui/qt/paytoedit.py index d97b4d94..cbd9fde0 100644 --- a/electrum/gui/qt/paytoedit.py +++ b/electrum/gui/qt/paytoedit.py @@ -23,10 +23,11 @@ # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -from PyQt5.QtGui import * import re from decimal import Decimal +from PyQt5.QtGui import * + from electrum import bitcoin from electrum.util import bfh from electrum.transaction import TxOutput @@ -40,6 +41,7 @@ RE_ALIAS = '(.*?)\s*\<([0-9A-Za-z]{1,})\>' frozen_style = "QWidget { background-color:none; border:none;}" normal_style = "QPlainTextEdit { }" + class PayToEdit(CompletionTextEdit, ScanQRTextEdit): def __init__(self, win): diff --git a/electrum/gui/qt/qrcodewidget.py b/electrum/gui/qt/qrcodewidget.py index dc444d6a..5482854b 100644 --- a/electrum/gui/qt/qrcodewidget.py +++ b/electrum/gui/qt/qrcodewidget.py @@ -1,3 +1,5 @@ +import os +import qrcode from PyQt5.QtCore import * from PyQt5.QtGui import * @@ -5,9 +7,6 @@ import PyQt5.QtGui as QtGui from PyQt5.QtWidgets import ( QApplication, QVBoxLayout, QTextEdit, QHBoxLayout, QPushButton, QWidget) -import os -import qrcode - import electrum from electrum.i18n import _ from .util import WindowModalDialog diff --git a/electrum/gui/qt/qrtextedit.py b/electrum/gui/qt/qrtextedit.py index 6f3d1e05..09d5e2ce 100644 --- a/electrum/gui/qt/qrtextedit.py +++ b/electrum/gui/qt/qrtextedit.py @@ -1,10 +1,10 @@ - -from electrum.i18n import _ -from electrum.plugin import run_hook from PyQt5.QtGui import * from PyQt5.QtCore import * from PyQt5.QtWidgets import QFileDialog +from electrum.i18n import _ +from electrum.plugin import run_hook + from .util import ButtonsTextEdit, MessageBoxMixin, ColorScheme diff --git a/electrum/gui/qt/qrwindow.py b/electrum/gui/qt/qrwindow.py index 9abcc26f..c4c3963d 100644 --- a/electrum/gui/qt/qrwindow.py +++ b/electrum/gui/qt/qrwindow.py @@ -41,6 +41,7 @@ else: column_index = 4 + class QR_Window(QWidget): def __init__(self, win): diff --git a/electrum/gui/qt/request_list.py b/electrum/gui/qt/request_list.py index 7b55c5d8..19ec5970 100644 --- a/electrum/gui/qt/request_list.py +++ b/electrum/gui/qt/request_list.py @@ -23,13 +23,15 @@ # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +from PyQt5.QtGui import * +from PyQt5.QtCore import * +from PyQt5.QtWidgets import QTreeWidgetItem, QMenu + from electrum.i18n import _ from electrum.util import format_time, age from electrum.plugin import run_hook from electrum.paymentrequest import PR_UNKNOWN -from PyQt5.QtGui import * -from PyQt5.QtCore import * -from PyQt5.QtWidgets import QTreeWidgetItem, QMenu + from .util import MyTreeWidget, pr_tooltips, pr_icons diff --git a/electrum/gui/qt/transaction_dialog.py b/electrum/gui/qt/transaction_dialog.py index 4e8e6e67..db215149 100644 --- a/electrum/gui/qt/transaction_dialog.py +++ b/electrum/gui/qt/transaction_dialog.py @@ -38,7 +38,6 @@ from electrum.bitcoin import base_encode from electrum.i18n import _ from electrum.plugin import run_hook from electrum import simple_config - from electrum.util import bfh from electrum.transaction import SerializationError diff --git a/electrum/gui/qt/utxo_list.py b/electrum/gui/qt/utxo_list.py index f75ffb37..5985d9c8 100644 --- a/electrum/gui/qt/utxo_list.py +++ b/electrum/gui/qt/utxo_list.py @@ -22,9 +22,11 @@ # ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -from .util import * + from electrum.i18n import _ +from .util import * + class UTXOList(MyTreeWidget): filter_columns = [0, 2] # Address, Label diff --git a/electrum/gui/stdio.py b/electrum/gui/stdio.py index aeb43408..f4a86173 100644 --- a/electrum/gui/stdio.py +++ b/electrum/gui/stdio.py @@ -1,15 +1,18 @@ from decimal import Decimal -_ = lambda x:x -#from i18n import _ +import getpass +import datetime + from electrum import WalletStorage, Wallet from electrum.util import format_satoshis, set_verbosity from electrum.bitcoin import is_address, COIN, TYPE_ADDRESS from electrum.transaction import TxOutput -import getpass, datetime + +_ = lambda x:x # i18n # minimal fdisk like gui for console usage # written by rofl0r, with some bits stolen from the text gui (ncurses) + class ElectrumGui: def __init__(self, config, daemon, plugins): diff --git a/electrum/gui/text.py b/electrum/gui/text.py index af1db4d1..ed3faa0b 100644 --- a/electrum/gui/text.py +++ b/electrum/gui/text.py @@ -1,5 +1,8 @@ -import tty, sys -import curses, datetime, locale +import tty +import sys +import curses +import datetime +import locale from decimal import Decimal import getpass @@ -15,7 +18,6 @@ from electrum.interface import deserialize_server _ = lambda x:x - class ElectrumGui: def __init__(self, config, daemon, plugins): From deda6535e0a39fbd692276f95d0d979408a832ac Mon Sep 17 00:00:00 2001 From: SomberNight Date: Tue, 25 Sep 2018 19:22:37 +0200 Subject: [PATCH 05/12] bump min aiorpcx to 0.8.2 --- contrib/requirements/requirements.txt | 2 +- electrum/interface.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/contrib/requirements/requirements.txt b/contrib/requirements/requirements.txt index 59fb22b4..854ecdae 100644 --- a/contrib/requirements/requirements.txt +++ b/contrib/requirements/requirements.txt @@ -6,6 +6,6 @@ protobuf dnspython jsonrpclib-pelix qdarkstyle<3.0 -aiorpcx>=0.8.1,<0.9 +aiorpcx>=0.8.2,<0.9 aiohttp aiohttp_socks diff --git a/electrum/interface.py b/electrum/interface.py index ddad78e7..2b3b4454 100644 --- a/electrum/interface.py +++ b/electrum/interface.py @@ -51,8 +51,6 @@ class NotificationSession(ClientSession): self.subscriptions = defaultdict(list) self.cache = {} self.in_flight_requests_semaphore = asyncio.Semaphore(100) - # disable bandwidth limiting (used by superclass): - self.bw_limit = 0 async def handle_request(self, request): # note: if server sends malformed request and we raise, the superclass From c4f3fbaca0cd88fa0f7769a162f1d01bc2e9d0e9 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Tue, 25 Sep 2018 21:23:44 +0200 Subject: [PATCH 06/12] labels: fix potential threading issues also handle --offline --- electrum/plugins/labels/labels.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/electrum/plugins/labels/labels.py b/electrum/plugins/labels/labels.py index 623023cb..8ca63e6d 100644 --- a/electrum/plugins/labels/labels.py +++ b/electrum/plugins/labels/labels.py @@ -54,7 +54,7 @@ class LabelsPlugin(BasePlugin): "walletNonce": nonce, "externalId": self.encode(wallet, item), "encryptedLabel": self.encode(wallet, label)} - asyncio.get_event_loop().create_task(self.do_post_safe("/label", bundle)) + asyncio.run_coroutine_threadsafe(self.do_post_safe("/label", bundle), wallet.network.asyncio_loop) # Caller will write the wallet self.set_nonce(wallet, nonce + 1) @@ -134,12 +134,15 @@ class LabelsPlugin(BasePlugin): await self.pull_thread(wallet, force) def pull(self, wallet, force): + if not wallet.network: raise Exception(_('You are offline.')) return asyncio.run_coroutine_threadsafe(self.pull_thread(wallet, force), wallet.network.asyncio_loop).result() def push(self, wallet): + if not wallet.network: raise Exception(_('You are offline.')) return asyncio.run_coroutine_threadsafe(self.push_thread(wallet), wallet.network.asyncio_loop).result() def start_wallet(self, wallet): + if not wallet.network: return # 'offline' mode nonce = self.get_nonce(wallet) self.print_error("wallet", wallet.basename(), "nonce is", nonce) mpk = wallet.get_fingerprint() @@ -151,11 +154,12 @@ class LabelsPlugin(BasePlugin): wallet_id = hashlib.sha256(mpk).hexdigest() self.wallets[wallet] = (password, iv, wallet_id) # If there is an auth token we can try to actually start syncing - asyncio.get_event_loop().create_task(self.pull_safe_thread(wallet, False)) + asyncio.run_coroutine_threadsafe(self.pull_safe_thread(wallet, False), wallet.network.asyncio_loop) self.proxy = wallet.network.proxy wallet.network.register_callback(self.set_proxy, ['proxy_set']) def stop_wallet(self, wallet): + if not wallet.network: return # 'offline' mode wallet.network.unregister_callback('proxy_set') self.wallets.pop(wallet, None) From 3b9a55fab4165f935b60525e45b5f4138930b78f Mon Sep 17 00:00:00 2001 From: SomberNight Date: Wed, 26 Sep 2018 19:33:12 +0200 Subject: [PATCH 07/12] rerun freeze packages --- .../requirements-binaries.txt | 6 +- .../deterministic-build/requirements-hw.txt | 18 ++--- contrib/deterministic-build/requirements.txt | 78 +++++++++---------- 3 files changed, 48 insertions(+), 54 deletions(-) diff --git a/contrib/deterministic-build/requirements-binaries.txt b/contrib/deterministic-build/requirements-binaries.txt index 1ae2b837..574c71a6 100644 --- a/contrib/deterministic-build/requirements-binaries.txt +++ b/contrib/deterministic-build/requirements-binaries.txt @@ -37,9 +37,9 @@ PyQt5==5.10.1 \ --hash=sha256:4db7113f464c733a99fcb66c4c093a47cf7204ad3f8b3bda502efcc0839ac14b \ --hash=sha256:9c17ab3974c1fc7bbb04cc1c9dae780522c0ebc158613f3025fccae82227b5f7 \ --hash=sha256:f6035baa009acf45e5f460cf88f73580ad5dc0e72330029acd99e477f20a5d61 -setuptools==40.2.0 \ - --hash=sha256:47881d54ede4da9c15273bac65f9340f8929d4f0213193fa7894be384f2dcfa6 \ - --hash=sha256:ea3796a48a207b46ea36a9d26de4d0cc87c953a683a7b314ea65d666930ea8e6 +setuptools==40.4.3 \ + --hash=sha256:acbc5740dd63f243f46c2b4b8e2c7fd92259c2ddb55a4115b16418a2ed371b15 \ + --hash=sha256:ce4137d58b444bac11a31d4e0c1805c69d89e8ed4e91fde1999674ecc2f6f9ff SIP==4.19.8 \ --hash=sha256:09f9a4e6c28afd0bafedb26ffba43375b97fe7207bd1a0d3513f79b7d168b331 \ --hash=sha256:105edaaa1c8aa486662226360bd3999b4b89dd56de3e314d82b83ed0587d8783 \ diff --git a/contrib/deterministic-build/requirements-hw.txt b/contrib/deterministic-build/requirements-hw.txt index 361258e4..10ad43ca 100644 --- a/contrib/deterministic-build/requirements-hw.txt +++ b/contrib/deterministic-build/requirements-hw.txt @@ -9,9 +9,9 @@ chardet==3.0.4 \ ckcc-protocol==0.7.2 \ --hash=sha256:31ee5178cfba8895eb2a6b8d06dc7830b51461a0ff767a670a64707c63e6b264 \ --hash=sha256:498db4ccdda018cd9f40210f5bd02ddcc98e7df583170b2eab4035c86c3cc03b -click==6.7 \ - --hash=sha256:29f99fc6125fbc931b758dc053b3114e55c77a6e4c6c3a2674a2dc986016381d \ - --hash=sha256:f15516df478d5a56180fbf80e68f206010e6d160fc39fa508b65e035fd75130b +click==7.0 \ + --hash=sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13 \ + --hash=sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7 Cython==0.28.5 \ --hash=sha256:022592d419fc754509d0e0461eb2958dbaa45fb60d51c8a61778c58994edbe36 \ --hash=sha256:07659f4c57582104d9486c071de512fbd7e087a3a630535298442cc0e20a3f5a \ @@ -102,9 +102,9 @@ requests==2.19.1 \ safet==0.1.4 \ --hash=sha256:522c257910f9472e9c77c487425ed286f6721c314653e232bc41c6cedece1bb1 \ --hash=sha256:b152874acdc89ff0c8b2d680bfbf020b3e53527c2ad3404489dd61a548aa56a1 -setuptools==40.2.0 \ - --hash=sha256:47881d54ede4da9c15273bac65f9340f8929d4f0213193fa7894be384f2dcfa6 \ - --hash=sha256:ea3796a48a207b46ea36a9d26de4d0cc87c953a683a7b314ea65d666930ea8e6 +setuptools==40.4.3 \ + --hash=sha256:acbc5740dd63f243f46c2b4b8e2c7fd92259c2ddb55a4115b16418a2ed371b15 \ + --hash=sha256:ce4137d58b444bac11a31d4e0c1805c69d89e8ed4e91fde1999674ecc2f6f9ff six==1.11.0 \ --hash=sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9 \ --hash=sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb @@ -114,9 +114,9 @@ trezor==0.10.2 \ urllib3==1.23 \ --hash=sha256:a68ac5e15e76e7e5dd2b8f94007233e01effe3e50e8daddf69acfd81cb686baf \ --hash=sha256:b5725a0bd4ba422ab0e66e89e030c806576753ea3ee08554382c14e685d117b5 -websocket-client==0.52.0 \ - --hash=sha256:03763384c530b331ec3822d0b52ffdc28c3aeb8a900ac8c98b2ceea3128a7b4e \ - --hash=sha256:3c9924675eaf0b27ae22feeeab4741bb4149b94820bd3a143eeaf8b62f64d821 +websocket-client==0.53.0 \ + --hash=sha256:c42b71b68f9ef151433d6dcc6a7cb98ac72d2ad1e3a74981ca22bc5d9134f166 \ + --hash=sha256:f5889b1d0a994258cfcbc8f2dc3e457f6fc7b32a8d74873033d12e4eab4bdf63 wheel==0.31.1 \ --hash=sha256:0a2e54558a0628f2145d2fc822137e322412115173e8a2ddbe1c9024338ae83c \ --hash=sha256:80044e51ec5bbf6c894ba0bc48d26a8c20a9ba629f4ca19ea26ecfcf87685f5f diff --git a/contrib/deterministic-build/requirements.txt b/contrib/deterministic-build/requirements.txt index 45ca2c6a..21d8086e 100644 --- a/contrib/deterministic-build/requirements.txt +++ b/contrib/deterministic-build/requirements.txt @@ -23,9 +23,9 @@ aiohttp==3.4.4 \ --hash=sha256:f3df52362be39908f9c028a65490fae0475e4898b43a03d8aa29d1e765b45e07 aiohttp_socks==0.1.6 \ --hash=sha256:943148a3797ba9ffb6df6ddb006ffdd40538885b410589d589bda42a8e8bcd5a -aiorpcX==0.7.3 \ - --hash=sha256:24dd4fe2f65f743cb74c8626570470e325bb777bb66d1932e7d2965ae71d1164 \ - --hash=sha256:5120ca40beef6b6a45d3a7055e343815401385dc607da2fd93baca2762c8a97d +aiorpcX==0.8.2 \ + --hash=sha256:980d1d85a831688163ad087a1c1a88b6695a06e5e9914824676bab4251b2b1f2 \ + --hash=sha256:e53ff8917a87843875526be1261d80171f5ad09187917ff29dfdc003c1526a65 async_timeout==3.0.0 \ --hash=sha256:474d4bc64cee20603e225eb1ece15e248962958b45a3648a9f5cc29e827a610c \ --hash=sha256:b3c0ddc416736619bd4a95ca31de8da6920c3b9a140c64dbef2b2fa7bf521287 @@ -52,36 +52,36 @@ idna_ssl==1.1.0 \ jsonrpclib-pelix==0.3.1 \ --hash=sha256:5417b1508d5a50ec64f6e5b88907f111155d52607b218ff3ba9a777afb2e49e3 \ --hash=sha256:bd89a6093bc4d47dc8a096197aacb827359944a4533be5193f3845f57b9f91b4 -multidict==4.4.0 \ - --hash=sha256:112eeeddd226af681dc82b756ed34aa7b6d98f9c4a15760050298c21d715473d \ - --hash=sha256:13b64ecb692effcabc5e29569ba9b5eb69c35112f990a16d6833ec3a9d9f8ec0 \ - --hash=sha256:1725373fb8f18c2166f8e0e5789851ccf98453c849b403945fa4ef59a16ca44e \ - --hash=sha256:2061a50b7cae60a1f987503a995b2fc38e47027a937a355a124306ed9c629041 \ - --hash=sha256:35b062288a9a478f627c520fd27983160fc97591017d170f966805b428d17e07 \ - --hash=sha256:467b134bcc227b91b8e2ef8d2931f28b50bf7eb7a04c0403d102ded22e66dbfc \ - --hash=sha256:475a3ece8bb450e49385414ebfae7f8fdb33f62f1ac0c12935c1cfb1b7c1076a \ - --hash=sha256:49b885287e227a24545a1126d9ac17ae43138610713dc6219b781cc0ad5c6dfc \ - --hash=sha256:4c95b2725592adb5c46642be2875c1234c32af841732c5504c17726b92082021 \ - --hash=sha256:4ea7ed00f4be0f7335c9a2713a65ac3d986be789ce5ebc10821da9664cbe6b85 \ - --hash=sha256:5e2d5e1d999e941b4a626aea46bdc4206877cf727107fdaa9d46a8a773a6e49b \ - --hash=sha256:8039c520ef7bb9ec7c3db3df14c570be6362f43c200ae9854d2422d4ffe175a4 \ - --hash=sha256:81459a0ebcca09c1fcb8fe887ed13cf267d9b60fe33718fc5fd1a2a1ab49470a \ - --hash=sha256:847c3b7b9ca3268e883685dc1347a4d09f84de7bd7597310044d847590447492 \ - --hash=sha256:8551d1db45f0ca4e8ec99130767009a29a4e0dc6558a4a6808491bcd3472d325 \ - --hash=sha256:8fa7679ffe615e0c1c7b80946ab4194669be74848719adf2d7867b5e861eb073 \ - --hash=sha256:a42a36f09f0f907579ff0fde547f2fde8a739a69efe4a2728835979d2bb5e17b \ - --hash=sha256:a5fcad0070685c5b2d04b468bf5f4c735f5c176432f495ad055fcc4bc0a79b23 \ - --hash=sha256:ae22195b2a7494619b73c01129ddcddc0dfaa9e42727404b1d9a77253da3f420 \ - --hash=sha256:b360e82bdbbd862e1ce2a41cc3bbd0ab614350e813ca74801b34aac0f73465aa \ - --hash=sha256:b96417899344c5e96bef757f4963a72d02e52653a4e0f99bbea3a531cedac59f \ - --hash=sha256:b9e921140b797093edfc13ac08dc2a4fd016dd711dc42bb0e1aaf180e48425a7 \ - --hash=sha256:c5022b94fc330e6d177f3eb38097fb52c7df96ca0e04842c068cf0d9fc38b1e6 \ - --hash=sha256:cf2b117f2a8d951638efc7592fb72d3eeb2d38cc2194c26ba7f00e7190451d92 \ - --hash=sha256:d79620b542d9d0e23ae9790ca2fe44f1af40ffad9936efa37bd14954bc3e2818 \ - --hash=sha256:e2860691c11d10dac7c91bddae44f6211b3da4122d9a2ebb509c2247674d6070 \ - --hash=sha256:e3a293553715afecf7e10ea02da40593f9d7f48fe48a74fc5dd3ce08a0c46188 \ - --hash=sha256:e465be3fe7e992e5a6e16731afa6f41cb6ca53afccb4f28ea2fa6457783edf15 \ - --hash=sha256:e6d27895ef922bc859d969452f247bfbe5345d9aba69b9c8dbe1ea7704f0c5d9 +multidict==4.4.2 \ + --hash=sha256:05eeab69bf2b0664644c62bd92fabb045163e5b8d4376a31dfb52ce0210ced7b \ + --hash=sha256:0c85880efa7cadb18e3b5eef0aa075dc9c0a3064cbbaef2e20be264b9cf47a64 \ + --hash=sha256:136f5a4a6a4adeacc4dc820b8b22f0a378fb74f326e259c54d1817639d1d40a0 \ + --hash=sha256:14906ad3347c7d03e9101749b16611cf2028547716d0840838d3c5e2b3b0f2d3 \ + --hash=sha256:1ade4a3b71b1bf9e90c5f3d034a87fe4949c087ef1f6cd727fdd766fe8bbd121 \ + --hash=sha256:22939a00a511a59f9ecc0158b8db728afef57975ce3782b3a265a319d05b9b12 \ + --hash=sha256:2b86b02d872bc5ba5b3a4530f6a7ba0b541458ab4f7c1429a12ac326231203f7 \ + --hash=sha256:3c11e92c3dfc321014e22fb442bc9eb70e01af30d6ce442026b0c35723448c66 \ + --hash=sha256:4ba3bd26f282b201fdbce351f1c5d17ceb224cbedb73d6e96e6ce391b354aacc \ + --hash=sha256:4c6e78d042e93751f60672989efbd6a6bc54213ed7ff695fff82784bbb9ea035 \ + --hash=sha256:4d80d1901b89cc935a6cf5b9fd89df66565272722fe2e5473168927a9937e0ca \ + --hash=sha256:4fcf71d33178a00cc34a57b29f5dab1734b9ce0f1c97fb34666deefac6f92037 \ + --hash=sha256:52f7670b41d4b4d97866ebc38121de8bcb9813128b7c4942b07794d08193c0ab \ + --hash=sha256:5368e2b7649a26b7253c6c9e53241248aab9da49099442f5be238fde436f18c9 \ + --hash=sha256:5bb65fbb48999044938f0c0508e929b14a9b8bf4939d8263e9ea6691f7b54663 \ + --hash=sha256:60672bb5577472800fcca1ac9dae232d1461db9f20f055184be8ce54b0052572 \ + --hash=sha256:669e9be6d148fc0283f53e17dd140cde4dc7c87edac8319147edd5aa2a830771 \ + --hash=sha256:6a0b7a804e8d1716aa2c72e73210b48be83d25ba9ec5cf52cf91122285707bb1 \ + --hash=sha256:79034ea3da3cf2a815e3e52afdc1f6c1894468c98bdce5d2546fa2342585497f \ + --hash=sha256:79247feeef6abcc11137ad17922e865052f23447152059402fc320f99ff544bb \ + --hash=sha256:81671c2049e6bf42c7fd11a060f8bc58f58b7b3d6f3f951fc0b15e376a6a5a98 \ + --hash=sha256:82ac4a5cb56cc9280d4ae52c2d2ebcd6e0668dd0f9ef17f0a9d7c82bd61e24fa \ + --hash=sha256:9436267dbbaa49dad18fbbb54f85386b0f5818d055e7b8e01d219661b6745279 \ + --hash=sha256:94e4140bb1343115a1afd6d84ebf8fca5fb7bfb50e1c2cbd6f2fb5d3117ef102 \ + --hash=sha256:a2cab366eae8a0ffe0813fd8e335cf0d6b9bb6c5227315f53bb457519b811537 \ + --hash=sha256:a596019c3eafb1b0ae07db9f55a08578b43c79adb1fe1ab1fd818430ae59ee6f \ + --hash=sha256:e8848ae3cd6a784c29fae5055028bee9bffcc704d8bcad09bd46b42b44a833e2 \ + --hash=sha256:e8a048bfd7d5a280f27527d11449a509ddedf08b58a09a24314828631c099306 \ + --hash=sha256:f6dd28a0ac60e2426a6918f36f1b4e2620fc785a0de7654cd206ba842eee57fd pip==18.0 \ --hash=sha256:070e4bf493c7c2c9f6a08dd797dd3c066d64074c38e9e8a0fb4e6541f266d96c \ --hash=sha256:a0e11645ee37c90b40c46d607070c4fd583e2cd46231b1c06e389c5e814eed76 @@ -103,8 +103,6 @@ protobuf==3.6.1 \ --hash=sha256:fcfc907746ec22716f05ea96b7f41597dfe1a1c088f861efb8a0d4f4196a6f10 pyaes==1.6.1 \ --hash=sha256:02c1b1405c38d3c370b085fb952dd8bea3fadcee6411ad99f312cc129c536d8f -PySocks==1.6.8 \ - --hash=sha256:3fe52c55890a248676fd69dc9e3c4e811718b777834bcaab7a8125cf9deac672 QDarkStyle==2.5.4 \ --hash=sha256:3eb60922b8c4d9cedecb6897ca4c9f8a259d81bdefe5791976ccdf12432de1f0 \ --hash=sha256:51331fc6490b38c376e6ba8d8c814320c8d2d1c2663055bc396321a7c28fa8be @@ -114,16 +112,12 @@ qrcode==6.0 \ requests==2.19.1 \ --hash=sha256:63b52e3c866428a224f97cab011de738c36aec0185aa91cfacd418b5d58911d1 \ --hash=sha256:ec22d826a36ed72a7358ff3fe56cbd4ba69dd7a6718ffd450ff0e9df7a47ce6a -setuptools==40.2.0 \ - --hash=sha256:47881d54ede4da9c15273bac65f9340f8929d4f0213193fa7894be384f2dcfa6 \ - --hash=sha256:ea3796a48a207b46ea36a9d26de4d0cc87c953a683a7b314ea65d666930ea8e6 +setuptools==40.4.3 \ + --hash=sha256:acbc5740dd63f243f46c2b4b8e2c7fd92259c2ddb55a4115b16418a2ed371b15 \ + --hash=sha256:ce4137d58b444bac11a31d4e0c1805c69d89e8ed4e91fde1999674ecc2f6f9ff six==1.11.0 \ --hash=sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9 \ --hash=sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb -typing==3.6.6 \ - --hash=sha256:4027c5f6127a6267a435201981ba156de91ad0d1d98e9ddc2aa173453453492d \ - --hash=sha256:57dcf675a99b74d64dacf6fba08fb17cf7e3d5fdff53d4a30ea2a5e7e52543d4 \ - --hash=sha256:a4c8473ce11a65999c8f59cb093e70686b6c84c98df58c1dae9b3b196089858a urllib3==1.23 \ --hash=sha256:a68ac5e15e76e7e5dd2b8f94007233e01effe3e50e8daddf69acfd81cb686baf \ --hash=sha256:b5725a0bd4ba422ab0e66e89e030c806576753ea3ee08554382c14e685d117b5 From 6b8ad2d1263124fcc157536bd41c219736d552ae Mon Sep 17 00:00:00 2001 From: SomberNight Date: Thu, 27 Sep 2018 18:01:25 +0200 Subject: [PATCH 08/12] fix some CLI/RPC commands --- electrum/commands.py | 11 +++++----- electrum/interface.py | 4 +++- electrum/network.py | 46 ++++++++++++++++++++++++++++++++++++---- electrum/synchronizer.py | 5 +++-- 4 files changed, 54 insertions(+), 12 deletions(-) diff --git a/electrum/commands.py b/electrum/commands.py index 19f45d6e..03f7d33a 100644 --- a/electrum/commands.py +++ b/electrum/commands.py @@ -181,7 +181,7 @@ class Commands: walletless server query, results are not checked by SPV. """ sh = bitcoin.address_to_scripthash(address) - return self.network.get_history_for_scripthash(sh) + return self.network.run_from_another_thread(self.network.get_history_for_scripthash(sh)) @command('w') def listunspent(self): @@ -199,7 +199,7 @@ class Commands: is a walletless server query, results are not checked by SPV. """ sh = bitcoin.address_to_scripthash(address) - return self.network.listunspent_for_scripthash(sh) + return self.network.run_from_another_thread(self.network.listunspent_for_scripthash(sh)) @command('') def serialize(self, jsontx): @@ -322,7 +322,7 @@ class Commands: server query, results are not checked by SPV. """ sh = bitcoin.address_to_scripthash(address) - out = self.network.get_balance_for_scripthash(sh) + out = self.network.run_from_another_thread(self.network.get_balance_for_scripthash(sh)) out["confirmed"] = str(Decimal(out["confirmed"])/COIN) out["unconfirmed"] = str(Decimal(out["unconfirmed"])/COIN) return out @@ -331,7 +331,7 @@ class Commands: def getmerkle(self, txid, height): """Get Merkle branch of a transaction included in a block. Electrum uses this to verify transactions (Simple Payment Verification).""" - return self.network.get_merkle_for_transaction(txid, int(height)) + return self.network.run_from_another_thread(self.network.get_merkle_for_transaction(txid, int(height))) @command('n') def getservers(self): @@ -517,7 +517,7 @@ class Commands: if self.wallet and txid in self.wallet.transactions: tx = self.wallet.transactions[txid] else: - raw = self.network.get_transaction(txid) + raw = self.network.run_from_another_thread(self.network.get_transaction(txid)) if raw: tx = Transaction(raw) else: @@ -637,6 +637,7 @@ class Commands: @command('n') def notify(self, address, URL): """Watch an address. Every time the address changes, a http POST is sent to the URL.""" + raise NotImplementedError() # TODO this method is currently broken def callback(x): import urllib.request headers = {'content-type':'application/json'} diff --git a/electrum/interface.py b/electrum/interface.py index 2b3b4454..78c114f6 100644 --- a/electrum/interface.py +++ b/electrum/interface.py @@ -76,7 +76,7 @@ class NotificationSession(ClientSession): super().send_request(*args, **kwargs), timeout) except asyncio.TimeoutError as e: - raise GracefulDisconnect('request timed out: {}'.format(args)) from e + raise RequestTimedOut('request timed out: {}'.format(args)) from e async def subscribe(self, method, params, queue): # note: until the cache is written for the first time, @@ -105,6 +105,7 @@ class NotificationSession(ClientSession): class GracefulDisconnect(Exception): pass +class RequestTimedOut(GracefulDisconnect): pass class ErrorParsingSSLCert(Exception): pass class ErrorGettingSSLCertFromServer(Exception): pass @@ -140,6 +141,7 @@ class Interface(PrintError): self._requested_chunks = set() self.network = network self._set_proxy(proxy) + self.session = None self.tip_header = None self.tip = 0 diff --git a/electrum/network.py b/electrum/network.py index f54c9e06..a449b0ae 100644 --- a/electrum/network.py +++ b/electrum/network.py @@ -45,7 +45,7 @@ from .bitcoin import COIN from . import constants from . import blockchain from .blockchain import Blockchain, HEADER_SIZE -from .interface import Interface, serialize_server, deserialize_server +from .interface import Interface, serialize_server, deserialize_server, RequestTimedOut from .version import PROTOCOL_VERSION from .simple_config import SimpleConfig @@ -638,13 +638,34 @@ class Network(PrintError): with b.lock: b.update_size() - async def get_merkle_for_transaction(self, tx_hash, tx_height): + def best_effort_reliable(func): + async def make_reliable_wrapper(self, *args, **kwargs): + for i in range(10): + iface = self.interface + session = iface.session if iface else None + if not session: + # no main interface; try again + await asyncio.sleep(0.1) + continue + try: + return await func(self, *args, **kwargs) + except RequestTimedOut: + if self.interface != iface: + # main interface changed; try again + continue + raise + raise Exception('no interface to do request on... gave up.') + return make_reliable_wrapper + + @best_effort_reliable + async def get_merkle_for_transaction(self, tx_hash: str, tx_height: int) -> dict: return await self.interface.session.send_request('blockchain.transaction.get_merkle', [tx_hash, tx_height]) + @best_effort_reliable async def broadcast_transaction(self, tx, timeout=10): try: out = await self.interface.session.send_request('blockchain.transaction.broadcast', [str(tx)], timeout=timeout) - except asyncio.TimeoutError as e: + except RequestTimedOut as e: return False, "error: operation timed out" except Exception as e: return False, "error: " + str(e) @@ -653,10 +674,27 @@ class Network(PrintError): return False, "error: " + out return True, out + @best_effort_reliable async def request_chunk(self, height, tip=None, *, can_return_early=False): return await self.interface.request_chunk(height, tip=tip, can_return_early=can_return_early) - def blockchain(self): + @best_effort_reliable + async def get_transaction(self, tx_hash: str) -> str: + return await self.interface.session.send_request('blockchain.transaction.get', [tx_hash]) + + @best_effort_reliable + async def get_history_for_scripthash(self, sh: str) -> List[dict]: + return await self.interface.session.send_request('blockchain.scripthash.get_history', [sh]) + + @best_effort_reliable + async def listunspent_for_scripthash(self, sh: str) -> List[dict]: + return await self.interface.session.send_request('blockchain.scripthash.listunspent', [sh]) + + @best_effort_reliable + async def get_balance_for_scripthash(self, sh: str) -> dict: + return await self.interface.session.send_request('blockchain.scripthash.get_balance', [sh]) + + def blockchain(self) -> Blockchain: interface = self.interface if interface and interface.blockchain is not None: self.blockchain_index = interface.blockchain.forkpoint diff --git a/electrum/synchronizer.py b/electrum/synchronizer.py index 568d2f84..cb84f963 100644 --- a/electrum/synchronizer.py +++ b/electrum/synchronizer.py @@ -51,6 +51,7 @@ class Synchronizer(PrintError): ''' def __init__(self, wallet): self.wallet = wallet + self.network = wallet.network self.asyncio_loop = wallet.network.asyncio_loop self.requested_tx = {} self.requested_histories = {} @@ -86,7 +87,7 @@ class Synchronizer(PrintError): # request address history self.requested_histories[addr] = status h = address_to_scripthash(addr) - result = await self.session.send_request("blockchain.scripthash.get_history", [h]) + result = await self.network.get_history_for_scripthash(h) self.print_error("receiving history", addr, len(result)) hashes = set(map(lambda item: item['tx_hash'], result)) hist = list(map(lambda item: (item['tx_hash'], item['height']), result)) @@ -125,7 +126,7 @@ class Synchronizer(PrintError): await group.spawn(self._get_transaction, tx_hash) async def _get_transaction(self, tx_hash): - result = await self.session.send_request('blockchain.transaction.get', [tx_hash]) + result = await self.network.get_transaction(tx_hash) tx = Transaction(result) try: tx.deserialize() From 4984890265f0f8a83c8bca23be318a02abb7f8b2 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Thu, 27 Sep 2018 20:04:36 +0200 Subject: [PATCH 09/12] follow-up prev: make best_effort_reliable react faster to disconnects --- electrum/interface.py | 2 ++ electrum/network.py | 17 +++++++++-------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/electrum/interface.py b/electrum/interface.py index 78c114f6..34ac8ef4 100644 --- a/electrum/interface.py +++ b/electrum/interface.py @@ -132,6 +132,7 @@ class Interface(PrintError): def __init__(self, network, server, config_path, proxy): self.exception = None self.ready = asyncio.Future() + self.got_disconnected = asyncio.Future() self.server = server self.host, self.port, self.protocol = deserialize_server(self.server) self.port = int(self.port) @@ -246,6 +247,7 @@ class Interface(PrintError): self.print_error("disconnecting gracefully. {}".format(e)) finally: await self.network.connection_down(self.server) + self.got_disconnected.set_result(1) return wrapper_func @aiosafe diff --git a/electrum/network.py b/electrum/network.py index a449b0ae..22a44e18 100644 --- a/electrum/network.py +++ b/electrum/network.py @@ -216,7 +216,7 @@ class Network(PrintError): # kick off the network. interface is the main server we are currently # communicating with. interfaces is the set of servers we are connecting # to or have an ongoing connection with - self.interface = None + self.interface = None # type: Interface self.interfaces = {} self.auto_connect = self.config.get('auto_connect', True) self.connecting = set() @@ -647,13 +647,14 @@ class Network(PrintError): # no main interface; try again await asyncio.sleep(0.1) continue - try: - return await func(self, *args, **kwargs) - except RequestTimedOut: - if self.interface != iface: - # main interface changed; try again - continue - raise + success_fut = asyncio.ensure_future(func(self, *args, **kwargs)) + disconnected_fut = asyncio.shield(iface.got_disconnected) + await asyncio.wait([success_fut, disconnected_fut], return_when=asyncio.FIRST_COMPLETED) + if success_fut.done(): + if success_fut.exception(): + raise success_fut.exception() + return success_fut.result() + # otherwise; try again raise Exception('no interface to do request on... gave up.') return make_reliable_wrapper From 3e2c5e8656dbbaa46405bec94bf4e58e83d4a113 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Thu, 27 Sep 2018 21:15:07 +0200 Subject: [PATCH 10/12] network.best_effort_reliable: force DC if req times out; retry on new iface --- electrum/interface.py | 1 - electrum/network.py | 27 +++++++++++++++++++-------- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/electrum/interface.py b/electrum/interface.py index 34ac8ef4..ac5541db 100644 --- a/electrum/interface.py +++ b/electrum/interface.py @@ -130,7 +130,6 @@ def serialize_server(host: str, port: Union[str, int], protocol: str) -> str: class Interface(PrintError): def __init__(self, network, server, config_path, proxy): - self.exception = None self.ready = asyncio.Future() self.got_disconnected = asyncio.Future() self.server = server diff --git a/electrum/network.py b/electrum/network.py index 22a44e18..65d0774f 100644 --- a/electrum/network.py +++ b/electrum/network.py @@ -642,17 +642,28 @@ class Network(PrintError): async def make_reliable_wrapper(self, *args, **kwargs): for i in range(10): iface = self.interface - session = iface.session if iface else None - if not session: - # no main interface; try again + # retry until there is a main interface + if not iface: await asyncio.sleep(0.1) - continue + continue # try again + # wait for it to be usable + iface_ready = iface.ready + iface_disconnected = iface.got_disconnected + await asyncio.wait([iface_ready, iface_disconnected], return_when=asyncio.FIRST_COMPLETED) + if not iface_ready.done() or iface_ready.cancelled(): + await asyncio.sleep(0.1) + continue # try again + # try actual request success_fut = asyncio.ensure_future(func(self, *args, **kwargs)) - disconnected_fut = asyncio.shield(iface.got_disconnected) - await asyncio.wait([success_fut, disconnected_fut], return_when=asyncio.FIRST_COMPLETED) - if success_fut.done(): + await asyncio.wait([success_fut, iface_disconnected], return_when=asyncio.FIRST_COMPLETED) + if success_fut.done() and not success_fut.cancelled(): if success_fut.exception(): - raise success_fut.exception() + try: + raise success_fut.exception() + except RequestTimedOut: + await iface.close() + await iface_disconnected + continue # try again return success_fut.result() # otherwise; try again raise Exception('no interface to do request on... gave up.') From 12e79ecd6075e81c69f545066c8b5b1d9057a003 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Thu, 27 Sep 2018 21:44:18 +0200 Subject: [PATCH 11/12] qt tx dialog: make input/output fields expand based on Electron-Cash/Electron-Cash@169c13721147a5c7d2727062f1d4c72863080cec --- electrum/gui/qt/transaction_dialog.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/electrum/gui/qt/transaction_dialog.py b/electrum/gui/qt/transaction_dialog.py index db215149..d2affb6c 100644 --- a/electrum/gui/qt/transaction_dialog.py +++ b/electrum/gui/qt/transaction_dialog.py @@ -116,8 +116,6 @@ class TxDialog(QDialog, MessageBoxMixin): self.add_io(vbox) - vbox.addStretch(1) - self.sign_button = b = QPushButton(_("Sign")) b.clicked.connect(self.sign) @@ -296,10 +294,9 @@ class TxDialog(QDialog, MessageBoxMixin): def format_amount(amt): return self.main_window.format_amount(amt, whitespaces=True) - i_text = QTextEdit() + i_text = QTextEditWithDefaultSize() i_text.setFont(QFont(MONOSPACE_FONT)) i_text.setReadOnly(True) - i_text.setMaximumHeight(100) cursor = i_text.textCursor() for x in self.tx.inputs(): if x['type'] == 'coinbase': @@ -318,10 +315,9 @@ class TxDialog(QDialog, MessageBoxMixin): vbox.addWidget(i_text) vbox.addWidget(QLabel(_("Outputs") + ' (%d)'%len(self.tx.outputs()))) - o_text = QTextEdit() + o_text = QTextEditWithDefaultSize() o_text.setFont(QFont(MONOSPACE_FONT)) o_text.setReadOnly(True) - o_text.setMaximumHeight(100) cursor = o_text.textCursor() for addr, v in self.tx.get_outputs(): cursor.insertText(addr, text_format(addr)) @@ -330,3 +326,8 @@ class TxDialog(QDialog, MessageBoxMixin): cursor.insertText(format_amount(v), ext) cursor.insertBlock() vbox.addWidget(o_text) + + +class QTextEditWithDefaultSize(QTextEdit): + def sizeHint(self): + return QSize(0, 100) From 071bc27016125f214be0f433c3a0cb2ed73ed0d6 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Fri, 28 Sep 2018 02:47:36 +0200 Subject: [PATCH 12/12] setup.py: rm deprecated 'imp'. dedupe min py version --- setup.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index 9da14c01..363e419a 100755 --- a/setup.py +++ b/setup.py @@ -5,23 +5,30 @@ import os import sys import platform -import imp +import importlib.util import argparse import subprocess from setuptools import setup, find_packages from setuptools.command.install import install +MIN_PYTHON_VERSION = "3.6" +_min_python_version_tuple = tuple(map(int, (MIN_PYTHON_VERSION.split(".")))) + + +if sys.version_info[:3] < _min_python_version_tuple: + sys.exit("Error: Electrum requires Python version >= {}...".format(MIN_PYTHON_VERSION)) + with open('contrib/requirements/requirements.txt') as f: requirements = f.read().splitlines() with open('contrib/requirements/requirements-hw.txt') as f: requirements_hw = f.read().splitlines() -version = imp.load_source('version', 'electrum/version.py') - -if sys.version_info[:3] < (3, 6, 0): - sys.exit("Error: Electrum requires Python version >= 3.6.0...") +# load version.py; needlessly complicated alternative to "imp.load_source": +version_spec = importlib.util.spec_from_file_location('version', 'electrum/version.py') +version_module = version = importlib.util.module_from_spec(version_spec) +version_spec.loader.exec_module(version_module) data_files = [] @@ -71,7 +78,7 @@ class CustomInstallCommand(install): setup( name="Electrum", version=version.ELECTRUM_VERSION, - python_requires='>=3.6', + python_requires='>={}'.format(MIN_PYTHON_VERSION), install_requires=requirements, extras_require=extras_require, packages=[