From e9e7ff16ad5e8e8ec78ea07ffd420b2cb809bff2 Mon Sep 17 00:00:00 2001 From: Johann Bauer Date: Sun, 27 Nov 2016 21:44:42 +0100 Subject: [PATCH 1/9] Add tests for `close`. --- tests/test_storage.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/test_storage.py b/tests/test_storage.py index 2c3d429..504f3f5 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -22,8 +22,8 @@ def db(tmpdir, request): cwd = os.getcwd() os.chdir(str(tmpdir)) db = open_db("db", request.param, False) - os.chdir(cwd) yield db + os.chdir(cwd) # Make sure all the locks and handles are closed del db gc.collect() @@ -67,3 +67,10 @@ def test_iterator_reverse(db): (b"abc" + str.encode(str(i)), str.encode(str(i))) for i in reversed(range(5)) ] + + +def test_close(db): + db.put(b"a", b"b") + db.close() + db = open_db("db", db.__class__.__name__, False) + assert db.get(b"a") == b"b" \ No newline at end of file From a412531ccb1de51986953ba48ce9794bee5dc39b Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Wed, 30 Nov 2016 20:52:20 +0900 Subject: [PATCH 2/9] Fix typos in docs --- RELEASE-NOTES | 6 +++--- lib/jsonrpc.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/RELEASE-NOTES b/RELEASE-NOTES index ffe8355..779cae4 100644 --- a/RELEASE-NOTES +++ b/RELEASE-NOTES @@ -1,7 +1,7 @@ version 0.7.17 -------------- -- upped read buffer limit to 1000000 bytes. +- upped read buffer limit to 1,000,000 bytes. version 0.7.16 -------------- @@ -12,7 +12,7 @@ version 0.7.15 -------------- The following meta variables in your banner file are now replaced in -addition to $VERSION described in the notes to 0.17.11. If you type +addition to $VERSION described in the notes to 0.7.11. If you type getnetworkinfo in your daemon's debug console you will see what they are based on: @@ -60,7 +60,7 @@ version 0.7.11 - increased MAX_SEND default value to 1 million bytes so as to be able to serve large historical transactions of up to ~500K in size. The - MAX_SEND floor remains at 350,000 bytes so you can reduct it if you + MAX_SEND floor remains at 350,000 bytes so you can reduce it if you wish. To serve any historical transaction for bitcoin youd should set this to around 2,000,100 bytes (one byte becomes 2 ASCII hex chars) - issue #46: fix reorgs for coinbase-only blocks. We would not distinguish diff --git a/lib/jsonrpc.py b/lib/jsonrpc.py index 1cdc5fa..6329a83 100644 --- a/lib/jsonrpc.py +++ b/lib/jsonrpc.py @@ -311,7 +311,7 @@ class JSONRPC(asyncio.Protocol, LoggedClass): async def json_notification(self, message): try: method, params = self.method_and_params(message) - except RCPError: + except self.RPCError: pass else: await self.handle_notification(method, params) @@ -336,7 +336,7 @@ class JSONRPC(asyncio.Protocol, LoggedClass): def raise_unknown_method(self, method): '''Respond to a request with an unknown method.''' - raise self.RPCError('unknown method: "{}"'.format(method), + raise self.RPCError("unknown method: '{}'".format(method), self.METHOD_NOT_FOUND) # --- derived classes are intended to override these functions From 95c848a7201ab47e635bf54263777937998df8cb Mon Sep 17 00:00:00 2001 From: Shane Moore Date: Wed, 30 Nov 2016 23:50:20 -0800 Subject: [PATCH 3/9] Add IRC option to publish Tor address --- docs/ENV-NOTES | 7 +++-- server/env.py | 1 + server/irc.py | 79 +++++++++++++++++++++++++++++++++++--------------- 3 files changed, 62 insertions(+), 25 deletions(-) diff --git a/docs/ENV-NOTES b/docs/ENV-NOTES index f51004e..32fe2b0 100644 --- a/docs/ENV-NOTES +++ b/docs/ENV-NOTES @@ -90,8 +90,11 @@ IRC - set to anything non-empty IRC_NICK - the nick to use when connecting to IRC. The default is a hash of REPORT_HOST. Either way 'E_' will be prepended. REPORT_HOST - the host to advertise. Defaults to HOST. -REPORT_SSL_PORT - the SSL port to advertise. Defaults to SSL_PORT. +REPORT_HOST_TOR - Tor .onion address to advertise. Uses TCP/SSL_PORT rather + - than REPORT_* ports. REPORT_TCP_PORT - the TCP port to advertise. Defaults to TCP_PORT. + - '0' disables publishing the port for public use. +REPORT_SSL_PORT - the SSL port to advertise. Defaults to SSL_PORT. If synchronizing from the Genesis block your performance might change by tweaking the following cache variables. Cache size is only checked @@ -124,4 +127,4 @@ FORCE_REORG - if set to a positive integer, it will simulate a reorg of the blockchain for that number of blocks on startup. Although it should fail gracefully if set to a value greater than REORG_LIMIT, I do not recommend it as I have - not tried it and there is a chance your DB might corrupt. \ No newline at end of file + not tried it and there is a chance your DB might corrupt. diff --git a/server/env.py b/server/env.py index 58b8b9d..57d931f 100644 --- a/server/env.py +++ b/server/env.py @@ -54,6 +54,7 @@ class Env(LoggedClass): self.report_tcp_port = self.integer('REPORT_TCP_PORT', self.tcp_port) self.report_ssl_port = self.integer('REPORT_SSL_PORT', self.ssl_port) self.report_host = self.default('REPORT_HOST', self.host) + self.report_host_tor = self.default('REPORT_HOST_TOR', None) self.irc_nick = self.default('IRC_NICK', None) self.irc = self.default('IRC', False) # Debugging diff --git a/server/irc.py b/server/irc.py index 7930d07..6365fd1 100644 --- a/server/irc.py +++ b/server/irc.py @@ -20,12 +20,8 @@ from lib.hash import double_sha256 from lib.util import LoggedClass -def port_text(letter, port, default): - if not port: - return '' - if port == default: - return letter - return letter + str(port) +VERSION = '1.0' +DEFAULT_PORTS = {'t': 50001, 's': 50002} class IRC(LoggedClass): @@ -37,22 +33,29 @@ class IRC(LoggedClass): def __init__(self, env): super().__init__() - tcp_text = port_text('t', env.report_tcp_port, 50001) - ssl_text = port_text('s', env.report_ssl_port, 50002) - # If this isn't something the client expects you won't appear - # in the client's network dialog box self.env = env - version = '1.0' - self.real_name = '{} v{} {} {}'.format(env.report_host, version, - tcp_text, ssl_text) + + # If this isn't something a peer or client expects + # then you won't appear in the client's network dialog box + irc_address = (env.coin.IRC_SERVER, env.coin.IRC_PORT) + self.channel = env.coin.IRC_CHANNEL self.prefix = env.coin.IRC_PREFIX + + self.clients = [] self.nick = '{}{}'.format(self.prefix, env.irc_nick if env.irc_nick else double_sha256(env.report_host.encode()) [:5].hex()) - self.channel = env.coin.IRC_CHANNEL - self.irc_server = env.coin.IRC_SERVER - self.irc_port = env.coin.IRC_PORT + self.clients.append( IrcClient(irc_address, self.nick, + env.report_host, + env.report_tcp_port, + env.report_ssl_port) ) + if env.report_host_tor: + self.clients.append( IrcClient(irc_address, self.nick + '_tor', + env.report_host_tor, + env.tcp_port, + env.ssl_port) ) + self.peer_regexp = re.compile('({}[^!]*)!'.format(self.prefix)) self.peers = {} @@ -72,20 +75,23 @@ class IRC(LoggedClass): async def join(self): import irc.client as irc_client - self.logger.info('joining IRC with nick "{}" and real name "{}"' - .format(self.nick, self.real_name)) - reactor = irc_client.Reactor() for event in ['welcome', 'join', 'quit', 'kick', 'whoreply', 'namreply', 'disconnect']: reactor.add_global_handler(event, getattr(self, 'on_' + event)) - connection = reactor.server() + # Note: Multiple nicks in same channel will trigger duplicate events + for client in self.clients: + client.connection = reactor.server() + while True: try: - connection.connect(self.irc_server, self.irc_port, - self.nick, ircname=self.real_name) - connection.set_keepalive(60) + for client in self.clients: + self.logger.info('Joining IRC in {} as "{}" with ' + 'real name "{}"' + .format(self.channel, client.nick, + client.realname)) + client.connect() while True: reactor.process_once() await asyncio.sleep(2) @@ -155,3 +161,30 @@ class IRC(LoggedClass): self.peers[nick] = peer except IndexError: pass + + +class IrcClient(LoggedClass): + + def __init__(self, irc_address, nick, host, tcp_port, ssl_port): + super().__init__() + self.irc_host, self.irc_port = irc_address + self.nick = nick + self.realname = IrcClient.create_realname(host, tcp_port, ssl_port) + self.connection = None + + + def connect(self, keepalive=60): + '''Connect this client to its IRC server''' + self.connection.connect(self.irc_host, self.irc_port, self.nick, + ircname=self.realname) + self.connection.set_keepalive(keepalive) + + + def create_realname(host, tcp_port, ssl_port): + def port_text(letter, port): + return letter if letter in DEFAULT_PORTS + and port == DEFAULT_PORTS[letter] + else letter + str(port) + tcp = ' ' + port_text('t', tcp_port) if tcp_port else '' + ssl = ' ' + port_text('s', ssl_port) if ssl_port else '' + return '{} v{}{}{}'.format(host, VERSION, tcp, ssl) From 33c5bd65aa79df133fd9d6b3d22c89e655925d94 Mon Sep 17 00:00:00 2001 From: Shane Moore Date: Thu, 1 Dec 2016 02:25:21 -0800 Subject: [PATCH 4/9] Fix IRC port bug, more readable --- server/irc.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/server/irc.py b/server/irc.py index 6365fd1..c0d8cc1 100644 --- a/server/irc.py +++ b/server/irc.py @@ -182,9 +182,10 @@ class IrcClient(LoggedClass): def create_realname(host, tcp_port, ssl_port): def port_text(letter, port): - return letter if letter in DEFAULT_PORTS - and port == DEFAULT_PORTS[letter] - else letter + str(port) + if letter in DEFAULT_PORTS and port == DEFAULT_PORTS[letter]: + return letter + else: + return letter + str(port) tcp = ' ' + port_text('t', tcp_port) if tcp_port else '' ssl = ' ' + port_text('s', ssl_port) if ssl_port else '' return '{} v{}{}{}'.format(host, VERSION, tcp, ssl) From 0edff0056de038eb46e0133ef15de48581976732 Mon Sep 17 00:00:00 2001 From: Shane Moore Date: Thu, 1 Dec 2016 03:21:52 -0800 Subject: [PATCH 5/9] Add Tor-specific port options for IRC --- docs/ENV-NOTES | 24 +++++++++++++++--------- server/env.py | 12 ++++++++++-- server/irc.py | 4 ++-- 3 files changed, 27 insertions(+), 13 deletions(-) diff --git a/docs/ENV-NOTES b/docs/ENV-NOTES index 32fe2b0..26048a6 100644 --- a/docs/ENV-NOTES +++ b/docs/ENV-NOTES @@ -86,15 +86,21 @@ BANDWIDTH_LIMIT - per-session periodic bandwith usage limit in bytes. If you want IRC connectivity to advertise your node: -IRC - set to anything non-empty -IRC_NICK - the nick to use when connecting to IRC. The default is a - hash of REPORT_HOST. Either way 'E_' will be prepended. -REPORT_HOST - the host to advertise. Defaults to HOST. -REPORT_HOST_TOR - Tor .onion address to advertise. Uses TCP/SSL_PORT rather - - than REPORT_* ports. -REPORT_TCP_PORT - the TCP port to advertise. Defaults to TCP_PORT. - - '0' disables publishing the port for public use. -REPORT_SSL_PORT - the SSL port to advertise. Defaults to SSL_PORT. +IRC - set to anything non-empty +IRC_NICK - the nick to use when connecting to IRC. The default is a + hash of REPORT_HOST. Either way 'E_' will be prepended. +REPORT_HOST - the host to advertise. Defaults to HOST. +REPORT_TCP_PORT - the TCP port to advertise. Defaults to TCP_PORT. + '0' disables publishing the port. +REPORT_SSL_PORT - the SSL port to advertise. Defaults to SSL_PORT. + '0' disables publishing the port. +REPORT_HOST_TOR - Tor .onion address to advertise. Appends '_tor" to nick. +REPORT_TCP_PORT_TOR - the TCP port to advertise for Tor. Defaults to + REPORT_TCP_PORT, unless it is '0', then use TCP_PORT. + '0' disables publishing the port. +REPORT_SSL_PORT_TOR - the SSL port to advertise for Tor. Defaults to + REPORT_SSL_PORT, unless it is '0', then use SSL_PORT. + '0' disables publishing the port. If synchronizing from the Genesis block your performance might change by tweaking the following cache variables. Cache size is only checked diff --git a/server/env.py b/server/env.py index 57d931f..2b875a4 100644 --- a/server/env.py +++ b/server/env.py @@ -51,12 +51,20 @@ class Env(LoggedClass): self.max_session_subs = self.integer('MAX_SESSION_SUBS', 50000) self.bandwidth_limit = self.integer('BANDWIDTH_LIMIT', 2000000) # IRC + self.irc = self.default('IRC', False) + self.irc_nick = self.default('IRC_NICK', None) self.report_tcp_port = self.integer('REPORT_TCP_PORT', self.tcp_port) self.report_ssl_port = self.integer('REPORT_SSL_PORT', self.ssl_port) self.report_host = self.default('REPORT_HOST', self.host) + self.report_tcp_port_tor = self.integer('REPORT_TCP_PORT_TOR', + self.report_tcp_port + if self.report_tcp_port else + self.tcp_port) + self.report_ssl_port_tor = self.integer('REPORT_SSL_PORT_TOR', + self.report_ssl_port + if self.report_ssl_port else + self.ssl_port) self.report_host_tor = self.default('REPORT_HOST_TOR', None) - self.irc_nick = self.default('IRC_NICK', None) - self.irc = self.default('IRC', False) # Debugging self.force_reorg = self.integer('FORCE_REORG', 0) diff --git a/server/irc.py b/server/irc.py index c0d8cc1..fb73da5 100644 --- a/server/irc.py +++ b/server/irc.py @@ -53,8 +53,8 @@ class IRC(LoggedClass): if env.report_host_tor: self.clients.append( IrcClient(irc_address, self.nick + '_tor', env.report_host_tor, - env.tcp_port, - env.ssl_port) ) + env.report_tcp_port_tor, + env.report_ssl_port_tor) ) self.peer_regexp = re.compile('({}[^!]*)!'.format(self.prefix)) self.peers = {} From 6c95644ae0ed41da103c0fc61e4501b779be80e2 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Fri, 2 Dec 2016 07:22:30 +0900 Subject: [PATCH 6/9] Fix ref leak in mempool updates --- server/protocol.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/server/protocol.py b/server/protocol.py index bae092d..d6c06eb 100644 --- a/server/protocol.py +++ b/server/protocol.py @@ -77,6 +77,16 @@ class MemPool(util.LoggedClass): touched = set() missing_utxos = [] + def drop_tx(hex_hash): + txin_pairs, txout_pairs, _u = self.txs.pop(hex_hash) + hash168s = set(hash168 for hash168, value in txin_pairs) + hash168s.update(hash168 for hash168, value in txout_pairs) + for hash168 in hash168s: + self.hash168s[hash168].remove(hex_hash) + if not self.hash168s[hash168]: + del self.hash168s[hash168] + touched.update(hash168s) + initial = self.count < 0 if initial: self.logger.info('beginning import of {:,d} mempool txs' @@ -85,14 +95,7 @@ class MemPool(util.LoggedClass): # Remove gone items gone = set(self.txs).difference(hex_hashes) for hex_hash in gone: - txin_pairs, txout_pairs, unconfirmed = self.txs.pop(hex_hash) - hash168s = set(hash168 for hash168, value in txin_pairs) - hash168s.update(hash168 for hash168, value in txout_pairs) - for hash168 in hash168s: - self.hash168s[hash168].remove(hex_hash) - if not self.hash168s[hash168]: - del self.hash168s[hash168] - touched.update(hash168s) + drop_tx(hex_hash) # Get the raw transactions for the new hashes. Ignore the # ones the daemon no longer has (it will return None). Put @@ -119,7 +122,7 @@ class MemPool(util.LoggedClass): if n % 20 == 0: await asyncio.sleep(0) txout_pairs = [txout_pair(txout) for txout in tx.outputs] - self.txs[hex_hash] = (None, txout_pairs, None) + self.txs[hex_hash] = ([], txout_pairs, None) def txin_info(txin): hex_hash = hash_to_str(txin.prev_hash) @@ -153,7 +156,7 @@ class MemPool(util.LoggedClass): # it's harmless - next time the mempool is refreshed # they'll either be cleaned up or the UTXOs will no # longer be missing. - del self.txs[hex_hash] + drop_tx(hex_hash) continue self.txs[hex_hash] = (txin_pairs, txout_pairs, any(unconfs)) From 2f9c8dd38a7ec9a4f7176803a45446e4b3e48fe4 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Fri, 2 Dec 2016 07:25:32 +0900 Subject: [PATCH 7/9] Add a message to the suppressed list I hope this fixes #52 --- electrumx_server.py | 1 + 1 file changed, 1 insertion(+) diff --git a/electrumx_server.py b/electrumx_server.py index 940828e..7129a8a 100755 --- a/electrumx_server.py +++ b/electrumx_server.py @@ -21,6 +21,7 @@ from server.protocol import ServerManager SUPPRESS_MESSAGES = [ 'Fatal read error on socket transport', + 'Fatal write error on socket transport', ] def main_loop(): From 7a9e8c7fef7f65262869f498e36588979406e84c Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Fri, 2 Dec 2016 07:43:50 +0900 Subject: [PATCH 8/9] Small tweaks to IRC code --- server/irc.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/server/irc.py b/server/irc.py index fb73da5..694c81a 100644 --- a/server/irc.py +++ b/server/irc.py @@ -20,10 +20,6 @@ from lib.hash import double_sha256 from lib.util import LoggedClass -VERSION = '1.0' -DEFAULT_PORTS = {'t': 50001, 's': 50002} - - class IRC(LoggedClass): Peer = namedtuple('Peer', 'ip_addr host ports') @@ -165,27 +161,32 @@ class IRC(LoggedClass): class IrcClient(LoggedClass): + VERSION = '1.0' + DEFAULT_PORTS = {'t': 50001, 's': 50002} + def __init__(self, irc_address, nick, host, tcp_port, ssl_port): super().__init__() self.irc_host, self.irc_port = irc_address self.nick = nick - self.realname = IrcClient.create_realname(host, tcp_port, ssl_port) + self.realname = self.create_realname(host, tcp_port, ssl_port) self.connection = None - def connect(self, keepalive=60): '''Connect this client to its IRC server''' self.connection.connect(self.irc_host, self.irc_port, self.nick, ircname=self.realname) self.connection.set_keepalive(keepalive) - - def create_realname(host, tcp_port, ssl_port): + @classmethod + def create_realname(cls, host, tcp_port, ssl_port): def port_text(letter, port): - if letter in DEFAULT_PORTS and port == DEFAULT_PORTS[letter]: - return letter + if not port: + return '' + if port == cls.DEFAULT_PORTS.get(letter): + return ' ' + letter else: - return letter + str(port) - tcp = ' ' + port_text('t', tcp_port) if tcp_port else '' - ssl = ' ' + port_text('s', ssl_port) if ssl_port else '' - return '{} v{}{}{}'.format(host, VERSION, tcp, ssl) + return ' ' + letter + str(port) + + tcp = port_text('t', tcp_port) + ssl = port_text('s', ssl_port) + return '{} v{}{}{}'.format(host, cls.VERSION, tcp, ssl) From 35f118edc671a6e23fcd015db057573375d5a6d3 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Fri, 2 Dec 2016 07:49:35 +0900 Subject: [PATCH 9/9] Prepare 0.7.18 --- RELEASE-NOTES | 7 +++++++ server/version.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/RELEASE-NOTES b/RELEASE-NOTES index 779cae4..016f7a6 100644 --- a/RELEASE-NOTES +++ b/RELEASE-NOTES @@ -1,3 +1,10 @@ +version 0.7.18 +-------------- + +- better IRC support for tor (valesi) +- issues: suppressed some uninteresting socket logging to fix #52 +- mempool: fixed small memory leak + version 0.7.17 -------------- diff --git a/server/version.py b/server/version.py index 3a2a92c..71fdbbb 100644 --- a/server/version.py +++ b/server/version.py @@ -1 +1 @@ -VERSION = "ElectrumX 0.7.17" +VERSION = "ElectrumX 0.7.18"