Merge branch 'release-0.7.18'

RELEASE-NOTES

@@ -1,7 +1,14 @@
+version 0.7.18
+--------------
+
+- better IRC support for tor (valesi)
+- issues: suppressed some uninteresting socket logging to fix #52
+- mempool: fixed small memory leak
+
 version 0.7.17
 --------------
 
-- upped read buffer limit to 1000000 bytes.
+- upped read buffer limit to 1,000,000 bytes.
 
 version 0.7.16
 --------------
@@ -12,7 +19,7 @@ version 0.7.15
 --------------
 
 The following meta variables in your banner file are now replaced in
-addition to $VERSION described in the notes to 0.17.11.  If you type
+addition to $VERSION described in the notes to 0.7.11.  If you type
 getnetworkinfo in your daemon's debug console you will see what they
 are based on:
 
@@ -60,7 +67,7 @@ version 0.7.11
 
 - increased MAX_SEND default value to 1 million bytes so as to be able
   to serve large historical transactions of up to ~500K in size.  The
-  MAX_SEND floor remains at 350,000 bytes so you can reduct it if you
+  MAX_SEND floor remains at 350,000 bytes so you can reduce it if you
   wish.  To serve any historical transaction for bitcoin youd should
   set this to around 2,000,100 bytes (one byte becomes 2 ASCII hex chars)
 - issue #46: fix reorgs for coinbase-only blocks.  We would not distinguish

docs/ENV-NOTES

@@ -86,12 +86,21 @@ BANDWIDTH_LIMIT  - per-session periodic bandwith usage limit in bytes.
 
 If you want IRC connectivity to advertise your node:
 
-IRC             - set to anything non-empty
+IRC                 - set to anything non-empty
-IRC_NICK        - the nick to use when connecting to IRC.  The default is a
+IRC_NICK            - the nick to use when connecting to IRC.  The default is a
-                  hash of REPORT_HOST.  Either way 'E_' will be prepended.
+                      hash of REPORT_HOST.  Either way 'E_' will be prepended.
-REPORT_HOST     - the host to advertise.  Defaults to HOST.
+REPORT_HOST         - the host to advertise.  Defaults to HOST.
-REPORT_SSL_PORT - the SSL port to advertise.  Defaults to SSL_PORT.
+REPORT_TCP_PORT     - the TCP port to advertise.  Defaults to TCP_PORT.
-REPORT_TCP_PORT - the TCP port to advertise.  Defaults to TCP_PORT.
+                      '0' disables publishing the port.
+REPORT_SSL_PORT     - the SSL port to advertise.  Defaults to SSL_PORT.
+                      '0' disables publishing the port.
+REPORT_HOST_TOR     - Tor .onion address to advertise.  Appends '_tor" to nick.
+REPORT_TCP_PORT_TOR - the TCP port to advertise for Tor.  Defaults to
+                      REPORT_TCP_PORT, unless it is '0', then use TCP_PORT.
+                      '0' disables publishing the port.
+REPORT_SSL_PORT_TOR - the SSL port to advertise for Tor.  Defaults to
+                      REPORT_SSL_PORT, unless it is '0', then use SSL_PORT.
+                      '0' disables publishing the port.
 
 If synchronizing from the Genesis block your performance might change
 by tweaking the following cache variables.  Cache size is only checked

electrumx_server.py

@@ -21,6 +21,7 @@ from server.protocol import ServerManager
 
 SUPPRESS_MESSAGES = [
     'Fatal read error on socket transport',
+    'Fatal write error on socket transport',
 ]
 
 def main_loop():

lib/jsonrpc.py

@@ -311,7 +311,7 @@ class JSONRPC(asyncio.Protocol, LoggedClass):
     async def json_notification(self, message):
         try:
             method, params = self.method_and_params(message)
-        except RCPError:
+        except self.RPCError:
             pass
         else:
             await self.handle_notification(method, params)
@@ -336,7 +336,7 @@ class JSONRPC(asyncio.Protocol, LoggedClass):
 
     def raise_unknown_method(self, method):
         '''Respond to a request with an unknown method.'''
-        raise self.RPCError('unknown method: "{}"'.format(method),
+        raise self.RPCError("unknown method: '{}'".format(method),
                             self.METHOD_NOT_FOUND)
 
     # --- derived classes are intended to override these functions

server/env.py

@@ -51,11 +51,20 @@ class Env(LoggedClass):
         self.max_session_subs = self.integer('MAX_SESSION_SUBS', 50000)
         self.bandwidth_limit = self.integer('BANDWIDTH_LIMIT', 2000000)
         # IRC
+        self.irc = self.default('IRC', False)
+        self.irc_nick = self.default('IRC_NICK', None)
         self.report_tcp_port = self.integer('REPORT_TCP_PORT', self.tcp_port)
         self.report_ssl_port = self.integer('REPORT_SSL_PORT', self.ssl_port)
         self.report_host = self.default('REPORT_HOST', self.host)
-        self.irc_nick = self.default('IRC_NICK', None)
+        self.report_tcp_port_tor = self.integer('REPORT_TCP_PORT_TOR',
-        self.irc = self.default('IRC', False)
+                                                self.report_tcp_port
+                                                if self.report_tcp_port else
+                                                self.tcp_port)
+        self.report_ssl_port_tor = self.integer('REPORT_SSL_PORT_TOR',
+                                                self.report_ssl_port
+                                                if self.report_ssl_port else
+                                                self.ssl_port)
+        self.report_host_tor = self.default('REPORT_HOST_TOR', None)
         # Debugging
         self.force_reorg = self.integer('FORCE_REORG', 0)
 

server/irc.py

@@ -20,14 +20,6 @@ from lib.hash import double_sha256
 from lib.util import LoggedClass
 
 
-def port_text(letter, port, default):
-    if not port:
-        return ''
-    if port == default:
-        return letter
-    return letter + str(port)
-
-
 class IRC(LoggedClass):
 
     Peer = namedtuple('Peer', 'ip_addr host ports')
@@ -37,22 +29,29 @@ class IRC(LoggedClass):
 
     def __init__(self, env):
         super().__init__()
-        tcp_text = port_text('t', env.report_tcp_port, 50001)
-        ssl_text = port_text('s', env.report_ssl_port, 50002)
-        # If this isn't something the client expects you won't appear
-        # in the client's network dialog box
         self.env = env
-        version = '1.0'
+
-        self.real_name = '{} v{} {} {}'.format(env.report_host, version,
+        # If this isn't something a peer or client expects
-                                               tcp_text, ssl_text)
+        # then you won't appear in the client's network dialog box
+        irc_address = (env.coin.IRC_SERVER, env.coin.IRC_PORT)
+        self.channel = env.coin.IRC_CHANNEL
         self.prefix = env.coin.IRC_PREFIX
+
+        self.clients = []
         self.nick = '{}{}'.format(self.prefix,
                                   env.irc_nick if env.irc_nick else
                                   double_sha256(env.report_host.encode())
                                   [:5].hex())
-        self.channel = env.coin.IRC_CHANNEL
+        self.clients.append( IrcClient(irc_address, self.nick,
-        self.irc_server = env.coin.IRC_SERVER
+                                       env.report_host,
-        self.irc_port = env.coin.IRC_PORT
+                                       env.report_tcp_port,
+                                       env.report_ssl_port) )
+        if env.report_host_tor:
+            self.clients.append( IrcClient(irc_address, self.nick + '_tor',
+                                           env.report_host_tor,
+                                           env.report_tcp_port_tor,
+                                           env.report_ssl_port_tor) )
+
         self.peer_regexp = re.compile('({}[^!]*)!'.format(self.prefix))
         self.peers = {}
 
@@ -72,20 +71,23 @@ class IRC(LoggedClass):
     async def join(self):
         import irc.client as irc_client
 
-        self.logger.info('joining IRC with nick "{}" and real name "{}"'
-                         .format(self.nick, self.real_name))
-
         reactor = irc_client.Reactor()
         for event in ['welcome', 'join', 'quit', 'kick', 'whoreply',
                       'namreply', 'disconnect']:
             reactor.add_global_handler(event, getattr(self, 'on_' + event))
 
-        connection = reactor.server()
+        # Note: Multiple nicks in same channel will trigger duplicate events
+        for client in self.clients:
+            client.connection = reactor.server()
+
         while True:
             try:
-                connection.connect(self.irc_server, self.irc_port,
+                for client in self.clients:
-                                   self.nick, ircname=self.real_name)
+                    self.logger.info('Joining IRC in {} as "{}" with '
-                connection.set_keepalive(60)
+                                     'real name "{}"'
+                                     .format(self.channel, client.nick,
+                                             client.realname))
+                    client.connect()
                 while True:
                     reactor.process_once()
                     await asyncio.sleep(2)
@@ -155,3 +157,36 @@ class IRC(LoggedClass):
             self.peers[nick] = peer
         except IndexError:
             pass
+
+
+class IrcClient(LoggedClass):
+
+    VERSION = '1.0'
+    DEFAULT_PORTS = {'t': 50001, 's': 50002}
+
+    def __init__(self, irc_address, nick, host, tcp_port, ssl_port):
+        super().__init__()
+        self.irc_host, self.irc_port = irc_address
+        self.nick = nick
+        self.realname = self.create_realname(host, tcp_port, ssl_port)
+        self.connection = None
+
+    def connect(self, keepalive=60):
+        '''Connect this client to its IRC server'''
+        self.connection.connect(self.irc_host, self.irc_port, self.nick,
+                                ircname=self.realname)
+        self.connection.set_keepalive(keepalive)
+
+    @classmethod
+    def create_realname(cls, host, tcp_port, ssl_port):
+        def port_text(letter, port):
+            if not port:
+                return ''
+            if port == cls.DEFAULT_PORTS.get(letter):
+                return ' ' + letter
+            else:
+                return ' ' + letter + str(port)
+
+        tcp = port_text('t', tcp_port)
+        ssl = port_text('s', ssl_port)
+        return '{} v{}{}{}'.format(host, cls.VERSION, tcp, ssl)

server/protocol.py

@@ -77,6 +77,16 @@ class MemPool(util.LoggedClass):
         touched = set()
         missing_utxos = []
 
+        def drop_tx(hex_hash):
+            txin_pairs, txout_pairs, _u = self.txs.pop(hex_hash)
+            hash168s = set(hash168 for hash168, value in txin_pairs)
+            hash168s.update(hash168 for hash168, value in txout_pairs)
+            for hash168 in hash168s:
+                self.hash168s[hash168].remove(hex_hash)
+                if not self.hash168s[hash168]:
+                    del self.hash168s[hash168]
+            touched.update(hash168s)
+
         initial = self.count < 0
         if initial:
             self.logger.info('beginning import of {:,d} mempool txs'
@@ -85,14 +95,7 @@ class MemPool(util.LoggedClass):
         # Remove gone items
         gone = set(self.txs).difference(hex_hashes)
         for hex_hash in gone:
-            txin_pairs, txout_pairs, unconfirmed = self.txs.pop(hex_hash)
+            drop_tx(hex_hash)
-            hash168s = set(hash168 for hash168, value in txin_pairs)
-            hash168s.update(hash168 for hash168, value in txout_pairs)
-            for hash168 in hash168s:
-                self.hash168s[hash168].remove(hex_hash)
-                if not self.hash168s[hash168]:
-                    del self.hash168s[hash168]
-            touched.update(hash168s)
 
         # Get the raw transactions for the new hashes.  Ignore the
         # ones the daemon no longer has (it will return None).  Put
@@ -119,7 +122,7 @@ class MemPool(util.LoggedClass):
             if n % 20 == 0:
                 await asyncio.sleep(0)
             txout_pairs = [txout_pair(txout) for txout in tx.outputs]
-            self.txs[hex_hash] = (None, txout_pairs, None)
+            self.txs[hex_hash] = ([], txout_pairs, None)
 
         def txin_info(txin):
             hex_hash = hash_to_str(txin.prev_hash)
@@ -153,7 +156,7 @@ class MemPool(util.LoggedClass):
                 # it's harmless - next time the mempool is refreshed
                 # they'll either be cleaned up or the UTXOs will no
                 # longer be missing.
-                del self.txs[hex_hash]
+                drop_tx(hex_hash)
                 continue
             self.txs[hex_hash] = (txin_pairs, txout_pairs, any(unconfs))
 

server/version.py

@@ -1 +1 @@
-VERSION = "ElectrumX 0.7.17"
+VERSION = "ElectrumX 0.7.18"

tests/test_storage.py

@@ -22,8 +22,8 @@ def db(tmpdir, request):
     cwd = os.getcwd()
     os.chdir(str(tmpdir))
     db = open_db("db", request.param, False)
-    os.chdir(cwd)
     yield db
+    os.chdir(cwd)
     # Make sure all the locks and handles are closed
     del db
     gc.collect()
@@ -67,3 +67,10 @@ def test_iterator_reverse(db):
             (b"abc" + str.encode(str(i)), str.encode(str(i))) for
             i in reversed(range(5))
         ]
+
+
+def test_close(db):
+    db.put(b"a", b"b")
+    db.close()
+    db = open_db("db", db.__class__.__name__, False)
+    assert db.get(b"a") == b"b"