Merge branch 'release-0.10.1'

README.rst

@@ -142,13 +142,19 @@ version prior to the release of 1.0.
 ChangeLog
 =========
 
+Version 0.10.1
+--------------
+
+* Includes what should be a fix for issue `#94#` - stale references to
+  old sessions.  This would effectively memory and network handles.
+
 Version 0.10.0
 --------------
 
-Major rewrite of DB layer as per issue `#72`_.  UTXOs and history are
+* Major rewrite of DB layer as per issue `#72`_.  UTXOs and history
-now indexed by the hash of the pay to script, making the index
+  are now indexed by the hash of the pay to script, making the index
-independent of the address scheme.  The history and UTXO DBs are also
+  independent of the address scheme.
-now separate.
+* The history and UTXO DBs are also now separate.
 
 Together these changes reduce the size of the DB by approximately 15%
 and the time taken to sync from genesis by about 20%.
@@ -329,4 +335,5 @@ Version 0.9.0
 .. _#88: https://github.com/kyuupichan/electrumx/issues/88
 .. _#89: https://github.com/kyuupichan/electrumx/issues/89
 .. _#93: https://github.com/kyuupichan/electrumx/issues/93
+.. _#94: https://github.com/kyuupichan/electrumx/issues/94
 .. _docs/HOWTO.rst: https://github.com/kyuupichan/electrumx/blob/master/docs/HOWTO.rst

samples/daemontools/env/HIST_MB

@@ -1 +0,0 @@
-300

samples/daemontools/env/UTXO_MB

@@ -1 +0,0 @@
-1000

samples/systemd/electrumx.conf

@@ -65,31 +65,3 @@
         #Maximum number of address subscriptions across all sessions
     #MAX_SESSION_SUBS = 50000
         #Maximum number of address subscriptions permitted to a single session.
-
-
-#If synchronizing from the Genesis block your performance might change
-#by tweaking the following cache variables.  Cache size is only checked
-#roughly every minute, so the caches can grow beyond the specified
-#size.  Also the Python process is often quite a bit fatter than the
-#combined cache size, because of Python overhead and also because
-#leveldb consumes a lot of memory during UTXO flushing.  So I recommend
-#you set the sum of these to nothing over half your available physical
-#RAM:
-
-    #HIST_MB = 300
-        #amount of history cache, in MB, to retain before flushing to
-        #disk.  Default is 300; probably no benefit being much larger
-        #as history is append-only and not searched.
-
-    #UTXO_MB = 1000
-       #amount of UTXO and history cache, in MB, to retain before
-       #flushing to disk.  Default is 1000.  This may be too large
-       #for small boxes or too small for machines with lots of RAM.
-       #Larger caches generally perform better as there is
-       #significant searching of the UTXO cache during indexing.
-       #However, I don't see much benefit in my tests pushing this
-       #too high, and in fact performance begins to fall.  My
-       #machine has 24GB RAM; the slow down is probably because of
-       #leveldb caching and Python GC effects.  However this may be
-       #very dependent on hardware and you may have different
-       #results.

server/controller.py

@@ -53,8 +53,9 @@ class Controller(util.LoggedClass):
         self.irc = IRC(env)
         self.env = env
         self.servers = {}
+        # Map of session to the key of its list in self.groups
         self.sessions = {}
-        self.groups = defaultdict(set)
+        self.groups = defaultdict(list)
         self.txs_sent = 0
         self.next_log_sessions = 0
         self.state = self.CATCHING_UP
@@ -108,9 +109,10 @@ class Controller(util.LoggedClass):
     def session_priority(self, session):
         if isinstance(session, LocalRPC):
             return 0
-        group_bandwidth = sum(s.bandwidth_used for s in self.sessions[session])
+        gid = self.sessions[session]
+        group_bandwidth = sum(s.bandwidth_used for s in self.groups[gid])
         return 1 + (bisect_left(self.bands, session.bandwidth_used)
-                    + bisect_left(self.bands, group_bandwidth) + 1) // 2
+                    + bisect_left(self.bands, group_bandwidth)) // 2
 
     def is_deprioritized(self, session):
         return self.session_priority(session) > self.BANDS
@@ -333,9 +335,9 @@ class Controller(util.LoggedClass):
         if now > self.next_stale_check:
             self.next_stale_check = now + 300
             self.clear_stale_sessions()
-        group = self.groups[int(session.start - self.start) // 900]
+        gid = int(session.start - self.start) // 900
-        group.add(session)
+        self.groups[gid].append(session)
-        self.sessions[session] = group
+        self.sessions[session] = gid
         session.log_info('{} {}, {:,d} total'
                          .format(session.kind, session.peername(),
                                  len(self.sessions)))
@@ -350,8 +352,9 @@ class Controller(util.LoggedClass):
     def remove_session(self, session):
         '''Remove a session from our sessions list if there.'''
         if session in self.sessions:
-            group = self.sessions.pop(session)
+            gid = self.sessions.pop(session)
-            group.remove(session)
+            assert gid in self.groups
+            self.groups[gid].remove(session)
             self.subscription_count -= session.sub_count()
 
     def close_session(self, session):
@@ -385,13 +388,16 @@ class Controller(util.LoggedClass):
             self.logger.info('closing stale connections {}'.format(stale))
 
         # Consolidate small groups
-        keys = [k for k, v in self.groups.items() if len(v) <= 4
+        gids = [gid for gid, l in self.groups.items() if len(l) <= 4
-                and sum(session.bandwidth_used for session in v) < 10000]
+                and sum(session.bandwidth_used for session in l) < 10000]
-        if len(keys) > 1:
+        if len(gids) > 1:
-            group = set.union(*(self.groups[key] for key in keys))
+            sessions = sum([self.groups[gid] for gid in gids], [])
-            for key in keys:
+            new_gid = max(gids)
-                del self.groups[key]
+            for gid in gids:
-            self.groups[max(keys)] = group
+                del self.groups[gid]
+            for session in sessions:
+                self.sessions[session] = new_gid
+            self.groups[new_gid] = sessions
 
     def new_subscription(self):
         if self.subscription_count >= self.max_subs:
@@ -457,9 +463,9 @@ class Controller(util.LoggedClass):
     def group_data(self):
         '''Returned to the RPC 'groups' call.'''
         result = []
-        for group_id in sorted(self.groups.keys()):
+        for gid in sorted(self.groups.keys()):
-            sessions = self.groups[group_id]
+            sessions = self.groups[gid]
-            result.append([group_id,
+            result.append([gid,
                            len(sessions),
                            sum(s.bandwidth_used for s in sessions),
                            sum(s.requests_remaining() for s in sessions),

server/version.py

@@ -1 +1 @@
-VERSION = "ElectrumX 0.10.0"
+VERSION = "ElectrumX 0.10.1"