New protocol: (#330)

- add method mempool.get_fee_histogram
- bump protocol version to 1.2

docs/PROTOCOL.rst

@@ -774,6 +774,20 @@ Subscribe to a script hash.
     [**scripthash**, **status**]
 
 
+mempool.get_fee_histogram
+=========================
+
+Return a histogram of the fee rates paid by transactions in the memory
+pool, weighted by transaction size.
+
+The histogram is an array of (fee, vsize) values, where vsize_n is the
+cumulative virtual size of mempool transactions with a fee rate in the
+interval [fee_(n-1), fee_n)], and fee_(n-1) > fee_n.
+
+Fee intervals may have variable size. The choice of appropriate
+intervals is currently not part of the protocol.
+
+
 server.add_peer
 ===============
 

server/controller.py

@@ -825,6 +825,14 @@ class Controller(ServerBase):
         number = self.non_negative_integer(number)
         return await self.daemon_request('estimatefee', [number])
 
+    def mempool_get_fee_histogram(self):
+        '''Memory pool fee histogram.
+
+        TODO: The server should detect and discount transactions that
+        never get mined when they should.
+        '''
+        return self.mempool.get_fee_histogram()
+
     async def relayfee(self):
         '''The minimum fee a low-priority tx must pay in order to be accepted
         to the daemon's memory pool.'''

server/mempool.py

@@ -25,7 +25,7 @@ class MemPool(util.LoggedClass):
 
     To that end we maintain the following maps:
 
-       tx_hash -> (txin_pairs, txout_pairs)
+       tx_hash -> (txin_pairs, txout_pairs, tx_fee, tx_size)
        hashX   -> set of all tx hashes in which the hashX appears
 
     A pair is a (hashX, value) tuple.  tx hashes are hex strings.
@@ -42,6 +42,9 @@ class MemPool(util.LoggedClass):
         self.txs = {}
         self.hashXs = defaultdict(set)  # None can be a key
         self.synchronized_event = asyncio.Event()
+        self.fee_histogram = defaultdict(int)
+        self.compact_fee_histogram = []
+        self.histogram_time = 0
 
     def _resync_daemon_hashes(self, unprocessed, unfetched):
         '''Re-sync self.txs with the list of hashes in the daemon's mempool.
@@ -52,6 +55,7 @@ class MemPool(util.LoggedClass):
         txs = self.txs
         hashXs = self.hashXs
         touched = self.touched
+        fee_hist = self.fee_histogram
 
         hashes = self.daemon.cached_mempool_hashes()
         gone = set(txs).difference(hashes)
@@ -60,7 +64,11 @@ class MemPool(util.LoggedClass):
             unprocessed.pop(hex_hash, None)
             item = txs.pop(hex_hash)
             if item:
-                txin_pairs, txout_pairs = item
+                txin_pairs, txout_pairs, tx_fee, tx_size = item
+                fee_rate = tx_fee // tx_size
+                fee_hist[fee_rate] -= tx_size
+                if fee_hist[fee_rate] == 0:
+                    fee_hist.pop(fee_rate)
                 tx_hashXs = set(hashX for hashX, value in txin_pairs)
                 tx_hashXs.update(hashX for hashX, value in txout_pairs)
                 for hashX in tx_hashXs:
@@ -138,6 +146,7 @@ class MemPool(util.LoggedClass):
     def _async_process_some(self, limit):
         pending = []
         txs = self.txs
+        fee_hist = self.fee_histogram
 
         async def process(unprocessed):
             nonlocal pending
@@ -160,10 +169,13 @@ class MemPool(util.LoggedClass):
             pending.extend(deferred)
             hashXs = self.hashXs
             touched = self.touched
-            for hex_hash, in_out_pairs in result.items():
+            for hex_hash, item in result.items():
                 if hex_hash in txs:
-                    txs[hex_hash] = in_out_pairs
+                    txs[hex_hash] = item
-                    for hashX, value in itertools.chain(*in_out_pairs):
+                    txin_pairs, txout_pairs, tx_fee, tx_size = item
+                    fee_rate = tx_fee // tx_size
+                    fee_hist[fee_rate] += tx_size
+                    for hashX, value in itertools.chain(txin_pairs, txout_pairs):
                         touched.add(hashX)
                         hashXs[hashX].add(hex_hash)
 
@@ -209,7 +221,7 @@ class MemPool(util.LoggedClass):
         for tx_hash, raw_tx in raw_tx_map.items():
             if tx_hash not in txs:
                 continue
-            tx = deserializer(raw_tx).read_tx()
+            tx, tx_size = deserializer(raw_tx).read_tx_and_vsize()
 
             # Convert the tx outputs into (hashX, value) pairs
             txout_pairs = [(script_hashX(txout.pk_script), txout.value)
@@ -219,7 +231,7 @@ class MemPool(util.LoggedClass):
             txin_pairs = [(hash_to_str(txin.prev_hash), txin.prev_idx)
                           for txin in tx.inputs]
 
-            pending.append((tx_hash, txin_pairs, txout_pairs))
+            pending.append((tx_hash, txin_pairs, txout_pairs, tx_size))
 
         # Now process what we can
         result = {}
@@ -229,7 +241,7 @@ class MemPool(util.LoggedClass):
             if self.stop:
                 break
 
-            tx_hash, old_txin_pairs, txout_pairs = item
+            tx_hash, old_txin_pairs, txout_pairs, tx_size = item
             if tx_hash not in txs:
                 continue
 
@@ -259,7 +271,10 @@ class MemPool(util.LoggedClass):
             if mempool_missing:
                 deferred.append(item)
             else:
-                result[tx_hash] = (txin_pairs, txout_pairs)
+                # Compute fee
+                tx_fee = (sum(v for hashX, v in txin_pairs) -
+                          sum(v for hashX, v in txout_pairs))
+                result[tx_hash] = (txin_pairs, txout_pairs, tx_fee, tx_size)
 
         return result, deferred
 
@@ -290,9 +305,7 @@ class MemPool(util.LoggedClass):
             item = self.txs.get(hex_hash)
             if not item or not raw_tx:
                 continue
-            txin_pairs, txout_pairs = item
+            txin_pairs, txout_pairs, tx_fee, tx_size = item
-            tx_fee = (sum(v for hashX, v in txin_pairs) -
-                      sum(v for hashX, v in txout_pairs))
             tx = deserializer(raw_tx).read_tx()
             unconfirmed = any(hash_to_str(txin.prev_hash) in self.txs
                               for txin in tx.inputs)
@@ -325,7 +338,36 @@ class MemPool(util.LoggedClass):
         # hashXs is a defaultdict
         if hashX in self.hashXs:
             for hex_hash in self.hashXs[hashX]:
-                txin_pairs, txout_pairs = self.txs[hex_hash]
+                txin_pairs, txout_pairs, tx_fee, tx_size = self.txs[hex_hash]
                 value -= sum(v for h168, v in txin_pairs if h168 == hashX)
                 value += sum(v for h168, v in txout_pairs if h168 == hashX)
         return value
+
+    def get_fee_histogram(self):
+        now = time.time()
+        if now > self.histogram_time + 30:
+            self.update_compact_histogram()
+            self.histogram_time = now
+        return self.compact_fee_histogram
+
+    def update_compact_histogram(self):
+        # For efficiency, get_fees returns a compact histogram with
+        # variable bin size.  The compact histogram is an array of
+        # (fee, vsize) values.  vsize_n is the cumulative virtual size
+        # of mempool transactions with a fee rate in the interval
+        # [fee_(n-1), fee_n)], and fee_(n-1) > fee_n. Fee intervals
+        # are chosen so as to create tranches that contain at least
+        # 100kb of transactions
+        l = list(reversed(sorted(self.fee_histogram.items())))
+        out = []
+        size = 0
+        r = 0
+        binsize = 100000
+        for fee, s in l:
+            size += s
+            if size + r > binsize:
+                out.append((fee, size))
+                r += size - binsize
+                size = 0
+                binsize *= 1.1
+        self.compact_fee_histogram = out

server/session.py

@@ -449,6 +449,12 @@ class ElectrumX(SessionBase):
                 'blockchain.transaction.get': controller.transaction_get,
             })
 
+        if ptuple >= (1, 2):
+            # New handler as of 1.2
+            handlers.update({
+                'mempool.get_fee_histogram': controller.mempool_get_fee_histogram,
+            })
+
         self.electrumx_handlers = handlers
 
     def request_handler(self, method):

server/version.py

@@ -2,4 +2,4 @@
 
 VERSION = 'ElectrumX 1.2.1'
 PROTOCOL_MIN = '0.9'
-PROTOCOL_MAX = '1.1'
+PROTOCOL_MAX = '1.2'