Merge branch 'develop'

README.rst

@@ -16,6 +16,34 @@ Getting Started
 
 See `docs/HOWTO.rst`_.
 
+Features
+========
+
+- Efficient, lightweight reimplementation of electrum-server
+- Efficient synchronization of bitcoin mainnet from Genesis.  Recent
+  hardware should synchronize in well under 24 hours.  The fastest
+  time to height 448k (mid January 2017) reported is under 4h 30m.  On
+  the same hardware JElectrum would take around 4 days and
+  electrum-server probably around 1 month.
+- The full Electrum protocol is implemented.  The only exception is
+  the blockchain.address.get_proof RPC call, which is not used by
+  Electrum GUI clients, and can only be invoked from the command line.
+- Various configurable means of controlling resource consumption and
+  handling denial of service attacks.  These include maximum
+  connection counts, subscription limits per-connection and across all
+  connections, maximum response size, per-session bandwidth limits,
+  and session timeouts.
+- Minimal resource usage once caught up and serving clients; tracking the
+  transaction mempool appears to be the most expensive part.
+- Fully asynchronous processing of new blocks, mempool updates, and
+  client requests.  Busy clients should not noticeably impede other
+  clients' requests and notifications, nor the processing of incoming
+  blocks and mempool updates.
+- Daemon failover.  More than one daemon can be specified, and
+  ElectrumX will failover round-robin style if the current one fails
+  for any reason.
+- Coin abstraction makes compatible altcoin and testnet support easy.
+
 Motivation
 ==========
 
@@ -45,34 +73,6 @@ that could easily be reused for those alts that are reasonably
 compatible with Bitcoin.  Such an abstraction is also useful for
 testnets.
 
-Features
-========
-
-- The full Electrum protocol is implemented.  The only exception is
-  the blockchain.address.get_proof RPC call, which is not used by
-  Electrum GUI clients, and can only be invoked from the command line.
-- Efficient synchronization from Genesis.  Recent hardware should
-  synchronize in well under 24 hours, possibly much faster for recent
-  CPUs or if you have an SSD.  The fastest time to height 439k (mid
-  November 2016) reported is under 5 hours.  For comparison, JElectrum
-  would take around 4 days, and electrum-server probably around 1
-  month, on the same hardware.
-- Various configurable means of controlling resource consumption and
-  handling denial of service attacks.  These include maximum
-  connection counts, subscription limits per-connection and across all
-  connections, maximum response size, per-session bandwidth limits,
-  and session timeouts.
-- Minimal resource usage once caught up and serving clients; tracking the
-  transaction mempool appears to be the most expensive part.
-- Fully asynchronous processing of new blocks, mempool updates, and
-  client requests.  Busy clients should not noticeably impede other
-  clients' requests and notifications, nor the processing of incoming
-  blocks and mempool updates.
-- Daemon failover.  More than one daemon can be specified, and
-  ElectrumX will failover round-robin style if the current one fails
-  for any reason.
-- Coin abstraction makes compatible altcoin and testnet support easy.
-
 Implementation
 ==============
 
@@ -135,6 +135,14 @@ version prior to the release of 1.0.
 ChangeLog
 =========
 
+Version 0.10.17
+---------------
+
+Minor upgrade
+
+* added current daemon URL and uptime to getinfo RPC call
+* altcoin cleanups / fixes (erasmospunk)
+
 Version 0.10.16
 ---------------
 

lib/coins.py

@@ -32,17 +32,18 @@ class CoinError(Exception):
 class Coin(object):
     '''Base class of coin hierarchy.'''
 
-    REORG_LIMIT=200
+    REORG_LIMIT = 200
     # Not sure if these are coin-specific
     RPC_URL_REGEX = re.compile('.+@[^:]+(:[0-9]+)?')
     VALUE_PER_COIN = 100000000
-    CHUNK_SIZE=2016
+    CHUNK_SIZE = 2016
     IRC_SERVER = "irc.freenode.net"
     IRC_PORT = 6667
     HASHX_LEN = 11
     # Peer discovery
-    PEER_DEFAULT_PORTS = {'t':'50001', 's':'50002'}
+    PEER_DEFAULT_PORTS = {'t': '50001', 's': '50002'}
     PEERS = []
+    TX_COUNT_HEIGHT = 0
 
     @classmethod
     def lookup_coin_class(cls, name, net):
@@ -98,11 +99,11 @@ class Coin(object):
     @util.cachedproperty
     def address_handlers(cls):
         return ScriptPubKey.PayToHandlers(
-            address = cls.P2PKH_address_from_hash160,
+            address=cls.P2PKH_address_from_hash160,
-            script_hash = cls.P2SH_address_from_hash160,
+            script_hash=cls.P2SH_address_from_hash160,
-            pubkey = cls.P2PKH_address_from_pubkey,
+            pubkey=cls.P2PKH_address_from_pubkey,
-            unspendable = lambda : None,
+            unspendable=lambda: None,
-            strange = lambda script: None,
+            strange=lambda script: None,
         )
 
     @classmethod
@@ -269,8 +270,8 @@ class Bitcoin(Coin):
     P2PKH_VERBYTE = 0x00
     P2SH_VERBYTE = 0x05
     WIF_BYTE = 0x80
-    GENESIS_HASH=('000000000019d6689c085ae165831e93'
+    GENESIS_HASH = ('000000000019d6689c085ae165831e93'
-                  '4ff763ae46a2a6c172b3f1b60a8ce26f')
+                    '4ff763ae46a2a6c172b3f1b60a8ce26f')
     TX_COUNT = 156335304
     TX_COUNT_HEIGHT = 429972
     TX_PER_BLOCK = 1800
@@ -302,15 +303,15 @@ class BitcoinTestnet(Bitcoin):
     P2PKH_VERBYTE = 0x6f
     P2SH_VERBYTE = 0xc4
     WIF_BYTE = 0xef
-    GENESIS_HASH=('000000000933ea01ad0ee984209779ba'
+    GENESIS_HASH = ('000000000933ea01ad0ee984209779ba'
-                  'aec3ced90fa3f408719526f8d77f4943')
+                    'aec3ced90fa3f408719526f8d77f4943')
     REORG_LIMIT = 2000
     TX_COUNT = 12242438
     TX_COUNT_HEIGHT = 1035428
     TX_PER_BLOCK = 21
     IRC_PREFIX = "ET_"
     RPC_PORT = 18332
-    PEER_DEFAULT_PORTS = {'t':'51001', 's':'51002'}
+    PEER_DEFAULT_PORTS = {'t': '51001', 's': '51002'}
     PEERS = [
         'electrum.akinbo.org s t',
         'he36kyperp3kbuxu.onion s t',
@@ -319,6 +320,7 @@ class BitcoinTestnet(Bitcoin):
         'testnet.not.fyi s t',
     ]
 
+
 class BitcoinTestnetSegWit(BitcoinTestnet):
     '''Bitcoin Testnet for Core bitcoind >= 0.13.1.
 
@@ -343,8 +345,8 @@ class Litecoin(Coin):
     P2PKH_VERBYTE = 0x30
     P2SH_VERBYTE = 0x05
     WIF_BYTE = 0xb0
-    GENESIS_HASH=('12a765e31ffd4059bada1e25190f6e98'
+    GENESIS_HASH = ('12a765e31ffd4059bada1e25190f6e98'
-                  'c99d9714d334efa41a195a7e7e04bfe2')
+                    'c99d9714d334efa41a195a7e7e04bfe2')
     TX_COUNT = 8908766
     TX_COUNT_HEIGHT = 1105256
     TX_PER_BLOCK = 10
@@ -361,7 +363,8 @@ class LitecoinTestnet(Litecoin):
     P2PKH_VERBYTE = 0x6f
     P2SH_VERBYTE = 0xc4
     WIF_BYTE = 0xef
-    # Some details missing...
+    GENESIS_HASH = ('f5ae71e26c74beacc88382716aced69c'
+                    'ddf3dffff24f384e1808905e0188f68f')
 
 
 # Source: namecoin.org
@@ -374,9 +377,11 @@ class Namecoin(Coin):
     P2PKH_VERBYTE = 0x34
     P2SH_VERBYTE = 0x0d
     WIF_BYTE = 0xe4
+    GENESIS_HASH = ('000000000062b72c5e2ceb45fbc8587e'
+                    '807c155b0da735e6483dfba2f0a9c770')
 
 
-class NamecoinTestnet(Coin):
+class NamecoinTestnet(Namecoin):
     NAME = "Namecoin"
     SHORTNAME = "XNM"
     NET = "testnet"
@@ -385,6 +390,7 @@ class NamecoinTestnet(Coin):
     P2PKH_VERBYTE = 0x6f
     P2SH_VERBYTE = 0xc4
     WIF_BYTE = 0xef
+    # TODO add GENESIS_HASH
 
 
 # For DOGE there is disagreement across sites like bip32.org and
@@ -398,9 +404,11 @@ class Dogecoin(Coin):
     P2PKH_VERBYTE = 0x1e
     P2SH_VERBYTE = 0x16
     WIF_BYTE = 0x9e
+    GENESIS_HASH = ('1a91e3dace36e2be3bf030a65679fe82'
+                    '1aa1d6ef92e7c9902eb318182c355691')
 
 
-class DogecoinTestnet(Coin):
+class DogecoinTestnet(Dogecoin):
     NAME = "Dogecoin"
     SHORTNAME = "XDT"
     NET = "testnet"
@@ -409,6 +417,8 @@ class DogecoinTestnet(Coin):
     P2PKH_VERBYTE = 0x71
     P2SH_VERBYTE = 0xc4
     WIF_BYTE = 0xf1
+    GENESIS_HASH = ('bb0a78264637406b6360aad926284d54'
+                    '4d7049f45189db5664f3c4d07350559e')
 
 
 # Source: https://github.com/dashpay/dash
@@ -436,6 +446,7 @@ class Dash(Coin):
         import x11_hash
         return x11_hash.getPoWHash(header)
 
+
 class DashTestnet(Dash):
     SHORTNAME = "tDASH"
     NET = "testnet"
@@ -462,8 +473,8 @@ class Argentum(Coin):
     P2PKH_VERBYTE = 0x17
     P2SH_VERBYTE = 0x05
     WIF_BYTE = 0x97
-    GENESIS_HASH=('88c667bc63167685e4e4da058fffdfe8'
+    GENESIS_HASH = ('88c667bc63167685e4e4da058fffdfe8'
-                  'e007e5abffd6855de52ad59df7bb0bb2')
+                    'e007e5abffd6855de52ad59df7bb0bb2')
     TX_COUNT = 2263089
     TX_COUNT_HEIGHT = 2050260
     TX_PER_BLOCK = 2000
@@ -473,14 +484,14 @@ class Argentum(Coin):
 
 
 class ArgentumTestnet(Argentum):
-   SHORTNAME = "XRG"
+    SHORTNAME = "XRG"
-   NET = "testnet"
+    NET = "testnet"
-   XPUB_VERBYTES = bytes.fromhex("043587cf")
+    XPUB_VERBYTES = bytes.fromhex("043587cf")
-   XPRV_VERBYTES = bytes.fromhex("04358394")
+    XPRV_VERBYTES = bytes.fromhex("04358394")
-   P2PKH_VERBYTE = 0x6f
+    P2PKH_VERBYTE = 0x6f
-   P2SH_VERBYTE = 0xc4
+    P2SH_VERBYTE = 0xc4
-   WIF_BYTE = 0xef
+    WIF_BYTE = 0xef
-   REORG_LIMIT = 2000
+    REORG_LIMIT = 2000
 
 
 class DigiByte(Coin):
@@ -492,8 +503,8 @@ class DigiByte(Coin):
     P2PKH_VERBYTE = 0x1E
     P2SH_VERBYTE = 0x05
     WIF_BYTE = 0x80
-    GENESIS_HASH=('7497ea1b465eb39f1c8f507bc877078f'
+    GENESIS_HASH = ('7497ea1b465eb39f1c8f507bc877078f'
-                  'e016d6fcb6dfad3a64c98dcc6e1e8496')
+                    'e016d6fcb6dfad3a64c98dcc6e1e8496')
     TX_COUNT = 1046018
     TX_COUNT_HEIGHT = 1435000
     TX_PER_BLOCK = 1000
@@ -509,8 +520,8 @@ class DigiByteTestnet(DigiByte):
     P2PKH_VERBYTE = 0x6f
     P2SH_VERBYTE = 0xc4
     WIF_BYTE = 0xef
-    GENESIS_HASH=('b5dca8039e300198e5fe7cd23bdd1728'
+    GENESIS_HASH = ('b5dca8039e300198e5fe7cd23bdd1728'
-                  'e2a444af34c447dbd0916fa3430a68c2')
+                    'e2a444af34c447dbd0916fa3430a68c2')
     IRC_PREFIX = "DET_"
     IRC_CHANNEL = "#electrum-dgb"
     RPC_PORT = 15022

lib/util.py

@@ -57,12 +57,20 @@ class cachedproperty(object):
         return value
 
 
-def formatted_time(t):
+def formatted_time(t, sep=' '):
     '''Return a number of seconds as a string in days, hours, mins and
-    secs.'''
+    maybe secs.'''
     t = int(t)
-    return '{:d}d {:02d}h {:02d}m {:02d}s'.format(
+    fmts = (('{:d}d', 86400), ('{:02d}h', 3600), ('{:02d}m', 60))
-        t // 86400, (t % 86400) // 3600, (t % 3600) // 60, t % 60)
+    parts = []
+    for fmt, n in fmts:
+        val = t // n
+        if parts or val:
+            parts.append(fmt.format(val))
+        t %= n
+    if len(parts) < 3:
+        parts.append('{:02d}s'.format(t))
+    return sep.join(parts)
 
 
 def deep_getsizeof(obj):

server/block_processor.py

@@ -383,26 +383,32 @@ class BlockProcessor(server.db.DB):
 
         # Catch-up stats
         if self.utxo_db.for_sync:
-            daemon_height = self.daemon.cached_height()
             tx_per_sec = int(self.tx_count / self.wall_time)
             this_tx_per_sec = 1 + int(tx_diff / (self.last_flush - last_flush))
-            if self.height > self.coin.TX_COUNT_HEIGHT:
-                tx_est = (daemon_height - self.height) * self.coin.TX_PER_BLOCK
-            else:
-                tx_est = ((daemon_height - self.coin.TX_COUNT_HEIGHT)
-                          * self.coin.TX_PER_BLOCK
-                          + (self.coin.TX_COUNT - self.tx_count))
-
-            # Damp the enthusiasm
-            realism = 2.0 - 0.9 * self.height / self.coin.TX_COUNT_HEIGHT
-            tx_est *= max(realism, 1.0)
-
             self.logger.info('tx/sec since genesis: {:,d}, '
                              'since last flush: {:,d}'
                              .format(tx_per_sec, this_tx_per_sec))
-            self.logger.info('sync time: {}  ETA: {}'
+            if self.coin.TX_COUNT_HEIGHT > 0:
-                             .format(formatted_time(self.wall_time),
+                daemon_height = self.daemon.cached_height()
-                                     formatted_time(tx_est / this_tx_per_sec)))
+                if self.height > self.coin.TX_COUNT_HEIGHT:
+                    tx_est = (daemon_height - self.height) * self.coin.TX_PER_BLOCK
+                else:
+                    tx_est = ((daemon_height - self.coin.TX_COUNT_HEIGHT)
+                              * self.coin.TX_PER_BLOCK
+                              + (self.coin.TX_COUNT - self.tx_count))
+
+                # Damp the enthusiasm
+                realism = 2.0 - 0.9 * self.height / self.coin.TX_COUNT_HEIGHT
+                tx_est *= max(realism, 1.0)
+
+
+                self.logger.info('sync time: {}  ETA: {}'
+                                 .format(formatted_time(self.wall_time),
+                                         formatted_time(tx_est / this_tx_per_sec)))
+            else:
+                self.logger.info('sync time: {}'
+                                 .format(formatted_time(self.wall_time)))
+
 
     def fs_flush(self):
         '''Flush the things stored on the filesystem.'''

server/controller.py

@@ -450,6 +450,7 @@ class Controller(util.LoggedClass):
     def getinfo(self):
         '''A one-line summary of server state.'''
         return {
+            'daemon': self.daemon.logged_url(),
             'daemon_height': self.daemon.cached_height(),
             'db_height': self.bp.db_height,
             'closing': len([s for s in self.sessions if s.is_closing()]),
@@ -463,6 +464,7 @@ class Controller(util.LoggedClass):
             'sessions': self.session_count(),
             'subs': self.sub_count(),
             'txs_sent': self.txs_sent,
+            'uptime': util.formatted_time(time.time() - self.start_time),
         }
 
     def sub_count(self):
@@ -514,12 +516,6 @@ class Controller(util.LoggedClass):
         '''A generator returning lines for a list of sessions.
 
         data is the return value of rpc_sessions().'''
-
-        def time_fmt(t):
-            t = int(t)
-            return ('{:3d}:{:02d}:{:02d}'
-                    .format(t // 3600, (t % 3600) // 60, t % 60))
-
         fmt = ('{:<6} {:<5} {:>17} {:>5} {:>5} '
                '{:>7} {:>7} {:>7} {:>7} {:>7} {:>9} {:>21}')
         yield fmt.format('ID', 'Flags', 'Client', 'Reqs', 'Txs', 'Subs',
@@ -534,7 +530,7 @@ class Controller(util.LoggedClass):
                              '{:,d}'.format(recv_size // 1024),
                              '{:,d}'.format(send_count),
                              '{:,d}'.format(send_size // 1024),
-                             time_fmt(time), peer)
+                             util.formatted_time(time, sep=''), peer)
 
     def session_data(self, for_log):
         '''Returned to the RPC 'sessions' call.'''
@@ -599,7 +595,7 @@ class Controller(util.LoggedClass):
             self.daemon.set_urls(self.env.coin.daemon_urls(daemon_url))
         except Exception as e:
             raise RPCError('an error occured: {}'.format(e))
-        return 'set daemon URL to {}'.format(daemon_url)
+        return 'now using daemon at {}'.format(self.daemon.logged_url())
 
     def rpc_stop(self):
         '''Shut down the server cleanly.'''

server/daemon.py

@@ -42,10 +42,27 @@ class Daemon(util.LoggedClass):
         '''Set the URLS to the given list, and switch to the first one.'''
         if not urls:
             raise DaemonError('no daemon URLs provided')
-        for url in urls:
-            self.logger.info('daemon at {}'.format(self.logged_url(url)))
         self.urls = urls
         self.url_index = 0
+        for n, url in enumerate(urls):
+            self.logger.info('daemon #{:d} at {}{}'
+                             .format(n + 1, self.logged_url(url),
+                                     '' if n else ' (current)'))
+
+    def url(self):
+        '''Returns the current daemon URL.'''
+        return self.urls[self.url_index]
+
+    def failover(self):
+        '''Call to fail-over to the next daemon URL.
+
+        Returns False if there is only one, otherwise True.
+        '''
+        if len(self.urls) > 1:
+            self.url_index = (self.url_index + 1) % len(self.urls)
+            self.logger.info('failing over to {}'.format(self.logged_url()))
+            return True
+        return False
 
     async def _send(self, payload, processor):
         '''Send a payload to be converted to JSON.
@@ -72,8 +89,7 @@ class Daemon(util.LoggedClass):
         while True:
             try:
                 async with self.workqueue_semaphore:
-                    url = self.urls[self.url_index]
+                    async with aiohttp.post(self.url(), data=data) as resp:
-                    async with aiohttp.post(url, data=data) as resp:
                         # If bitcoind can't find a tx, for some reason
                         # it returns 500 but fills out the JSON.
                         # Should still return 200 IMO.
@@ -99,17 +115,15 @@ class Daemon(util.LoggedClass):
             except Exception:
                 self.log_error(traceback.format_exc())
 
-            if secs >= max_secs and len(self.urls) > 1:
+            if secs >= max_secs and self.failover():
-                self.url_index = (self.url_index + 1) % len(self.urls)
-                logged_url = self.logged_url(self.urls[self.url_index])
-                self.logger.info('failing over to {}'.format(logged_url))
                 secs = 1
             else:
                 await asyncio.sleep(secs)
                 secs = min(max_secs, secs * 2)
 
-    def logged_url(self, url):
+    def logged_url(self, url=None):
         '''The host and port part, for logging.'''
+        url = url or self.url()
         return url[url.rindex('@') + 1:]
 
     async def _send_single(self, method, params=None):

server/version.py

@@ -1 +1 @@
-VERSION = "ElectrumX 0.10.16"
+VERSION = "ElectrumX 0.10.17"