Use precompiled structs for header parsing

electrumx/lib/coins.py

@@ -70,6 +70,9 @@ class Coin(object):
     DESERIALIZER = lib_tx.Deserializer
     DAEMON = daemon.Daemon
     BLOCK_PROCESSOR = block_proc.BlockProcessor
+    HEADER_VALUES = ('version', 'prev_block_hash', 'merkle_root', 'timestamp',
+                     'bits', 'nonce')
+    HEADER_UNPACK = struct.Struct('< I 32s 32s I I I').unpack_from
     MEMPOOL_HISTOGRAM_REFRESH_SECS = 500
     XPUB_VERBYTES = bytes('????', 'utf-8')
     XPRV_VERBYTES = bytes('????', 'utf-8')
@@ -296,18 +299,13 @@ class Coin(object):
 
     @classmethod
     def electrum_header(cls, header, height):
-        version, = struct.unpack('<I', header[:4])
+        h = dict(zip(cls.HEADER_VALUES, cls.HEADER_UNPACK(header)))
-        timestamp, bits, nonce = struct.unpack('<III', header[68:80])
+        # Add the height that is not present in the header itself
-
+        h['block_height'] = height
-        return {
+        # Convert bytes to str
-            'block_height': height,
+        h['prev_block_hash'] = hash_to_hex_str(h['prev_block_hash'])
-            'version': version,
+        h['merkle_root'] = hash_to_hex_str(h['merkle_root'])
-            'prev_block_hash': hash_to_hex_str(header[4:36]),
+        return h
-            'merkle_root': hash_to_hex_str(header[36:68]),
-            'timestamp': timestamp,
-            'bits': bits,
-            'nonce': nonce,
-        }
 
 
 class AuxPowMixin(object):
@@ -330,21 +328,21 @@ class EquihashMixin(object):
     STATIC_BLOCK_HEADERS = False
     BASIC_HEADER_SIZE = 140  # Excluding Equihash solution
     DESERIALIZER = lib_tx.DeserializerEquihash
+    HEADER_VALUES = ('version', 'prev_block_hash', 'merkle_root', 'reserved',
+                     'timestamp', 'bits', 'nonce')
+    HEADER_UNPACK = struct.Struct('< I 32s 32s 32s I I 32s').unpack_from
 
     @classmethod
     def electrum_header(cls, header, height):
-        version, = struct.unpack('<I', header[:4])
+        h = dict(zip(cls.HEADER_VALUES, cls.HEADER_UNPACK(header)))
-        timestamp, bits = struct.unpack('<II', header[100:108])
+        # Add the height that is not present in the header itself
-
+        h['block_height'] = height
-        return {
+        # Convert bytes to str
-            'block_height': height,
+        h['prev_block_hash'] = hash_to_hex_str(h['prev_block_hash'])
-            'version': version,
+        h['merkle_root'] = hash_to_hex_str(h['merkle_root'])
-            'prev_block_hash': hash_to_hex_str(header[4:36]),
+        h['reserved'] = hash_to_hex_str(h['reserved'])
-            'merkle_root': hash_to_hex_str(header[36:68]),
+        h['nonce'] = hash_to_hex_str(h['nonce'])
-            'timestamp': timestamp,
+        return h
-            'bits': bits,
-            'nonce': hash_to_hex_str(header[108:140]),
-        }
 
     @classmethod
     def block_header(cls, block, height):
@@ -365,7 +363,7 @@ class ScryptMixin(object):
             import scrypt
             cls.HEADER_HASH = lambda x: scrypt.hash(x, x, 1024, 1, 1, 32)
 
-        version, = struct.unpack('<I', header[:4])
+        version, = util.unpack_le_uint32_from(header)
         if version > 6:
             return super().header_hash(header)
         else:
@@ -475,8 +473,7 @@ class BitcoinGold(EquihashMixin, BitcoinMixin, Coin):
     @classmethod
     def header_hash(cls, header):
         '''Given a header return hash'''
-        height, = struct.unpack('<I', header[68:72])
+        height, = util.unpack_le_uint32_from(header, 68)
-
         if height >= cls.FORK_HEIGHT:
             return double_sha256(header)
         else:
@@ -484,18 +481,9 @@ class BitcoinGold(EquihashMixin, BitcoinMixin, Coin):
 
     @classmethod
     def electrum_header(cls, header, height):
-        h = dict(
+        h = super().electrum_header(header, height)
-            block_height=height,
+        h['reserved'] = hash_to_hex_str(header[72:100])
-            version=struct.unpack('<I', header[:4])[0],
+        h['solution'] = hash_to_hex_str(header[140:])
-            prev_block_hash=hash_to_hex_str(header[4:36]),
-            merkle_root=hash_to_hex_str(header[36:68]),
-            timestamp=struct.unpack('<I', header[100:104])[0],
-            reserved=hash_to_hex_str(header[72:100]),
-            bits=struct.unpack('<I', header[104:108])[0],
-            nonce=hash_to_hex_str(header[108:140]),
-            solution=hash_to_hex_str(header[140:])
-        )
-
         return h
 
 
@@ -948,6 +936,9 @@ class FairCoin(Coin):
     GENESIS_HASH = ('beed44fa5e96150d95d56ebd5d262578'
                     '1825a9407a5215dd7eda723373a0a1d7')
     BASIC_HEADER_SIZE = 108
+    HEADER_VALUES = ('version', 'prev_block_hash', 'merkle_root',
+                     'payload_hash', 'timestamp', 'creatorId')
+    HEADER_UNPACK = struct.Struct('< I 32s 32s 32s I I').unpack_from
     TX_COUNT = 505
     TX_COUNT_HEIGHT = 470
     TX_PER_BLOCK = 1
@@ -968,17 +959,9 @@ class FairCoin(Coin):
 
     @classmethod
     def electrum_header(cls, header, height):
-        version, = struct.unpack('<I', header[:4])
+        h = super().electrum_header(header, height)
-        timestamp, creatorId = struct.unpack('<II', header[100:108])
+        h['payload_hash'] = hash_to_hex_str(h['payload_hash'])
-        return {
+        return h
-            'block_height': height,
-            'version': version,
-            'prev_block_hash': hash_to_hex_str(header[4:36]),
-            'merkle_root': hash_to_hex_str(header[36:68]),
-            'payload_hash': hash_to_hex_str(header[68:100]),
-            'timestamp': timestamp,
-            'creatorId': creatorId,
-        }
 
 
 class Zcash(EquihashMixin, Coin):
@@ -1031,21 +1014,10 @@ class SnowGem(EquihashMixin, Coin):
 
     @classmethod
     def electrum_header(cls, header, height):
-        version, = struct.unpack('<I', header[:4])
+        h = super().electrum_header(header, height)
-        timestamp, bits = struct.unpack('<II', header[100:108])
+        h['n_solution'] = base64.b64encode(lib_tx.Deserializer(
-
+            header, start=140)._read_varbytes()).decode('utf8')
-        return {
+        return h
-            'block_height': height,
-            'version': version,
-            'prev_block_hash': hash_to_hex_str(header[4:36]),
-            'merkle_root': hash_to_hex_str(header[36:68]),
-            'hash_reserved': hash_to_hex_str(header[68:100]),
-            'timestamp': timestamp,
-            'bits': bits,
-            'nonce': hash_to_hex_str(header[108:140]),
-            'n_solution': base64.b64encode(lib_tx.Deserializer(
-                header, start=140)._read_varbytes()).decode('utf8')
-        }
 
 
 class BitcoinZ(EquihashMixin, Coin):
@@ -1670,7 +1642,7 @@ class BitcoinAtom(Coin):
         header_to_be_hashed = header[:cls.BASIC_HEADER_SIZE]
         # New block header format has some extra flags in the end
         if len(header) == cls.HEADER_SIZE_POST_FORK:
-            flags, = struct.unpack('<I', header[-4:])
+            flags, = util.unpack_le_uint32_from(header, len(header) - 4)
             # Proof of work blocks have special serialization
             if flags & cls.BLOCK_PROOF_OF_STAKE != 0:
                 header_to_be_hashed += cls.BLOCK_PROOF_OF_STAKE_FLAGS
@@ -1704,7 +1676,13 @@ class Decred(Coin):
                            hash_fn=lib_tx.DeserializerDecred.blake256d)
     DECODE_CHECK = partial(Base58.decode_check,
                            hash_fn=lib_tx.DeserializerDecred.blake256d)
-    HEADER_UNPACK = struct.Struct('<i32s32s32sH6sHBBIIQIIII32sI').unpack_from
+    HEADER_VALUES = ('version', 'prev_block_hash', 'merkle_root', 'stake_root',
+                     'vote_bits', 'final_state', 'voters', 'fresh_stake',
+                     'revocations', 'pool_size', 'bits', 'sbits',
+                     'block_height', 'size', 'timestamp', 'nonce',
+                     'extra_data', 'stake_version')
+    HEADER_UNPACK = struct.Struct(
+        '< i 32s 32s 32s H 6s H B B I I Q I I I I 32s I').unpack_from
     TX_COUNT = 4629388
     TX_COUNT_HEIGHT = 260628
     TX_PER_BLOCK = 17
@@ -1726,20 +1704,10 @@ class Decred(Coin):
 
     @classmethod
     def electrum_header(cls, header, height):
-        labels = ('version', 'prev_block_hash', 'merkle_root', 'stake_root',
+        h = super().electrum_header(header, height)
-                  'vote_bits', 'final_state', 'voters', 'fresh_stake',
-                  'revocations', 'pool_size', 'bits', 'sbits', 'block_height',
-                  'size', 'timestamp', 'nonce', 'extra_data', 'stake_version')
-        values = cls.HEADER_UNPACK(header)
-        h = dict(zip(labels, values))
-
-        # Convert some values
-        assert h['block_height'] == height
-        h['prev_block_hash'] = hash_to_hex_str(h['prev_block_hash'])
-        h['merkle_root'] = hash_to_hex_str(h['merkle_root'])
         h['stake_root'] = hash_to_hex_str(h['stake_root'])
-        h['final_state'] = h['final_state'].hex()
+        h['final_state'] = hash_to_hex_str(h['final_state'])
-        h['extra_data'] = h['extra_data'].hex()
+        h['extra_data'] = hash_to_hex_str(h['extra_data'])
         return h
 
 
@@ -1799,7 +1767,7 @@ class Xuez(Coin):
         Need to download `xevan_hash` module
         Source code: https://github.com/xuez/xuez
         '''
-        version, = struct.unpack('<I', header[:4])
+        version, = util.unpack_le_uint32_from(header)
 
         import xevan_hash
 
@@ -1810,29 +1778,10 @@ class Xuez(Coin):
 
     @classmethod
     def electrum_header(cls, header, height):
-        version, = struct.unpack('<I', header[:4])
+        h = super().electrum_header(header, height)
-        timestamp, bits, nonce = struct.unpack('<III', header[68:80])
+        if h['version'] > 1:
-        if version == 1:
+            h['nAccumulatorCheckpoint'] = hash_to_hex_str(header[80:])
-            return {
+        return h
-                'block_height': height,
-                'version': version,
-                'prev_block_hash': hash_to_hex_str(header[4:36]),
-                'merkle_root': hash_to_hex_str(header[36:68]),
-                'timestamp': timestamp,
-                'bits': bits,
-                'nonce': nonce,
-            }
-        else:
-            return {
-                'block_height': height,
-                'version': version,
-                'prev_block_hash': hash_to_hex_str(header[4:36]),
-                'merkle_root': hash_to_hex_str(header[36:68]),
-                'timestamp': timestamp,
-                'bits': bits,
-                'nonce': nonce,
-                'nAccumulatorCheckpoint': hash_to_hex_str(header[80:112]),
-            }
 
 
 class Pac(Coin):
@@ -1998,35 +1947,7 @@ class Monoeci(Coin):
                 return x11_hash.getPoWHash(header)
 
 
-class MinexcoinMixin(object):
+class Minexcoin(EquihashMixin, Coin):
-    STATIC_BLOCK_HEADERS = True
-    BASIC_HEADER_SIZE = 209
-    DESERIALIZER = lib_tx.DeserializerEquihash
-
-    @classmethod
-    def electrum_header(cls, header, height):
-        version, = struct.unpack('<I', header[:4])
-        timestamp, bits = struct.unpack('<II', header[100:108])
-
-        return {
-            'block_height': height,
-            'version': version,
-            'prev_block_hash': hash_to_hex_str(header[4:36]),
-            'merkle_root': hash_to_hex_str(header[36:68]),
-            'timestamp': timestamp,
-            'bits': bits,
-            'nonce': hash_to_hex_str(header[108:140]),
-            'solution': hash_to_hex_str(header[140:209]),
-        }
-
-    @classmethod
-    def block_header(cls, block, height):
-        '''Return the block header bytes'''
-        deserializer = cls.DESERIALIZER(block)
-        return deserializer.read_header(height, 140)
-
-
-class Minexcoin(MinexcoinMixin, Coin):
     NAME = "Minexcoin"
     SHORTNAME = "MNX"
     NET = "mainnet"
@@ -2035,7 +1956,9 @@ class Minexcoin(MinexcoinMixin, Coin):
     WIF_BYTE = bytes.fromhex("80")
     GENESIS_HASH = ('490a36d9451a55ed197e34aca7414b35'
                     'd775baa4a8e896f1c577f65ce2d214cb')
-    DESERIALIZER = lib_tx.DeserializerEquihash
+    STATIC_BLOCK_HEADERS = True
+    BASIC_HEADER_SIZE = 209
+    HEADER_SIZE_NO_SOLUTION = 140
     TX_COUNT = 327963
     TX_COUNT_HEIGHT = 74495
     TX_PER_BLOCK = 5
@@ -2046,6 +1969,18 @@ class Minexcoin(MinexcoinMixin, Coin):
         'eu.minexpool.nl s t'
     ]
 
+    @classmethod
+    def electrum_header(cls, header, height):
+        h = super().electrum_header(header, height)
+        h['solution'] = hash_to_hex_str(header[cls.HEADER_SIZE_NO_SOLUTION:])
+        return h
+
+    @classmethod
+    def block_header(cls, block, height):
+        '''Return the block header bytes'''
+        deserializer = cls.DESERIALIZER(block)
+        return deserializer.read_header(height, cls.HEADER_SIZE_NO_SOLUTION)
+
 
 class Groestlcoin(Coin):
     NAME = "Groestlcoin"
@@ -2133,7 +2068,7 @@ class Pivx(Coin):
 
     @classmethod
     def header_hash(cls, header):
-        version, = struct.unpack('<I', header[:4])
+        version, = util.unpack_le_uint32_from(header)
         if version >= 4:
             return super().header_hash(header)
         else:

tests/blocks/newyorkcoin_mainnet_3956926.json

@@ -7,7 +7,7 @@
     "0300b2fe16c049641ec63f9eb435b9d773c847bc0185c927d5b8d87cd1ad3095"
   ],
   "time": 1515308763,
-  "nonce": "1570457681",
+  "nonce": 1570457681,
   "bits": "1b172a4e",
   "previousblockhash": "10c91aefd6698f03a2820b8aa462738a5c1c21f4f975786a8eb1ba25e4b33af3",
   "block": "01000000f33ab3e425bab18e6a7875f9f4211c5c8a7362a48a0b82a2038f69d6ef1ac9109530add17cd8b8d527c98501bc47c873d7b935b49e3fc61e6449c016feb20003dbc6515a4e2a171b51489b5d0101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff5203be603c062f503253482f04dfc6515afabe6d6dce67985c41a5f1ba5d526614e1744010b5e6a34d33c50ece5d420d75da15454e100000000000000008b8000274020000000c2f30324d515156434c59582f00000000010010a5d4e80000001976a914f0a150ec5709fae1d1814227b69cd1f0baf528c588ac00000000"

tests/test_blocks.py

@@ -32,6 +32,7 @@ import pytest
 
 from electrumx.lib.coins import Coin
 from electrumx.lib.hash import hex_str_to_hash
+from electrumx.lib.util import pack_be_uint32
 
 BLOCKS_DIR = os.path.join(
     os.path.dirname(os.path.realpath(__file__)), 'blocks')
@@ -60,6 +61,13 @@ def test_block(block_details):
 
     raw_block = unhexlify(block_info['block'])
     block = coin.block(raw_block, block_info['height'])
+    h = coin.electrum_header(block.header, block_info['height'])
+    assert block_info['merkleroot'] == h['merkle_root']
+    assert block_info['time'] == h['timestamp']
+    assert block_info['previousblockhash'] == h['prev_block_hash']
+    assert block_info['height'] == h['block_height']
+    assert block_info['nonce'] == h['nonce']
+    assert block_info['bits'] == pack_be_uint32(h['bits']).hex()
 
     assert coin.header_hash(block.header) == hex_str_to_hash(block_info['hash'])
     assert (coin.header_prevhash(block.header)