From fa0a58a280fe89c629ce030ea70ea38fa395522b Mon Sep 17 00:00:00 2001 From: "John L. Jegutanis" Date: Mon, 13 Aug 2018 14:44:07 +0200 Subject: [PATCH 1/4] Rename struct pack/unpack to reflect endianess --- electrumx/lib/tx.py | 14 +++++++------- electrumx/lib/util.py | 30 +++++++++++++++--------------- electrumx/server/daemon.py | 4 ++-- tests/lib/test_util.py | 20 ++++++++++---------- 4 files changed, 34 insertions(+), 34 deletions(-) diff --git a/electrumx/lib/tx.py b/electrumx/lib/tx.py index 35fa419..8dc5d55 100644 --- a/electrumx/lib/tx.py +++ b/electrumx/lib/tx.py @@ -31,8 +31,8 @@ from collections import namedtuple from electrumx.lib.hash import sha256, double_sha256, hash_to_hex_str from electrumx.lib.util import ( - cachedproperty, unpack_int32_from, unpack_int64_from, - unpack_uint16_from, unpack_uint32_from, unpack_uint64_from, + cachedproperty, unpack_le_int32_from, unpack_le_int64_from, + unpack_le_uint16_from, unpack_le_uint32_from, unpack_le_uint64_from, pack_le_int32, pack_varint, pack_le_uint32, pack_le_uint32, pack_le_int64, pack_varbytes, ) @@ -185,27 +185,27 @@ class Deserializer(object): return self._read_le_uint64() def _read_le_int32(self): - result, = unpack_int32_from(self.binary, self.cursor) + result, = unpack_le_int32_from(self.binary, self.cursor) self.cursor += 4 return result def _read_le_int64(self): - result, = unpack_int64_from(self.binary, self.cursor) + result, = unpack_le_int64_from(self.binary, self.cursor) self.cursor += 8 return result def _read_le_uint16(self): - result, = unpack_uint16_from(self.binary, self.cursor) + result, = unpack_le_uint16_from(self.binary, self.cursor) self.cursor += 2 return result def _read_le_uint32(self): - result, = unpack_uint32_from(self.binary, self.cursor) + result, = unpack_le_uint32_from(self.binary, self.cursor) self.cursor += 4 return result def _read_le_uint64(self): - result, = unpack_uint64_from(self.binary, self.cursor) + result, = unpack_le_uint64_from(self.binary, self.cursor) self.cursor += 8 return result diff --git a/electrumx/lib/util.py b/electrumx/lib/util.py index 991cd88..c55037a 100644 --- a/electrumx/lib/util.py +++ b/electrumx/lib/util.py @@ -330,24 +330,24 @@ def protocol_version(client_req, min_tuple, max_tuple): return result, client_min -structi = Struct(' Date: Tue, 14 Aug 2018 00:17:00 +0200 Subject: [PATCH 2/4] Add big endian precompiled structs for 16 & 32 uints --- electrumx/lib/util.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/electrumx/lib/util.py b/electrumx/lib/util.py index c55037a..916ea89 100644 --- a/electrumx/lib/util.py +++ b/electrumx/lib/util.py @@ -335,6 +335,8 @@ struct_le_q = Struct('H') +struct_be_I = Struct('>I') structB = Struct('B') unpack_le_int32_from = struct_le_i.unpack_from @@ -342,12 +344,16 @@ unpack_le_int64_from = struct_le_q.unpack_from unpack_le_uint16_from = struct_le_H.unpack_from unpack_le_uint32_from = struct_le_I.unpack_from unpack_le_uint64_from = struct_le_Q.unpack_from +unpack_be_uint16_from = struct_be_H.unpack_from +unpack_be_uint32_from = struct_be_I.unpack_from pack_le_int32 = struct_le_i.pack pack_le_int64 = struct_le_q.pack pack_le_uint16 = struct_le_H.pack pack_le_uint32 = struct_le_I.pack pack_le_uint64 = struct_le_Q.pack +pack_be_uint16 = struct_be_H.pack +pack_be_uint32 = struct_be_I.pack pack_byte = structB.pack hex_to_bytes = bytes.fromhex From 31dbbe0ba5280090bf733921c210608de1bed701 Mon Sep 17 00:00:00 2001 From: "John L. Jegutanis" Date: Tue, 14 Aug 2018 00:26:13 +0200 Subject: [PATCH 3/4] Use precompiled structs for header parsing --- electrumx/lib/coins.py | 205 ++++++------------ tests/blocks/newyorkcoin_mainnet_3956926.json | 2 +- tests/test_blocks.py | 8 + 3 files changed, 79 insertions(+), 136 deletions(-) diff --git a/electrumx/lib/coins.py b/electrumx/lib/coins.py index 905b423..fcc3141 100644 --- a/electrumx/lib/coins.py +++ b/electrumx/lib/coins.py @@ -70,6 +70,9 @@ class Coin(object): DESERIALIZER = lib_tx.Deserializer DAEMON = daemon.Daemon BLOCK_PROCESSOR = block_proc.BlockProcessor + HEADER_VALUES = ('version', 'prev_block_hash', 'merkle_root', 'timestamp', + 'bits', 'nonce') + HEADER_UNPACK = struct.Struct('< I 32s 32s I I I').unpack_from MEMPOOL_HISTOGRAM_REFRESH_SECS = 500 XPUB_VERBYTES = bytes('????', 'utf-8') XPRV_VERBYTES = bytes('????', 'utf-8') @@ -296,18 +299,13 @@ class Coin(object): @classmethod def electrum_header(cls, header, height): - version, = struct.unpack(' 6: return super().header_hash(header) else: @@ -475,8 +473,7 @@ class BitcoinGold(EquihashMixin, BitcoinMixin, Coin): @classmethod def header_hash(cls, header): '''Given a header return hash''' - height, = struct.unpack('= cls.FORK_HEIGHT: return double_sha256(header) else: @@ -484,18 +481,9 @@ class BitcoinGold(EquihashMixin, BitcoinMixin, Coin): @classmethod def electrum_header(cls, header, height): - h = dict( - block_height=height, - version=struct.unpack(' 1: + h['nAccumulatorCheckpoint'] = hash_to_hex_str(header[80:]) + return h class Pac(Coin): @@ -1998,35 +1947,7 @@ class Monoeci(Coin): return x11_hash.getPoWHash(header) -class MinexcoinMixin(object): - STATIC_BLOCK_HEADERS = True - BASIC_HEADER_SIZE = 209 - DESERIALIZER = lib_tx.DeserializerEquihash - - @classmethod - def electrum_header(cls, header, height): - version, = struct.unpack('= 4: return super().header_hash(header) else: diff --git a/tests/blocks/newyorkcoin_mainnet_3956926.json b/tests/blocks/newyorkcoin_mainnet_3956926.json index 268f39d..ef29bad 100644 --- a/tests/blocks/newyorkcoin_mainnet_3956926.json +++ b/tests/blocks/newyorkcoin_mainnet_3956926.json @@ -7,7 +7,7 @@ "0300b2fe16c049641ec63f9eb435b9d773c847bc0185c927d5b8d87cd1ad3095" ], "time": 1515308763, - "nonce": "1570457681", + "nonce": 1570457681, "bits": "1b172a4e", "previousblockhash": "10c91aefd6698f03a2820b8aa462738a5c1c21f4f975786a8eb1ba25e4b33af3", "block": "01000000f33ab3e425bab18e6a7875f9f4211c5c8a7362a48a0b82a2038f69d6ef1ac9109530add17cd8b8d527c98501bc47c873d7b935b49e3fc61e6449c016feb20003dbc6515a4e2a171b51489b5d0101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff5203be603c062f503253482f04dfc6515afabe6d6dce67985c41a5f1ba5d526614e1744010b5e6a34d33c50ece5d420d75da15454e100000000000000008b8000274020000000c2f30324d515156434c59582f00000000010010a5d4e80000001976a914f0a150ec5709fae1d1814227b69cd1f0baf528c588ac00000000" diff --git a/tests/test_blocks.py b/tests/test_blocks.py index d82dc5b..c8803d6 100644 --- a/tests/test_blocks.py +++ b/tests/test_blocks.py @@ -32,6 +32,7 @@ import pytest from electrumx.lib.coins import Coin from electrumx.lib.hash import hex_str_to_hash +from electrumx.lib.util import pack_be_uint32 BLOCKS_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), 'blocks') @@ -60,6 +61,13 @@ def test_block(block_details): raw_block = unhexlify(block_info['block']) block = coin.block(raw_block, block_info['height']) + h = coin.electrum_header(block.header, block_info['height']) + assert block_info['merkleroot'] == h['merkle_root'] + assert block_info['time'] == h['timestamp'] + assert block_info['previousblockhash'] == h['prev_block_hash'] + assert block_info['height'] == h['block_height'] + assert block_info['nonce'] == h['nonce'] + assert block_info['bits'] == pack_be_uint32(h['bits']).hex() assert coin.header_hash(block.header) == hex_str_to_hash(block_info['hash']) assert (coin.header_prevhash(block.header) From 47d9fb4b30cd13bc02d0d0e49ad4cabf941f8e61 Mon Sep 17 00:00:00 2001 From: "John L. Jegutanis" Date: Tue, 14 Aug 2018 01:19:22 +0200 Subject: [PATCH 4/4] Use compiled structs and some code cleanup --- electrumx/lib/script.py | 10 ++++++---- electrumx/lib/util.py | 14 -------------- electrumx/server/daemon.py | 6 +++--- electrumx/server/history.py | 10 +++++----- electrumx/wallet/bip32.py | 11 ++++++----- tests/lib/test_util.py | 29 ++++++++++++++++------------- tests/server/test_compaction.py | 11 +++++------ 7 files changed, 41 insertions(+), 50 deletions(-) diff --git a/electrumx/lib/script.py b/electrumx/lib/script.py index 94283b1..b99fedb 100644 --- a/electrumx/lib/script.py +++ b/electrumx/lib/script.py @@ -32,6 +32,8 @@ from collections import namedtuple from electrumx.lib.enum import Enumeration from electrumx.lib.hash import hash160 +from electrumx.lib.util import unpack_le_uint16_from, unpack_le_uint32_from, \ + pack_le_uint16, pack_le_uint32 class ScriptError(Exception): @@ -196,10 +198,10 @@ class Script(object): dlen = script[n] n += 1 elif op == OpCodes.OP_PUSHDATA2: - dlen, = struct.unpack(' len(script): raise IndexError @@ -225,8 +227,8 @@ class Script(object): if n < 256: return bytes([OpCodes.OP_PUSHDATA1, n]) + data if n < 65536: - return bytes([OpCodes.OP_PUSHDATA2]) + struct.pack(' 0: - raw_block += int_to_varint(num_txs) + raw_block += pack_varint(num_txs) raw_block += b''.join(transactions) else: raw_block += b'\x00' diff --git a/electrumx/server/history.py b/electrumx/server/history.py index b42ca6c..bffdbc9 100644 --- a/electrumx/server/history.py +++ b/electrumx/server/history.py @@ -14,9 +14,9 @@ import bisect import time from collections import defaultdict from functools import partial -from struct import pack, unpack import electrumx.lib.util as util +from electrumx.lib.util import pack_be_uint16, unpack_be_uint16_from from electrumx.lib.hash import hash_to_hex_str, HASHX_LEN @@ -81,7 +81,7 @@ class History(object): keys = [] for key, hist in self.db.iterator(prefix=b''): - flush_id, = unpack('>H', key[-2:]) + flush_id, = unpack_be_uint16_from(key[-2:]) if flush_id > utxo_flush_count: keys.append(key) @@ -126,7 +126,7 @@ class History(object): def flush(self): start_time = time.time() self.flush_count += 1 - flush_id = pack('>H', self.flush_count) + flush_id = pack_be_uint16(self.flush_count) unflushed = self.unflushed with self.db.write_batch() as batch: @@ -250,7 +250,7 @@ class History(object): write_size = 0 keys_to_delete.update(hist_map) for n, chunk in enumerate(util.chunks(full_hist, max_row_size)): - key = hashX + pack('>H', n) + key = hashX + pack_be_uint16(n) if hist_map.get(key) == chunk: keys_to_delete.remove(key) else: @@ -302,7 +302,7 @@ class History(object): # Loop over 2-byte prefixes cursor = self.comp_cursor while write_size < limit and cursor < 65536: - prefix = pack('>H', cursor) + prefix = pack_be_uint16(cursor) write_size += self._compact_prefix(prefix, write_items, keys_to_delete) cursor += 1 diff --git a/electrumx/wallet/bip32.py b/electrumx/wallet/bip32.py index e8c2274..0d1ef95 100644 --- a/electrumx/wallet/bip32.py +++ b/electrumx/wallet/bip32.py @@ -15,7 +15,8 @@ import ecdsa.numbertheory as NT from electrumx.lib.coins import Coin from electrumx.lib.hash import Base58, hmac_sha512, hash160 -from electrumx.lib.util import cachedproperty, bytes_to_int, int_to_bytes +from electrumx.lib.util import cachedproperty, bytes_to_int, int_to_bytes, \ + pack_be_uint32, unpack_be_uint32_from class DerivationError(Exception): @@ -65,7 +66,7 @@ class _KeyBase(object): raise ValueError('raw_serkey must have length 33') return (ver_bytes + bytes([self.depth]) - + self.parent_fingerprint() + struct.pack('>I', self.n) + + self.parent_fingerprint() + pack_be_uint32(self.n) + self.chain_code + raw_serkey) def fingerprint(self): @@ -142,7 +143,7 @@ class PubKey(_KeyBase): if not 0 <= n < (1 << 31): raise ValueError('invalid BIP32 public key child number') - msg = self.pubkey_bytes + struct.pack('>I', n) + msg = self.pubkey_bytes + pack_be_uint32(n) L, R = self._hmac_sha512(msg) curve = self.CURVE @@ -244,7 +245,7 @@ class PrivKey(_KeyBase): else: serkey = self.public_key.pubkey_bytes - msg = serkey + struct.pack('>I', n) + msg = serkey + pack_be_uint32(n) L, R = self._hmac_sha512(msg) curve = self.CURVE @@ -282,7 +283,7 @@ def _from_extended_key(ekey): is_public, coin = Coin.lookup_xverbytes(ekey[:4]) depth = ekey[4] fingerprint = ekey[5:9] # Not used - n, = struct.unpack('>I', ekey[9:13]) + n, = unpack_be_uint32_from(ekey[9:13]) chain_code = ekey[13:45] if is_public: diff --git a/tests/lib/test_util.py b/tests/lib/test_util.py index dec5143..1635cd3 100644 --- a/tests/lib/test_util.py +++ b/tests/lib/test_util.py @@ -71,25 +71,14 @@ def test_increment_byte_string(): assert util.increment_byte_string(b'\x01\x01') == b'\x01\x02' assert util.increment_byte_string(b'\xff\xff') is None + def test_bytes_to_int(): assert util.bytes_to_int(b'\x07[\xcd\x15') == 123456789 + def test_int_to_bytes(): assert util.int_to_bytes(456789) == b'\x06\xf8U' -def test_int_to_varint(): - with pytest.raises(ValueError): - util.int_to_varint(-1) - assert util.int_to_varint(0) == b'\0' - assert util.int_to_varint(5) == b'\5' - assert util.int_to_varint(252) == b'\xfc' - assert util.int_to_varint(253) == b'\xfd\xfd\0' - assert util.int_to_varint(65535) == b'\xfd\xff\xff' - assert util.int_to_varint(65536) == b'\xfe\0\0\1\0' - assert util.int_to_varint(2**32-1) == b'\xfe\xff\xff\xff\xff' - assert util.int_to_varint(2**32) == b'\xff\0\0\0\0\1\0\0\0' - assert util.int_to_varint(2**64-1) \ - == b'\xff\xff\xff\xff\xff\xff\xff\xff\xff' def test_LogicalFile(tmpdir): prefix = os.path.join(tmpdir, 'log') @@ -234,6 +223,20 @@ def test_pack_varint(): deser = tx.Deserializer(data) assert deser._read_varint() == n + import struct + with pytest.raises(struct.error): + util.pack_varint(-1) + assert util.pack_varint(0) == b'\0' + assert util.pack_varint(5) == b'\5' + assert util.pack_varint(252) == b'\xfc' + assert util.pack_varint(253) == b'\xfd\xfd\0' + assert util.pack_varint(65535) == b'\xfd\xff\xff' + assert util.pack_varint(65536) == b'\xfe\0\0\1\0' + assert util.pack_varint(2**32-1) == b'\xfe\xff\xff\xff\xff' + assert util.pack_varint(2**32) == b'\xff\0\0\0\0\1\0\0\0' + assert util.pack_varint(2**64-1) \ + == b'\xff\xff\xff\xff\xff\xff\xff\xff\xff' + def test_pack_varbytes(): tests = [b'', b'1', b'2' * 253, b'3' * 254, b'4' * 256, b'5' * 65536] diff --git a/tests/server/test_compaction.py b/tests/server/test_compaction.py index f9c9eb6..44017f1 100644 --- a/tests/server/test_compaction.py +++ b/tests/server/test_compaction.py @@ -2,12 +2,11 @@ import array import asyncio -from collections import defaultdict from os import environ, urandom -from struct import pack import random from electrumx.lib.hash import HASHX_LEN +from electrumx.lib.util import pack_be_uint16 from electrumx.server.env import Env from electrumx.server.db import DB @@ -49,7 +48,7 @@ def check_hashX_compaction(history): hist_list = [] hist_map = {} for flush_count, count in pairs: - key = hashX + pack('>H', flush_count) + key = hashX + pack_be_uint16(flush_count) hist = full_hist[cum * 4: (cum+count) * 4] hist_map[key] = hist hist_list.append(hist) @@ -65,10 +64,10 @@ def check_hashX_compaction(history): assert len(keys_to_delete) == 3 assert len(hist_map) == len(pairs) for n, item in enumerate(write_items): - assert item == (hashX + pack('>H', n), + assert item == (hashX + pack_be_uint16(n), full_hist[n * row_size: (n + 1) * row_size]) for flush_count, count in pairs: - assert hashX + pack('>H', flush_count) in keys_to_delete + assert hashX + pack_be_uint16(flush_count) in keys_to_delete # Check re-compaction is null hist_map = {key: value for key, value in write_items} @@ -87,7 +86,7 @@ def check_hashX_compaction(history): write_size = history._compact_hashX(hashX, hist_map, hist_list, write_items, keys_to_delete) assert write_size == len(hist_list[-1]) - assert write_items == [(hashX + pack('>H', 2), hist_list[-1])] + assert write_items == [(hashX + pack_be_uint16(2), hist_list[-1])] assert len(keys_to_delete) == 1 assert write_items[0][0] in keys_to_delete assert len(hist_map) == len(pairs)