diff --git a/electrumx/lib/coins.py b/electrumx/lib/coins.py index 905b423..fcc3141 100644 --- a/electrumx/lib/coins.py +++ b/electrumx/lib/coins.py @@ -70,6 +70,9 @@ class Coin(object): DESERIALIZER = lib_tx.Deserializer DAEMON = daemon.Daemon BLOCK_PROCESSOR = block_proc.BlockProcessor + HEADER_VALUES = ('version', 'prev_block_hash', 'merkle_root', 'timestamp', + 'bits', 'nonce') + HEADER_UNPACK = struct.Struct('< I 32s 32s I I I').unpack_from MEMPOOL_HISTOGRAM_REFRESH_SECS = 500 XPUB_VERBYTES = bytes('????', 'utf-8') XPRV_VERBYTES = bytes('????', 'utf-8') @@ -296,18 +299,13 @@ class Coin(object): @classmethod def electrum_header(cls, header, height): - version, = struct.unpack(' 6: return super().header_hash(header) else: @@ -475,8 +473,7 @@ class BitcoinGold(EquihashMixin, BitcoinMixin, Coin): @classmethod def header_hash(cls, header): '''Given a header return hash''' - height, = struct.unpack('= cls.FORK_HEIGHT: return double_sha256(header) else: @@ -484,18 +481,9 @@ class BitcoinGold(EquihashMixin, BitcoinMixin, Coin): @classmethod def electrum_header(cls, header, height): - h = dict( - block_height=height, - version=struct.unpack(' 1: + h['nAccumulatorCheckpoint'] = hash_to_hex_str(header[80:]) + return h class Pac(Coin): @@ -1998,35 +1947,7 @@ class Monoeci(Coin): return x11_hash.getPoWHash(header) -class MinexcoinMixin(object): - STATIC_BLOCK_HEADERS = True - BASIC_HEADER_SIZE = 209 - DESERIALIZER = lib_tx.DeserializerEquihash - - @classmethod - def electrum_header(cls, header, height): - version, = struct.unpack('= 4: return super().header_hash(header) else: diff --git a/electrumx/lib/script.py b/electrumx/lib/script.py index 94283b1..b99fedb 100644 --- a/electrumx/lib/script.py +++ b/electrumx/lib/script.py @@ -32,6 +32,8 @@ from collections import namedtuple from electrumx.lib.enum import Enumeration from electrumx.lib.hash import hash160 +from electrumx.lib.util import unpack_le_uint16_from, unpack_le_uint32_from, \ + pack_le_uint16, pack_le_uint32 class ScriptError(Exception): @@ -196,10 +198,10 @@ class Script(object): dlen = script[n] n += 1 elif op == OpCodes.OP_PUSHDATA2: - dlen, = struct.unpack(' len(script): raise IndexError @@ -225,8 +227,8 @@ class Script(object): if n < 256: return bytes([OpCodes.OP_PUSHDATA1, n]) + data if n < 65536: - return bytes([OpCodes.OP_PUSHDATA2]) + struct.pack('H') +struct_be_I = Struct('>I') structB = Struct('B') -unpack_int32_from = structi.unpack_from -unpack_int64_from = structq.unpack_from -unpack_uint16_from = structH.unpack_from -unpack_uint32_from = structI.unpack_from -unpack_uint64_from = structQ.unpack_from +unpack_le_int32_from = struct_le_i.unpack_from +unpack_le_int64_from = struct_le_q.unpack_from +unpack_le_uint16_from = struct_le_H.unpack_from +unpack_le_uint32_from = struct_le_I.unpack_from +unpack_le_uint64_from = struct_le_Q.unpack_from +unpack_be_uint16_from = struct_be_H.unpack_from +unpack_be_uint32_from = struct_be_I.unpack_from -pack_le_int32 = structi.pack -pack_le_int64 = structq.pack -pack_le_uint16 = structH.pack -pack_le_uint32 = structI.pack -pack_le_uint64 = structQ.pack +pack_le_int32 = struct_le_i.pack +pack_le_int64 = struct_le_q.pack +pack_le_uint16 = struct_le_H.pack +pack_le_uint32 = struct_le_I.pack +pack_le_uint64 = struct_le_Q.pack +pack_be_uint16 = struct_be_H.pack +pack_be_uint32 = struct_be_I.pack pack_byte = structB.pack hex_to_bytes = bytes.fromhex diff --git a/electrumx/server/daemon.py b/electrumx/server/daemon.py index be22e99..4c47834 100644 --- a/electrumx/server/daemon.py +++ b/electrumx/server/daemon.py @@ -17,8 +17,8 @@ from time import strptime import aiohttp -from electrumx.lib.util import int_to_varint, hex_to_bytes, class_logger, \ - unpack_uint16_from +from electrumx.lib.util import hex_to_bytes, class_logger,\ + unpack_le_uint16_from, pack_varint from electrumx.lib.hash import hex_str_to_hash, hash_to_hex_str from electrumx.lib.tx import DeserializerDecred from aiorpcx import JSONRPC @@ -356,7 +356,7 @@ class LegacyRPCDaemon(Daemon): raw_block = header num_txs = len(transactions) if num_txs > 0: - raw_block += int_to_varint(num_txs) + raw_block += pack_varint(num_txs) raw_block += b''.join(transactions) else: raw_block += b'\x00' @@ -384,7 +384,7 @@ class DecredDaemon(Daemon): raw_blocks.append(raw_block) # Check if previous block is valid prev = self.prev_hex_hash(raw_block) - votebits = unpack_uint16_from(raw_block[100:102])[0] + votebits = unpack_le_uint16_from(raw_block[100:102])[0] valid_tx_tree[prev] = self.is_valid_tx_tree(votebits) processed_raw_blocks = [] diff --git a/electrumx/server/history.py b/electrumx/server/history.py index b42ca6c..bffdbc9 100644 --- a/electrumx/server/history.py +++ b/electrumx/server/history.py @@ -14,9 +14,9 @@ import bisect import time from collections import defaultdict from functools import partial -from struct import pack, unpack import electrumx.lib.util as util +from electrumx.lib.util import pack_be_uint16, unpack_be_uint16_from from electrumx.lib.hash import hash_to_hex_str, HASHX_LEN @@ -81,7 +81,7 @@ class History(object): keys = [] for key, hist in self.db.iterator(prefix=b''): - flush_id, = unpack('>H', key[-2:]) + flush_id, = unpack_be_uint16_from(key[-2:]) if flush_id > utxo_flush_count: keys.append(key) @@ -126,7 +126,7 @@ class History(object): def flush(self): start_time = time.time() self.flush_count += 1 - flush_id = pack('>H', self.flush_count) + flush_id = pack_be_uint16(self.flush_count) unflushed = self.unflushed with self.db.write_batch() as batch: @@ -250,7 +250,7 @@ class History(object): write_size = 0 keys_to_delete.update(hist_map) for n, chunk in enumerate(util.chunks(full_hist, max_row_size)): - key = hashX + pack('>H', n) + key = hashX + pack_be_uint16(n) if hist_map.get(key) == chunk: keys_to_delete.remove(key) else: @@ -302,7 +302,7 @@ class History(object): # Loop over 2-byte prefixes cursor = self.comp_cursor while write_size < limit and cursor < 65536: - prefix = pack('>H', cursor) + prefix = pack_be_uint16(cursor) write_size += self._compact_prefix(prefix, write_items, keys_to_delete) cursor += 1 diff --git a/electrumx/wallet/bip32.py b/electrumx/wallet/bip32.py index e8c2274..0d1ef95 100644 --- a/electrumx/wallet/bip32.py +++ b/electrumx/wallet/bip32.py @@ -15,7 +15,8 @@ import ecdsa.numbertheory as NT from electrumx.lib.coins import Coin from electrumx.lib.hash import Base58, hmac_sha512, hash160 -from electrumx.lib.util import cachedproperty, bytes_to_int, int_to_bytes +from electrumx.lib.util import cachedproperty, bytes_to_int, int_to_bytes, \ + pack_be_uint32, unpack_be_uint32_from class DerivationError(Exception): @@ -65,7 +66,7 @@ class _KeyBase(object): raise ValueError('raw_serkey must have length 33') return (ver_bytes + bytes([self.depth]) - + self.parent_fingerprint() + struct.pack('>I', self.n) + + self.parent_fingerprint() + pack_be_uint32(self.n) + self.chain_code + raw_serkey) def fingerprint(self): @@ -142,7 +143,7 @@ class PubKey(_KeyBase): if not 0 <= n < (1 << 31): raise ValueError('invalid BIP32 public key child number') - msg = self.pubkey_bytes + struct.pack('>I', n) + msg = self.pubkey_bytes + pack_be_uint32(n) L, R = self._hmac_sha512(msg) curve = self.CURVE @@ -244,7 +245,7 @@ class PrivKey(_KeyBase): else: serkey = self.public_key.pubkey_bytes - msg = serkey + struct.pack('>I', n) + msg = serkey + pack_be_uint32(n) L, R = self._hmac_sha512(msg) curve = self.CURVE @@ -282,7 +283,7 @@ def _from_extended_key(ekey): is_public, coin = Coin.lookup_xverbytes(ekey[:4]) depth = ekey[4] fingerprint = ekey[5:9] # Not used - n, = struct.unpack('>I', ekey[9:13]) + n, = unpack_be_uint32_from(ekey[9:13]) chain_code = ekey[13:45] if is_public: diff --git a/tests/blocks/newyorkcoin_mainnet_3956926.json b/tests/blocks/newyorkcoin_mainnet_3956926.json index 268f39d..ef29bad 100644 --- a/tests/blocks/newyorkcoin_mainnet_3956926.json +++ b/tests/blocks/newyorkcoin_mainnet_3956926.json @@ -7,7 +7,7 @@ "0300b2fe16c049641ec63f9eb435b9d773c847bc0185c927d5b8d87cd1ad3095" ], "time": 1515308763, - "nonce": "1570457681", + "nonce": 1570457681, "bits": "1b172a4e", "previousblockhash": "10c91aefd6698f03a2820b8aa462738a5c1c21f4f975786a8eb1ba25e4b33af3", "block": "01000000f33ab3e425bab18e6a7875f9f4211c5c8a7362a48a0b82a2038f69d6ef1ac9109530add17cd8b8d527c98501bc47c873d7b935b49e3fc61e6449c016feb20003dbc6515a4e2a171b51489b5d0101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff5203be603c062f503253482f04dfc6515afabe6d6dce67985c41a5f1ba5d526614e1744010b5e6a34d33c50ece5d420d75da15454e100000000000000008b8000274020000000c2f30324d515156434c59582f00000000010010a5d4e80000001976a914f0a150ec5709fae1d1814227b69cd1f0baf528c588ac00000000" diff --git a/tests/lib/test_util.py b/tests/lib/test_util.py index 6186da6..1635cd3 100644 --- a/tests/lib/test_util.py +++ b/tests/lib/test_util.py @@ -71,25 +71,14 @@ def test_increment_byte_string(): assert util.increment_byte_string(b'\x01\x01') == b'\x01\x02' assert util.increment_byte_string(b'\xff\xff') is None + def test_bytes_to_int(): assert util.bytes_to_int(b'\x07[\xcd\x15') == 123456789 + def test_int_to_bytes(): assert util.int_to_bytes(456789) == b'\x06\xf8U' -def test_int_to_varint(): - with pytest.raises(ValueError): - util.int_to_varint(-1) - assert util.int_to_varint(0) == b'\0' - assert util.int_to_varint(5) == b'\5' - assert util.int_to_varint(252) == b'\xfc' - assert util.int_to_varint(253) == b'\xfd\xfd\0' - assert util.int_to_varint(65535) == b'\xfd\xff\xff' - assert util.int_to_varint(65536) == b'\xfe\0\0\1\0' - assert util.int_to_varint(2**32-1) == b'\xfe\xff\xff\xff\xff' - assert util.int_to_varint(2**32) == b'\xff\0\0\0\0\1\0\0\0' - assert util.int_to_varint(2**64-1) \ - == b'\xff\xff\xff\xff\xff\xff\xff\xff\xff' def test_LogicalFile(tmpdir): prefix = os.path.join(tmpdir, 'log') @@ -208,17 +197,17 @@ def test_protocol_version(): def test_unpackers(): b = bytes(range(256)) - assert util.unpack_int32_from(b, 0) == (50462976,) - assert util.unpack_int32_from(b, 42) == (757869354,) - assert util.unpack_int64_from(b, 0) == (506097522914230528,) - assert util.unpack_int64_from(b, 42) == (3544384782113450794,) + assert util.unpack_le_int32_from(b, 0) == (50462976,) + assert util.unpack_le_int32_from(b, 42) == (757869354,) + assert util.unpack_le_int64_from(b, 0) == (506097522914230528,) + assert util.unpack_le_int64_from(b, 42) == (3544384782113450794,) - assert util.unpack_uint16_from(b, 0) == (256,) - assert util.unpack_uint16_from(b, 42) == (11050,) - assert util.unpack_uint32_from(b, 0) == (50462976,) - assert util.unpack_uint32_from(b, 42) == (757869354,) - assert util.unpack_uint64_from(b, 0) == (506097522914230528,) - assert util.unpack_uint64_from(b, 42) == (3544384782113450794,) + assert util.unpack_le_uint16_from(b, 0) == (256,) + assert util.unpack_le_uint16_from(b, 42) == (11050,) + assert util.unpack_le_uint32_from(b, 0) == (50462976,) + assert util.unpack_le_uint32_from(b, 42) == (757869354,) + assert util.unpack_le_uint64_from(b, 0) == (506097522914230528,) + assert util.unpack_le_uint64_from(b, 42) == (3544384782113450794,) def test_hex_transforms(): h = "AABBCCDDEEFF" @@ -234,6 +223,20 @@ def test_pack_varint(): deser = tx.Deserializer(data) assert deser._read_varint() == n + import struct + with pytest.raises(struct.error): + util.pack_varint(-1) + assert util.pack_varint(0) == b'\0' + assert util.pack_varint(5) == b'\5' + assert util.pack_varint(252) == b'\xfc' + assert util.pack_varint(253) == b'\xfd\xfd\0' + assert util.pack_varint(65535) == b'\xfd\xff\xff' + assert util.pack_varint(65536) == b'\xfe\0\0\1\0' + assert util.pack_varint(2**32-1) == b'\xfe\xff\xff\xff\xff' + assert util.pack_varint(2**32) == b'\xff\0\0\0\0\1\0\0\0' + assert util.pack_varint(2**64-1) \ + == b'\xff\xff\xff\xff\xff\xff\xff\xff\xff' + def test_pack_varbytes(): tests = [b'', b'1', b'2' * 253, b'3' * 254, b'4' * 256, b'5' * 65536] diff --git a/tests/server/test_compaction.py b/tests/server/test_compaction.py index f9c9eb6..44017f1 100644 --- a/tests/server/test_compaction.py +++ b/tests/server/test_compaction.py @@ -2,12 +2,11 @@ import array import asyncio -from collections import defaultdict from os import environ, urandom -from struct import pack import random from electrumx.lib.hash import HASHX_LEN +from electrumx.lib.util import pack_be_uint16 from electrumx.server.env import Env from electrumx.server.db import DB @@ -49,7 +48,7 @@ def check_hashX_compaction(history): hist_list = [] hist_map = {} for flush_count, count in pairs: - key = hashX + pack('>H', flush_count) + key = hashX + pack_be_uint16(flush_count) hist = full_hist[cum * 4: (cum+count) * 4] hist_map[key] = hist hist_list.append(hist) @@ -65,10 +64,10 @@ def check_hashX_compaction(history): assert len(keys_to_delete) == 3 assert len(hist_map) == len(pairs) for n, item in enumerate(write_items): - assert item == (hashX + pack('>H', n), + assert item == (hashX + pack_be_uint16(n), full_hist[n * row_size: (n + 1) * row_size]) for flush_count, count in pairs: - assert hashX + pack('>H', flush_count) in keys_to_delete + assert hashX + pack_be_uint16(flush_count) in keys_to_delete # Check re-compaction is null hist_map = {key: value for key, value in write_items} @@ -87,7 +86,7 @@ def check_hashX_compaction(history): write_size = history._compact_hashX(hashX, hist_map, hist_list, write_items, keys_to_delete) assert write_size == len(hist_list[-1]) - assert write_items == [(hashX + pack('>H', 2), hist_list[-1])] + assert write_items == [(hashX + pack_be_uint16(2), hist_list[-1])] assert len(keys_to_delete) == 1 assert write_items[0][0] in keys_to_delete assert len(hist_map) == len(pairs) diff --git a/tests/test_blocks.py b/tests/test_blocks.py index d82dc5b..c8803d6 100644 --- a/tests/test_blocks.py +++ b/tests/test_blocks.py @@ -32,6 +32,7 @@ import pytest from electrumx.lib.coins import Coin from electrumx.lib.hash import hex_str_to_hash +from electrumx.lib.util import pack_be_uint32 BLOCKS_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), 'blocks') @@ -60,6 +61,13 @@ def test_block(block_details): raw_block = unhexlify(block_info['block']) block = coin.block(raw_block, block_info['height']) + h = coin.electrum_header(block.header, block_info['height']) + assert block_info['merkleroot'] == h['merkle_root'] + assert block_info['time'] == h['timestamp'] + assert block_info['previousblockhash'] == h['prev_block_hash'] + assert block_info['height'] == h['block_height'] + assert block_info['nonce'] == h['nonce'] + assert block_info['bits'] == pack_be_uint32(h['bits']).hex() assert coin.header_hash(block.header) == hex_str_to_hash(block_info['hash']) assert (coin.header_prevhash(block.header)