RanchiMall/electrumx
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'develop'

Browse Source
This commit is contained in:
Neil Booth 2017-01-08 10:51:24 +09:00
commit defadc580d
11 changed files with 188 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
README.rst
View File

@ -115,7 +115,6 @@ Roadmap Pre-1.0
===============
- minor code cleanups.
- support bitcoin testnet with Satoshi bitcoind 0.13.1
- implement simple protocol to discover peers without resorting to IRC.
This may slip to post 1.0
@ -142,6 +141,24 @@ version prior to the release of 1.0.
ChangeLog
=========
Version 0.10.2
--------------
* Note the **NETWORK** environment variable was renamed **NET** to
bring it into line with lib/coins.py.
* The genesis hash is now compared with the genesis hash expected by
**COIN** and **NET**. This sanity check was not done previously, so
you could easily be syncing to a network daemon different to what
you thought.
* SegWit-compatible testnet support for bitcoin core versions 0.13.1
or higher. Resolves issue `#92#`. Testnet worked with prior
versions of ElectrumX as long as you used an older bitcoind too,
such as 0.13.0 or Bitcoin Unlimited.
**Note**: for testnet, you need to set *NET** to *testnet-segwit* if
using recent RPC incompatible core bitcoinds, or *testnet* if using
older RPC compatible bitcoinds.
Version 0.10.1
--------------
@ -167,6 +184,12 @@ variables to use roughly the same amount of memory.
For now this code should be considered experimental; if you want
stability please stick with the 0.9 series.
Version 0.9.23
--------------
* Backport of the fix for issue `#94#` - stale references to old
sessions. This would effectively memory and network handles.
Version 0.9.22
--------------
@ -334,6 +357,7 @@ Version 0.9.0
.. _#75: https://github.com/kyuupichan/electrumx/issues/75
.. _#88: https://github.com/kyuupichan/electrumx/issues/88
.. _#89: https://github.com/kyuupichan/electrumx/issues/89
.. _#92: https://github.com/kyuupichan/electrumx/issues/92
.. _#93: https://github.com/kyuupichan/electrumx/issues/93
.. _#94: https://github.com/kyuupichan/electrumx/issues/94
.. _docs/HOWTO.rst: https://github.com/kyuupichan/electrumx/blob/master/docs/HOWTO.rst

10
docs/ENVIRONMENT.rst
View File

@ -31,7 +31,7 @@ These environment variables are always required:
The leading `http://` is optional, as is the trailing slash. The
`:port` part is also optional and will default to the standard RPC
port for **COIN** and **NETWORK** if omitted.
port for **COIN** and **NET** if omitted.
For the `run` script
@ -58,7 +58,7 @@ These environment variables are optional:
Must be a *NAME* from one of the **Coin** classes in
`lib/coins.py`_. Defaults to `Bitcoin`.
* **NETWORK**
* **NET**
Must be a *NET* from one of the **Coin** classes in `lib/coins.py`_.
Defaults to `mainnet`.
@ -77,7 +77,7 @@ These environment variables are optional:
The maximum number of blocks to be able to handle in a chain
reorganisation. ElectrumX retains some fairly compact undo
information for this many blocks in levelDB. The default is a
function of **COIN** and **NETWORK**; for Bitcoin mainnet it is 200.
function of **COIN** and **NET**; for Bitcoin mainnet it is 200.
* **HOST**
@ -98,7 +98,7 @@ These environment variables are optional:
ElectrumX will listen on this port for local RPC connections.
ElectrumX listens for RPC connections unless this is explicitly set
to blank. The default is appropriate for **COIN** and **NETWORK**
to blank. The default is appropriate for **COIN** and **NET**
(e.g., 8000 for Bitcoin mainnet) if not set.
* **DONATION_ADDRESS**
@ -223,7 +223,7 @@ connectivity on IRC:
The nick to use when connecting to IRC. The default is a hash of
**REPORT_HOST**. Either way a prefix will be prepended depending on
**COIN** and **NETWORK**.
**COIN** and **NET**.
* **REPORT_HOST**

71
lib/coins.py
View File

@ -21,8 +21,8 @@ import sys
from lib.hash import Base58, hash160, ripemd160, double_sha256, hash_to_str
from lib.script import ScriptPubKey
from lib.tx import Deserializer
from lib.util import cachedproperty, subclasses
from lib.tx import Deserializer, DeserializerSegWit
import lib.util as util
class CoinError(Exception):
@ -46,7 +46,7 @@ class Coin(object):
'''Return a coin class given name and network.
Raise an exception if unrecognised.'''
for coin in subclasses(Coin):
for coin in util.subclasses(Coin):
if (coin.NAME.lower() == name.lower()
and coin.NET.lower() == net.lower()):
return coin
@ -70,6 +70,20 @@ class Coin(object):
def daemon_urls(cls, urls):
return [cls.sanitize_url(url) for url in urls.split(',')]
@classmethod
def genesis_block(cls, block):
'''Check the Genesis block is the right one for this coin.
Return the block less its unspendable coinbase.
'''
header = block[:cls.header_len(0)]
header_hex_hash = hash_to_str(cls.header_hash(header))
if header_hex_hash != cls.GENESIS_HASH:
raise CoinError('genesis block has hash {} expected {}'
.format(header_hex_hash, cls.GENESIS_HASH))
return header + bytes(1)
@classmethod
def hashX_from_script(cls, script):
'''Returns a hashX from a script.'''
@ -78,7 +92,7 @@ class Coin(object):
return None
return sha256(script).digest()[:cls.HASHX_LEN]
@cachedproperty
@util.cachedproperty
def address_handlers(cls):
return ScriptPubKey.PayToHandlers(
address = cls.P2PKH_address_from_hash160,
@ -204,11 +218,14 @@ class Coin(object):
@classmethod
def read_block(cls, block, height):
'''Return a tuple (header, tx_hashes, txs) given a raw block at
the given height.'''
'''Returns a pair (header, tx_list) given a raw block and height.
tx_list is a list of (deserialized_tx, tx_hash) pairs.
'''
deserializer = cls.deserializer()
hlen = cls.header_len(height)
header, rest = block[:hlen], block[hlen:]
return (header, ) + Deserializer(rest).read_block()
return (header, deserializer(rest).read_block())
@classmethod
def decimal_value(cls, value):
@ -234,6 +251,10 @@ class Coin(object):
'nonce': nonce,
}
@classmethod
def deserializer(cls):
return Deserializer
class Bitcoin(Coin):
NAME = "Bitcoin"
@ -244,8 +265,8 @@ class Bitcoin(Coin):
P2PKH_VERBYTE = 0x00
P2SH_VERBYTE = 0x05
WIF_BYTE = 0x80
GENESIS_HASH=(b'000000000019d6689c085ae165831e93'
b'4ff763ae46a2a6c172b3f1b60a8ce26f')
GENESIS_HASH=('000000000019d6689c085ae165831e93'
'4ff763ae46a2a6c172b3f1b60a8ce26f')
TX_COUNT = 156335304
TX_COUNT_HEIGHT = 429972
TX_PER_BLOCK = 1800
@ -262,13 +283,29 @@ class BitcoinTestnet(Bitcoin):
P2PKH_VERBYTE = 0x6f
P2SH_VERBYTE = 0xc4
WIF_BYTE = 0xef
GENESIS_HASH=(b'000000000933ea01ad0ee984209779ba'
b'aec3ced90fa3f408719526f8d77f4943')
GENESIS_HASH=('000000000933ea01ad0ee984209779ba'
'aec3ced90fa3f408719526f8d77f4943')
REORG_LIMIT = 2000
TX_COUNT = 12242438
TX_COUNT_HEIGHT = 1035428
TX_PER_BLOCK = 21
IRC_PREFIX = "ET_"
RPC_PORT = 18332
class BitcoinTestnetSegWit(BitcoinTestnet):
'''Bitcoin Testnet for Core bitcoind >= 0.13.1.
Unfortunately 0.13.1 broke backwards compatibility of the RPC
interface's TX serialization, SegWit transactions serialize
differently than with earlier versions. If you are using such a
bitcoind on testnet, you must use this class as your "COIN".
'''
NET = "testnet-segwit"
@classmethod
def deserializer(cls):
return DeserializerSegWit
class Litecoin(Coin):
@ -280,8 +317,8 @@ class Litecoin(Coin):
P2PKH_VERBYTE = 0x30
P2SH_VERBYTE = 0x05
WIF_BYTE = 0xb0
GENESIS_HASH=(b'000000000019d6689c085ae165831e93'
b'4ff763ae46a2a6c172b3f1b60a8ce26f')
GENESIS_HASH=('000000000019d6689c085ae165831e93'
'4ff763ae46a2a6c172b3f1b60a8ce26f')
TX_COUNT = 8908766
TX_COUNT_HEIGHT = 1105256
TX_PER_BLOCK = 10
@ -355,8 +392,8 @@ class Dash(Coin):
NET = "mainnet"
XPUB_VERBYTES = bytes.fromhex("02fe52cc")
XPRV_VERBYTES = bytes.fromhex("02fe52f8")
GENESIS_HASH = (b'00000ffd590b1485b3caadc19b22e637'
b'9c733355108f107a430458cdf3407ab6')
GENESIS_HASH = ('00000ffd590b1485b3caadc19b22e637'
'9c733355108f107a430458cdf3407ab6')
P2PKH_VERBYTE = 0x4c
P2SH_VERBYTE = 0x10
WIF_BYTE = 0xcc
@ -378,8 +415,8 @@ class DashTestnet(Dash):
NET = "testnet"
XPUB_VERBYTES = bytes.fromhex("3a805837")
XPRV_VERBYTES = bytes.fromhex("3a8061a0")
GENESIS_HASH = (b'00000bafbc94add76cb75e2ec9289483'
b'7288a481e5c005f6563d91623bf8bc2c')
GENESIS_HASH = ('00000bafbc94add76cb75e2ec9289483'
'7288a481e5c005f6563d91623bf8bc2c')
P2PKH_VERBYTE = 0x8c
P2SH_VERBYTE = 0x13
WIF_BYTE = 0xef

84
lib/tx.py
View File

@ -72,28 +72,25 @@ class Deserializer(object):
self.cursor = 0
def read_tx(self):
'''Return a (Deserialized TX, TX_HASH) pair.
The hash needs to be reversed for human display; for efficiency
we process it in the natural serialized order.
'''
start = self.cursor
return Tx(
self._read_le_int32(), # version
self._read_inputs(), # inputs
self._read_outputs(), # outputs
self._read_le_uint32() # locktime
)
), double_sha256(self.binary[start:self.cursor])
def read_block(self):
tx_hashes = []
txs = []
binary = self.binary
hash = double_sha256
'''Returns a list of (deserialized_tx, tx_hash) pairs.'''
read_tx = self.read_tx
append_hash = tx_hashes.append
for n in range(self._read_varint()):
start = self.cursor
txs.append(read_tx())
# Note this hash needs to be reversed for human display
# For efficiency we store it in the natural serialized order
append_hash(hash(binary[start:self.cursor]))
assert self.cursor == len(binary)
return tx_hashes, txs
txs = [read_tx() for n in range(self._read_varint())]
assert self.cursor == len(self.binary)
return txs
def _read_inputs(self):
read_input = self._read_input
@ -161,3 +158,62 @@ class Deserializer(object):
result, = unpack_from('<Q', self.binary, self.cursor)
self.cursor += 8
return result
class TxSegWit(namedtuple("Tx", "version marker flag inputs outputs "
"witness locktime")):
'''Class representing a SegWit transaction.'''
@cachedproperty
def is_coinbase(self):
return self.inputs[0].is_coinbase
class DeserializerSegWit(Deserializer):
# https://bitcoincore.org/en/segwit_wallet_dev/#transaction-serialization
def _read_byte(self):
cursor = self.cursor
self.cursor += 1
return self.binary[cursor]
def _read_witness(self, fields):
read_witness_field = self._read_witness_field
return [read_witness_field() for i in range(fields)]
def _read_witness_field(self):
read_varbytes = self._read_varbytes
return [read_varbytes() for i in range(self._read_varint())]
def read_tx(self):
'''Return a (Deserialized TX, TX_HASH) pair.
The hash needs to be reversed for human display; for efficiency
we process it in the natural serialized order.
'''
marker = self.binary[self.cursor + 4]
if marker:
return super().read_tx()
# Ugh, this is nasty.
start = self.cursor
version = self._read_le_int32()
orig_ser = self.binary[start:self.cursor]
marker = self._read_byte()
flag = self._read_byte()
start = self.cursor
inputs = self._read_inputs()
outputs = self._read_outputs()
orig_ser += self.binary[start:self.cursor]
witness = self._read_witness(len(inputs))
start = self.cursor
locktime = self._read_le_uint32()
orig_ser += self.binary[start:self.cursor]
return TxSegWit(version, marker, flag, inputs,
outputs, witness, locktime), double_sha256(orig_ser)

2
samples/systemd/electrumx.conf
View File

@ -42,7 +42,7 @@
#MISC
#
#COIN = Bitcoin # lib/coins.py
#NETWORK = mainnet # lib/coins.py
#NET = mainnet # lib/coins.py
#DB_ENGINE = leveldb
#leveldb, rocksdb, lmdb (You'll need to install appropriate python packages)

35
server/block_processor.py
View File

@ -125,9 +125,11 @@ class Prefetcher(LoggedClass):
assert count == len(blocks)
# Strip the unspendable genesis coinbase
# Special handling for genesis block
if first == 0:
blocks[0] = blocks[0][:self.coin.header_len(0)] + bytes(1)
blocks[0] = self.coin.genesis_block(blocks[0])
self.logger.info('verified genesis block with hash {}'
.format(hex_hashes[0]))
# Update our recent average block size estimate
size = sum(len(block) for block in blocks)
@ -444,7 +446,9 @@ class BlockProcessor(server.db.DB):
utxo_cache_size = len(self.utxo_cache) * 205
db_deletes_size = len(self.db_deletes) * 57
hist_cache_size = len(self.history) * 180 + self.history_size * 4
tx_hash_size = (self.tx_count - self.fs_tx_count) * 74
# Roughly ntxs * 32 + nblocks * 42
tx_hash_size = ((self.tx_count - self.fs_tx_count) * 32
+ (self.height - self.fs_height) * 42)
utxo_MB = (db_deletes_size + utxo_cache_size) // one_MB
hist_MB = (hist_cache_size + tx_hash_size) // one_MB
@ -458,28 +462,28 @@ class BlockProcessor(server.db.DB):
if utxo_MB + hist_MB >= self.cache_MB or hist_MB >= self.cache_MB // 5:
self.flush(utxo_MB >= self.cache_MB * 4 // 5)
def fs_advance_block(self, header, tx_hashes, txs):
def fs_advance_block(self, header, txs):
'''Update unflushed FS state for a new block.'''
prior_tx_count = self.tx_counts[-1] if self.tx_counts else 0
# Cache the new header, tx hashes and cumulative tx count
self.headers.append(header)
self.tx_hashes.append(tx_hashes)
self.tx_hashes.append(b''.join(tx_hash for tx, tx_hash in txs))
self.tx_counts.append(prior_tx_count + len(txs))
def advance_block(self, block, touched):
header, tx_hashes, txs = self.coin.read_block(block, self.height + 1)
header, txs = self.coin.read_block(block, self.height + 1)
if self.tip != self.coin.header_prevhash(header):
raise ChainReorg
self.fs_advance_block(header, tx_hashes, txs)
self.fs_advance_block(header, txs)
self.tip = self.coin.header_hash(header)
self.height += 1
undo_info = self.advance_txs(tx_hashes, txs, touched)
undo_info = self.advance_txs(txs, touched)
if self.daemon.cached_height() - self.height <= self.env.reorg_limit:
self.write_undo_info(self.height, b''.join(undo_info))
def advance_txs(self, tx_hashes, txs, touched):
def advance_txs(self, txs, touched):
undo_info = []
# Use local vars for speed in the loops
@ -492,7 +496,7 @@ class BlockProcessor(server.db.DB):
spend_utxo = self.spend_utxo
undo_info_append = undo_info.append
for tx, tx_hash in zip(txs, tx_hashes):
for tx, tx_hash in txs:
hashXs = set()
add_hashX = hashXs.add
tx_numb = s_pack('<I', tx_num)
@ -533,14 +537,14 @@ class BlockProcessor(server.db.DB):
self.assert_flushed()
for block in blocks:
header, tx_hashes, txs = self.coin.read_block(block, self.height)
header, txs = self.coin.read_block(block, self.height)
header_hash = self.coin.header_hash(header)
if header_hash != self.tip:
raise ChainError('backup block {} is not tip {} at height {:,d}'
.format(hash_to_str(header_hash),
hash_to_str(self.tip), self.height))
self.backup_txs(tx_hashes, txs, touched)
self.backup_txs(txs, touched)
self.tip = self.coin.header_prevhash(header)
assert self.height >= 0
self.height -= 1
@ -553,7 +557,7 @@ class BlockProcessor(server.db.DB):
touched.discard(None)
self.backup_flush(touched)
def backup_txs(self, tx_hashes, txs, touched):
def backup_txs(self, txs, touched):
# Prevout values, in order down the block (coinbase first if present)
# undo_info is in reverse block order
undo_info = self.read_undo_info(self.height)
@ -569,10 +573,7 @@ class BlockProcessor(server.db.DB):
script_hashX = self.coin.hashX_from_script
undo_entry_len = 12 + self.coin.HASHX_LEN
rtxs = reversed(txs)
rtx_hashes = reversed(tx_hashes)
for tx_hash, tx in zip(rtx_hashes, rtxs):
for tx, tx_hash in reversed(txs):
for idx, txout in enumerate(tx.outputs):
# Spend the TX outputs. Be careful with unspendable
# outputs - we didn't save those in the first place.

9
server/db.py
View File

@ -10,7 +10,6 @@
import array
import ast
import itertools
import os
from struct import pack, unpack
from bisect import bisect_left, bisect_right
@ -143,7 +142,11 @@ class DB(util.LoggedClass):
raise self.DBError('your DB version is {} but this software '
'only handles versions {}'
.format(self.db_version, self.DB_VERSIONS))
if state['genesis'] != self.coin.GENESIS_HASH:
# backwards compat
genesis_hash = state['genesis']
if isinstance(genesis_hash, bytes):
genesis_hash = genesis_hash.decode()
if genesis_hash != self.coin.GENESIS_HASH:
raise self.DBError('DB genesis hash {} does not match coin {}'
.format(state['genesis_hash'],
self.coin.GENESIS_HASH))
@ -234,7 +237,7 @@ class DB(util.LoggedClass):
assert len(self.tx_hashes) == blocks_done
assert len(self.tx_counts) == new_height + 1
hashes = b''.join(itertools.chain(*block_tx_hashes))
hashes = b''.join(block_tx_hashes)
assert len(hashes) % 32 == 0
assert len(hashes) // 32 == txs_done

4
server/env.py
View File

@ -22,9 +22,9 @@ class Env(LoggedClass):
def __init__(self):
super().__init__()
self.obsolete(['UTXO_MB', 'HIST_MB'])
self.obsolete(['UTXO_MB', 'HIST_MB', 'NETWORK'])
coin_name = self.default('COIN', 'Bitcoin')
network = self.default('NETWORK', 'mainnet')
network = self.default('NET', 'mainnet')
self.coin = Coin.lookup_coin_class(coin_name, network)
self.db_dir = self.required('DB_DIRECTORY')
self.cache_MB = self.integer('CACHE_MB', 1200)

7
server/mempool.py
View File

@ -13,7 +13,6 @@ import time
from collections import defaultdict
from lib.hash import hash_to_str, hex_str_to_hash
from lib.tx import Deserializer
import lib.util as util
from server.daemon import DaemonError
@ -200,6 +199,7 @@ class MemPool(util.LoggedClass):
not depend on the result remaining the same are fine.
'''
script_hashX = self.coin.hashX_from_script
deserializer = self.coin.deserializer()
db_utxo_lookup = self.db.db_utxo_lookup
txs = self.txs
@ -207,7 +207,7 @@ class MemPool(util.LoggedClass):
for tx_hash, raw_tx in raw_tx_map.items():
if not tx_hash in txs:
continue
tx = Deserializer(raw_tx).read_tx()
tx, _tx_hash = deserializer(raw_tx).read_tx()
# Convert the tx outputs into (hashX, value) pairs
txout_pairs = [(script_hashX(txout.pk_script), txout.value)
@ -271,6 +271,7 @@ class MemPool(util.LoggedClass):
if not hashX in self.hashXs:
return []
deserializer = self.coin.deserializer()
hex_hashes = self.hashXs[hashX]
raw_txs = await self.daemon.getrawtransactions(hex_hashes)
result = []
@ -281,7 +282,7 @@ class MemPool(util.LoggedClass):
txin_pairs, txout_pairs = item
tx_fee = (sum(v for hashX, v in txin_pairs)
- sum(v for hashX, v in txout_pairs))
tx = Deserializer(raw_tx).read_tx()
tx, tx_hash = deserializer(raw_tx).read_tx()
unconfirmed = any(txin.prev_hash in self.txs for txin in tx.inputs)
result.append((hex_hash, tx_fee, unconfirmed))
return result

4
server/protocol.py
View File

@ -14,7 +14,6 @@ import traceback
from lib.hash import sha256, double_sha256, hash_to_str, hex_str_to_hash
from lib.jsonrpc import JSONRPC
from lib.tx import Deserializer
from server.daemon import DaemonError
from server.version import VERSION
@ -427,7 +426,8 @@ class ElectrumX(Session):
if not raw_tx:
return None
raw_tx = bytes.fromhex(raw_tx)
tx = Deserializer(raw_tx).read_tx()
deserializer = self.coin.deserializer()
tx, tx_hash = deserializer(raw_tx).read_tx()
if index >= len(tx.outputs):
return None
return self.coin.address_from_script(tx.outputs[index].pk_script)

2
server/version.py
View File

@ -1 +1 @@
VERSION = "ElectrumX 0.10.1"
VERSION = "ElectrumX 0.10.2"