Save all UTXOs
Change the DB version
This commit is contained in:
Neil Booth
parent
4dac728984
commit
85786b87a2
19
lib/coins.py
19
lib/coins.py
@ -12,6 +12,7 @@ necessary for appropriate handling.
|
||||
'''
|
||||
|
||||
from decimal import Decimal
|
||||
from functools import partial
|
||||
import inspect
|
||||
import struct
|
||||
import sys
|
||||
@ -34,6 +35,7 @@ class Coin(object):
|
||||
DEFAULT_RPC_PORT = 8332
|
||||
VALUE_PER_COIN = 100000000
|
||||
CHUNK_SIZE=2016
|
||||
STRANGE_VERBYTE = 0xff
|
||||
|
||||
@classmethod
|
||||
def lookup_coin_class(cls, name, net):
|
||||
@ -53,11 +55,14 @@ class Coin(object):
|
||||
address = cls.P2PKH_hash168_from_hash160,
|
||||
script_hash = cls.P2SH_hash168_from_hash160,
|
||||
pubkey = cls.P2PKH_hash168_from_pubkey,
|
||||
unspendable = cls.hash168_from_unspendable,
|
||||
strange = cls.hash168_from_strange,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def hash168_from_script(cls, script):
|
||||
return ScriptPubKey.pay_to(script, cls.hash168_handlers)
|
||||
def hash168_from_script(cls):
|
||||
'''Returns a function that is passed a script to return a hash168.'''
|
||||
return partial(ScriptPubKey.pay_to, cls.hash168_handlers)
|
||||
|
||||
@staticmethod
|
||||
def lookup_xverbytes(verbytes):
|
||||
@ -86,6 +91,16 @@ class Coin(object):
|
||||
'''Return an address given a 21-byte hash.'''
|
||||
return Base58.encode_check(hash168)
|
||||
|
||||
@classmethod
|
||||
def hash168_from_unspendable(cls):
|
||||
'''Return a hash168 for an unspendable script.'''
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def hash168_from_strange(cls, script):
|
||||
'''Return a hash168 for a strange script.'''
|
||||
return bytes([cls.STRANGE_VERBYTE]) + hash160(script)
|
||||
|
||||
@classmethod
|
||||
def P2PKH_hash168_from_hash160(cls, hash160):
|
||||
'''Return a hash168 if hash160 is 160 bits otherwise None.'''
|
||||
|
||||
@ -56,6 +56,19 @@ assert OpCodes.OP_CHECKSIG == 0xac
|
||||
assert OpCodes.OP_CHECKMULTISIG == 0xae
|
||||
|
||||
|
||||
def _match_ops(ops, pattern):
|
||||
if len(ops) != len(pattern):
|
||||
return False
|
||||
for op, pop in zip(ops, pattern):
|
||||
if pop != op:
|
||||
# -1 means 'data push', whose op is an (op, data) tuple
|
||||
if pop == -1 and isinstance(op, tuple):
|
||||
continue
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
class ScriptPubKey(object):
|
||||
'''A class for handling a tx output script that gives conditions
|
||||
necessary for spending.
|
||||
@ -66,10 +79,11 @@ class ScriptPubKey(object):
|
||||
TO_P2SH_OPS = [OpCodes.OP_HASH160, -1, OpCodes.OP_EQUAL]
|
||||
TO_PUBKEY_OPS = [-1, OpCodes.OP_CHECKSIG]
|
||||
|
||||
PayToHandlers = namedtuple('PayToHandlers', 'address script_hash pubkey')
|
||||
PayToHandlers = namedtuple('PayToHandlers', 'address script_hash pubkey '
|
||||
'unspendable strange')
|
||||
|
||||
@classmethod
|
||||
def pay_to(cls, script, handlers):
|
||||
def pay_to(cls, handlers, script):
|
||||
'''Parse a script, invoke the appropriate handler and
|
||||
return the result.
|
||||
|
||||
@ -77,21 +91,25 @@ class ScriptPubKey(object):
|
||||
handlers.address(hash160)
|
||||
handlers.script_hash(hash160)
|
||||
handlers.pubkey(pubkey)
|
||||
or None is returned if the script is invalid or unregonised.
|
||||
handlers.unspendable()
|
||||
handlers.strange(script)
|
||||
'''
|
||||
try:
|
||||
ops, datas = Script.get_ops(script)
|
||||
ops = Script.get_ops(script)
|
||||
except ScriptError:
|
||||
return None
|
||||
return handlers.unspendable()
|
||||
|
||||
if Script.match_ops(ops, cls.TO_ADDRESS_OPS):
|
||||
return handlers.address(datas[2])
|
||||
if Script.match_ops(ops, cls.TO_P2SH_OPS):
|
||||
return handlers.script_hash(datas[1])
|
||||
if Script.match_ops(ops, cls.TO_PUBKEY_OPS):
|
||||
return handlers.pubkey(datas[0])
|
||||
match = _match_ops
|
||||
|
||||
return None
|
||||
if match(ops, cls.TO_ADDRESS_OPS):
|
||||
return handlers.address(ops[2][-1])
|
||||
if match(ops, cls.TO_P2SH_OPS):
|
||||
return handlers.script_hash(ops[1][-1])
|
||||
if match(ops, cls.TO_PUBKEY_OPS):
|
||||
return handlers.pubkey(ops[0][-1])
|
||||
if OpCodes.OP_RETURN in ops:
|
||||
return handlers.unspendable()
|
||||
return handlers.strange(script)
|
||||
|
||||
@classmethod
|
||||
def P2SH_script(cls, hash160):
|
||||
@ -141,54 +159,40 @@ class Script(object):
|
||||
|
||||
@classmethod
|
||||
def get_ops(cls, script):
|
||||
opcodes, datas = [], []
|
||||
ops = []
|
||||
|
||||
# The unpacks or script[n] below throw on truncated scripts
|
||||
try:
|
||||
n = 0
|
||||
while n < len(script):
|
||||
opcode, data = script[n], None
|
||||
op = script[n]
|
||||
n += 1
|
||||
|
||||
if opcode <= OpCodes.OP_PUSHDATA4:
|
||||
if op <= OpCodes.OP_PUSHDATA4:
|
||||
# Raw bytes follow
|
||||
if opcode < OpCodes.OP_PUSHDATA1:
|
||||
dlen = opcode
|
||||
elif opcode == OpCodes.OP_PUSHDATA1:
|
||||
if op < OpCodes.OP_PUSHDATA1:
|
||||
dlen = op
|
||||
elif op == OpCodes.OP_PUSHDATA1:
|
||||
dlen = script[n]
|
||||
n += 1
|
||||
elif opcode == OpCodes.OP_PUSHDATA2:
|
||||
(dlen,) = struct.unpack('<H', script[n: n + 2])
|
||||
elif op == OpCodes.OP_PUSHDATA2:
|
||||
dlen, = struct.unpack('<H', script[n: n + 2])
|
||||
n += 2
|
||||
else:
|
||||
(dlen,) = struct.unpack('<I', script[n: n + 4])
|
||||
dlen, = struct.unpack('<I', script[n: n + 4])
|
||||
n += 4
|
||||
data = script[n:n + dlen]
|
||||
if len(data) != dlen:
|
||||
raise ScriptError('truncated script')
|
||||
if n + dlen > len(script):
|
||||
raise IndexError
|
||||
op = (op, script[n:n + dlen])
|
||||
n += dlen
|
||||
|
||||
opcodes.append(opcode)
|
||||
datas.append(data)
|
||||
ops.append(op)
|
||||
except:
|
||||
# Truncated script; e.g. tx_hash
|
||||
# ebc9fa1196a59e192352d76c0f6e73167046b9d37b8302b6bb6968dfd279b767
|
||||
raise ScriptError('truncated script')
|
||||
|
||||
return opcodes, datas
|
||||
|
||||
@classmethod
|
||||
def match_ops(cls, ops, pattern):
|
||||
if len(ops) != len(pattern):
|
||||
return False
|
||||
for op, pop in zip(ops, pattern):
|
||||
if pop != op:
|
||||
# -1 Indicates data push expected
|
||||
if pop == -1 and OpCodes.OP_0 <= op <= OpCodes.OP_PUSHDATA4:
|
||||
continue
|
||||
return False
|
||||
|
||||
return True
|
||||
return ops
|
||||
|
||||
@classmethod
|
||||
def push_data(cls, data):
|
||||
|
||||
@ -28,8 +28,6 @@ from server.storage import open_db
|
||||
# Limits single address history to ~ 65536 * HIST_ENTRIES_PER_KEY entries
|
||||
HIST_ENTRIES_PER_KEY = 1024
|
||||
HIST_VALUE_BYTES = HIST_ENTRIES_PER_KEY * 4
|
||||
NO_HASH_168 = bytes([255]) * 21
|
||||
NO_CACHE_ENTRY = NO_HASH_168 + bytes(12)
|
||||
|
||||
|
||||
def formatted_time(t):
|
||||
@ -209,7 +207,7 @@ class MemPool(LoggedClass):
|
||||
|
||||
# The mempool is unordered, so process all outputs first so
|
||||
# that looking for inputs has full info.
|
||||
script_hash168 = self.bp.coin.hash168_from_script
|
||||
script_hash168 = self.bp.coin.hash168_from_script()
|
||||
db_utxo_lookup = self.bp.db_utxo_lookup
|
||||
|
||||
def txout_pair(txout):
|
||||
@ -658,8 +656,6 @@ class BlockProcessor(server.db.DB):
|
||||
self.logger.info('backing up history to height {:,d} tx_count {:,d}'
|
||||
.format(self.height, self.tx_count))
|
||||
|
||||
# Drop any NO_CACHE entry
|
||||
hash168s.discard(NO_CACHE_ENTRY)
|
||||
assert not self.history
|
||||
|
||||
nremoves = 0
|
||||
@ -765,7 +761,7 @@ class BlockProcessor(server.db.DB):
|
||||
# Use local vars for speed in the loops
|
||||
history = self.history
|
||||
tx_num = self.tx_count
|
||||
script_hash168 = self.coin.hash168_from_script
|
||||
script_hash168 = self.coin.hash168_from_script()
|
||||
s_pack = pack
|
||||
|
||||
for tx, tx_hash in zip(txs, tx_hashes):
|
||||
@ -781,15 +777,13 @@ class BlockProcessor(server.db.DB):
|
||||
|
||||
# Add the new UTXOs
|
||||
for idx, txout in enumerate(tx.outputs):
|
||||
# Get the hash168. Ignore scripts we can't grok.
|
||||
# Get the hash168. Ignore unspendable outputs
|
||||
hash168 = script_hash168(txout.pk_script)
|
||||
if hash168:
|
||||
hash168s.add(hash168)
|
||||
put_utxo(tx_hash + s_pack('<H', idx),
|
||||
hash168 + tx_numb + s_pack('<Q', txout.value))
|
||||
|
||||
# Drop any NO_CACHE entry
|
||||
hash168s.discard(NO_CACHE_ENTRY)
|
||||
for hash168 in hash168s:
|
||||
history[hash168].append(tx_num)
|
||||
self.history_size += len(hash168s)
|
||||
@ -913,15 +907,15 @@ class BlockProcessor(server.db.DB):
|
||||
the tx in which the UTXO was created. As this is not unique there
|
||||
will are potential collisions when saving and looking up UTXOs;
|
||||
hence why the second table has a list as its value. The collision
|
||||
can be resolved with the tx_num. The collision rate is almost
|
||||
zero (I believe there are around 100 collisions in the whole
|
||||
bitcoin blockchain).
|
||||
can be resolved with the tx_num. The collision rate is low (<0.1%).
|
||||
'''
|
||||
|
||||
def spend_utxo(self, tx_hash, tx_idx):
|
||||
'''Spend a UTXO and return the 33-byte value.
|
||||
|
||||
If the UTXO is not in the cache it may be on disk.
|
||||
If the UTXO is not in the cache it must be on disk. We store
|
||||
all UTXOs so not finding one indicates a logic error or DB
|
||||
corruption.
|
||||
'''
|
||||
# Fast track is it being in the cache
|
||||
idx_packed = pack('<H', tx_idx)
|
||||
@ -935,46 +929,42 @@ class BlockProcessor(server.db.DB):
|
||||
# The 4 is the COMPRESSED_TX_HASH_LEN
|
||||
db_key = b'h' + tx_hash[:4] + idx_packed
|
||||
db_value = self.db_cache_get(db_key)
|
||||
if db_value is None:
|
||||
# Probably a strange UTXO
|
||||
return NO_CACHE_ENTRY
|
||||
|
||||
# FIXME: this matches what we did previously but until we store
|
||||
# all UTXOs isn't safe
|
||||
if len(db_value) == 25:
|
||||
udb_key = b'u' + db_value + idx_packed
|
||||
utxo_value_packed = self.db.get(udb_key)
|
||||
if utxo_value_packed:
|
||||
# Remove the UTXO from both tables
|
||||
self.db_deletes += 1
|
||||
self.db_cache[db_key] = None
|
||||
self.db_cache[udb_key] = None
|
||||
return db_value + utxo_value_packed
|
||||
# Fall through to below
|
||||
|
||||
assert len(db_value) % 25 == 0
|
||||
|
||||
# Find which entry, if any, the TX_HASH matches.
|
||||
for n in range(0, len(db_value), 25):
|
||||
tx_num, = unpack('<I', db_value[n+21:n+25])
|
||||
hash, height = self.get_tx_hash(tx_num)
|
||||
if hash == tx_hash:
|
||||
match = db_value[n:n+25]
|
||||
udb_key = b'u' + match + idx_packed
|
||||
if db_value:
|
||||
# FIXME: this matches what we did previously but until we store
|
||||
# all UTXOs isn't safe
|
||||
if len(db_value) == 25:
|
||||
udb_key = b'u' + db_value + idx_packed
|
||||
utxo_value_packed = self.db.get(udb_key)
|
||||
if utxo_value_packed:
|
||||
# Remove the UTXO from both tables
|
||||
self.db_deletes += 1
|
||||
self.db_cache[db_key] = db_value[:n] + db_value[n + 25:]
|
||||
self.db_cache[db_key] = None
|
||||
self.db_cache[udb_key] = None
|
||||
return match + utxo_value_packed
|
||||
return db_value + utxo_value_packed
|
||||
# Fall through to below loop for error
|
||||
|
||||
# Uh-oh, this should not happen...
|
||||
raise self.DBError('UTXO {} / {:,d} not found, key {}'
|
||||
.format(hash_to_str(tx_hash), tx_idx,
|
||||
bytes(key).hex()))
|
||||
assert len(db_value) % 25 == 0
|
||||
|
||||
return NO_CACHE_ENTRY
|
||||
# Find which entry, if any, the TX_HASH matches.
|
||||
for n in range(0, len(db_value), 25):
|
||||
tx_num, = unpack('<I', db_value[n + 21:n + 25])
|
||||
hash, height = self.get_tx_hash(tx_num)
|
||||
if hash == tx_hash:
|
||||
match = db_value[n:n+25]
|
||||
udb_key = b'u' + match + idx_packed
|
||||
utxo_value_packed = self.db.get(udb_key)
|
||||
if utxo_value_packed:
|
||||
# Remove the UTXO from both tables
|
||||
self.db_deletes += 1
|
||||
self.db_cache[db_key] = db_value[:n] + db_value[n+25:]
|
||||
self.db_cache[udb_key] = None
|
||||
return match + utxo_value_packed
|
||||
|
||||
raise self.DBError('UTXO {} / {:,d} not found in "u" table'
|
||||
.format(hash_to_str(tx_hash), tx_idx))
|
||||
|
||||
raise ChainError('UTXO {} / {:,d} not found in "h" table'
|
||||
.format(hash_to_str(tx_hash), tx_idx))
|
||||
|
||||
def db_cache_get(self, key):
|
||||
'''Fetch a 'h' value from the DB through our write cache.'''
|
||||
|
||||
@ -29,7 +29,7 @@ class DB(LoggedClass):
|
||||
it was shutdown uncleanly.
|
||||
'''
|
||||
|
||||
VERSIONS = [0]
|
||||
VERSIONS = [2]
|
||||
|
||||
class MissingUTXOError(Exception):
|
||||
'''Raised if a mempool tx input UTXO couldn't be found.'''
|
||||
|
||||
Loading…
Reference in New Issue
Block a user