448 lines
17 KiB
Python
448 lines
17 KiB
Python
# Copyright (c) 2016, Neil Booth
|
|
# Copyright (c) 2017, the ElectrumX authors
|
|
#
|
|
# All rights reserved.
|
|
#
|
|
# See the file "LICENCE" for information about the copyright
|
|
# and warranty status of this software.
|
|
|
|
'''Interface to the blockchain database.'''
|
|
|
|
|
|
import array
|
|
import ast
|
|
import os
|
|
from bisect import bisect_right
|
|
from collections import namedtuple
|
|
from glob import glob
|
|
from struct import pack, unpack
|
|
|
|
import electrumx.lib.util as util
|
|
from electrumx.lib.hash import hash_to_hex_str, HASHX_LEN
|
|
from electrumx.server.storage import db_class
|
|
from electrumx.server.history import History
|
|
|
|
|
|
UTXO = namedtuple("UTXO", "tx_num tx_pos tx_hash height value")
|
|
|
|
|
|
class DB(object):
|
|
'''Simple wrapper of the backend database for querying.
|
|
|
|
Performs no DB update, though the DB will be cleaned on opening if
|
|
it was shutdown uncleanly.
|
|
'''
|
|
|
|
DB_VERSIONS = [6]
|
|
|
|
class DBError(Exception):
|
|
'''Raised on general DB errors generally indicating corruption.'''
|
|
|
|
def __init__(self, env):
|
|
self.logger = util.class_logger(__name__, self.__class__.__name__)
|
|
self.env = env
|
|
self.coin = env.coin
|
|
|
|
# Setup block header size handlers
|
|
if self.coin.STATIC_BLOCK_HEADERS:
|
|
self.header_offset = self.coin.static_header_offset
|
|
self.header_len = self.coin.static_header_len
|
|
else:
|
|
self.header_offset = self.dynamic_header_offset
|
|
self.header_len = self.dynamic_header_len
|
|
|
|
self.logger.info(f'switching current directory to {env.db_dir}')
|
|
os.chdir(env.db_dir)
|
|
|
|
self.db_class = db_class(self.env.db_engine)
|
|
self.history = History()
|
|
self.utxo_db = None
|
|
self.tx_counts = None
|
|
|
|
self.logger.info(f'using {self.env.db_engine} for DB backend')
|
|
|
|
self.headers_file = util.LogicalFile('meta/headers', 2, 16000000)
|
|
self.tx_counts_file = util.LogicalFile('meta/txcounts', 2, 2000000)
|
|
self.hashes_file = util.LogicalFile('meta/hashes', 4, 16000000)
|
|
if not self.coin.STATIC_BLOCK_HEADERS:
|
|
self.headers_offsets_file = util.LogicalFile(
|
|
'meta/headers_offsets', 2, 16000000)
|
|
|
|
async def _read_tx_counts(self):
|
|
if self.tx_counts is not None:
|
|
return
|
|
# tx_counts[N] has the cumulative number of txs at the end of
|
|
# height N. So tx_counts[0] is 1 - the genesis coinbase
|
|
size = (self.db_height + 1) * 4
|
|
tx_counts = self.tx_counts_file.read(0, size)
|
|
assert len(tx_counts) == size
|
|
self.tx_counts = array.array('I', tx_counts)
|
|
if self.tx_counts:
|
|
assert self.db_tx_count == self.tx_counts[-1]
|
|
else:
|
|
assert self.db_tx_count == 0
|
|
|
|
async def _open_dbs(self, for_sync):
|
|
assert self.utxo_db is None
|
|
|
|
# First UTXO DB
|
|
self.utxo_db = self.db_class('utxo', for_sync)
|
|
if self.utxo_db.is_new:
|
|
self.logger.info('created new database')
|
|
self.logger.info('creating metadata directory')
|
|
os.mkdir('meta')
|
|
with util.open_file('COIN', create=True) as f:
|
|
f.write(f'ElectrumX databases and metadata for '
|
|
f'{self.coin.NAME} {self.coin.NET}'.encode())
|
|
if not self.coin.STATIC_BLOCK_HEADERS:
|
|
self.headers_offsets_file.write(0, bytes(8))
|
|
else:
|
|
self.logger.info(f'opened UTXO DB (for sync: {for_sync})')
|
|
self.read_utxo_state()
|
|
|
|
# Then history DB
|
|
self.utxo_flush_count = self.history.open_db(self.db_class, for_sync,
|
|
self.utxo_flush_count)
|
|
self.clear_excess_undo_info()
|
|
|
|
# Read TX counts (requires meta directory)
|
|
await self._read_tx_counts()
|
|
|
|
async def open_for_sync(self):
|
|
'''Open the databases to sync to the daemon.
|
|
|
|
When syncing we want to reserve a lot of open files for the
|
|
synchronization. When serving clients we want the open files for
|
|
serving network connections.
|
|
'''
|
|
await self._open_dbs(True)
|
|
|
|
async def open_for_serving(self):
|
|
'''Open the databases for serving. If they are already open they are
|
|
closed first.
|
|
'''
|
|
if self.utxo_db:
|
|
self.logger.info('closing DBs to re-open for serving')
|
|
self.utxo_db.close()
|
|
self.history.close_db()
|
|
self.utxo_db = None
|
|
await self._open_dbs(False)
|
|
|
|
def fs_update_header_offsets(self, offset_start, height_start, headers):
|
|
if self.coin.STATIC_BLOCK_HEADERS:
|
|
return
|
|
offset = offset_start
|
|
offsets = []
|
|
for h in headers:
|
|
offset += len(h)
|
|
offsets.append(pack("<Q", offset))
|
|
# For each header we get the offset of the next header, hence we
|
|
# start writing from the next height
|
|
pos = (height_start + 1) * 8
|
|
self.headers_offsets_file.write(pos, b''.join(offsets))
|
|
|
|
def dynamic_header_offset(self, height):
|
|
assert not self.coin.STATIC_BLOCK_HEADERS
|
|
offset, = unpack('<Q', self.headers_offsets_file.read(height * 8, 8))
|
|
return offset
|
|
|
|
def dynamic_header_len(self, height):
|
|
return self.dynamic_header_offset(height + 1)\
|
|
- self.dynamic_header_offset(height)
|
|
|
|
def fs_update(self, fs_height, headers, block_tx_hashes):
|
|
'''Write headers, the tx_count array and block tx hashes to disk.
|
|
|
|
Their first height is fs_height. No recorded DB state is
|
|
updated. These arrays are all append only, so in a crash we
|
|
just pick up again from the DB height.
|
|
'''
|
|
blocks_done = len(headers)
|
|
height_start = fs_height + 1
|
|
new_height = fs_height + blocks_done
|
|
prior_tx_count = (self.tx_counts[fs_height] if fs_height >= 0 else 0)
|
|
cur_tx_count = self.tx_counts[-1] if self.tx_counts else 0
|
|
txs_done = cur_tx_count - prior_tx_count
|
|
|
|
assert len(block_tx_hashes) == blocks_done
|
|
assert len(self.tx_counts) == new_height + 1
|
|
hashes = b''.join(block_tx_hashes)
|
|
assert len(hashes) % 32 == 0
|
|
assert len(hashes) // 32 == txs_done
|
|
|
|
# Write the headers, tx counts, and tx hashes
|
|
offset = self.header_offset(height_start)
|
|
self.headers_file.write(offset, b''.join(headers))
|
|
self.fs_update_header_offsets(offset, height_start, headers)
|
|
offset = height_start * self.tx_counts.itemsize
|
|
self.tx_counts_file.write(offset,
|
|
self.tx_counts[height_start:].tobytes())
|
|
offset = prior_tx_count * 32
|
|
self.hashes_file.write(offset, hashes)
|
|
|
|
def read_headers(self, start_height, count):
|
|
'''Requires start_height >= 0, count >= 0. Reads as many headers as
|
|
are available starting at start_height up to count. This
|
|
would be zero if start_height is beyond self.db_height, for
|
|
example.
|
|
|
|
Returns a (binary, n) pair where binary is the concatenated
|
|
binary headers, and n is the count of headers returned.
|
|
'''
|
|
# Read some from disk
|
|
if start_height < 0 or count < 0:
|
|
raise self.DBError('{:,d} headers starting at {:,d} not on disk'
|
|
.format(count, start_height))
|
|
disk_count = max(0, min(count, self.db_height + 1 - start_height))
|
|
if disk_count:
|
|
offset = self.header_offset(start_height)
|
|
size = self.header_offset(start_height + disk_count) - offset
|
|
return self.headers_file.read(offset, size), disk_count
|
|
return b'', 0
|
|
|
|
def fs_tx_hash(self, tx_num):
|
|
'''Return a par (tx_hash, tx_height) for the given tx number.
|
|
|
|
If the tx_height is not on disk, returns (None, tx_height).'''
|
|
tx_height = bisect_right(self.tx_counts, tx_num)
|
|
if tx_height > self.db_height:
|
|
tx_hash = None
|
|
else:
|
|
tx_hash = self.hashes_file.read(tx_num * 32, 32)
|
|
return tx_hash, tx_height
|
|
|
|
def fs_block_hashes(self, height, count):
|
|
headers_concat, headers_count = self.read_headers(height, count)
|
|
if headers_count != count:
|
|
raise self.DBError('only got {:,d} headers starting at {:,d}, not '
|
|
'{:,d}'.format(headers_count, height, count))
|
|
offset = 0
|
|
headers = []
|
|
for n in range(count):
|
|
hlen = self.header_len(height + n)
|
|
headers.append(headers_concat[offset:offset + hlen])
|
|
offset += hlen
|
|
|
|
return [self.coin.header_hash(header) for header in headers]
|
|
|
|
def get_history(self, hashX, limit=1000):
|
|
'''Generator that returns an unpruned, sorted list of (tx_hash,
|
|
height) tuples of confirmed transactions that touched the address,
|
|
earliest in the blockchain first. Includes both spending and
|
|
receiving transactions. By default yields at most 1000 entries.
|
|
Set limit to None to get them all.
|
|
'''
|
|
for tx_num in self.history.get_txnums(hashX, limit):
|
|
yield self.fs_tx_hash(tx_num)
|
|
|
|
# -- Undo information
|
|
|
|
def min_undo_height(self, max_height):
|
|
'''Returns a height from which we should store undo info.'''
|
|
return max_height - self.env.reorg_limit + 1
|
|
|
|
def undo_key(self, height):
|
|
'''DB key for undo information at the given height.'''
|
|
return b'U' + pack('>I', height)
|
|
|
|
def read_undo_info(self, height):
|
|
'''Read undo information from a file for the current height.'''
|
|
return self.utxo_db.get(self.undo_key(height))
|
|
|
|
def flush_undo_infos(self, batch_put, undo_infos):
|
|
'''undo_infos is a list of (undo_info, height) pairs.'''
|
|
for undo_info, height in undo_infos:
|
|
batch_put(self.undo_key(height), b''.join(undo_info))
|
|
|
|
def raw_block_prefix(self):
|
|
return 'meta/block'
|
|
|
|
def raw_block_path(self, height):
|
|
return f'{self.raw_block_prefix()}{height:d}'
|
|
|
|
def read_raw_block(self, height):
|
|
'''Returns a raw block read from disk. Raises FileNotFoundError
|
|
if the block isn't on-disk.'''
|
|
with util.open_file(self.raw_block_path(height)) as f:
|
|
return f.read(-1)
|
|
|
|
def write_raw_block(self, block, height):
|
|
'''Write a raw block to disk.'''
|
|
with util.open_truncate(self.raw_block_path(height)) as f:
|
|
f.write(block)
|
|
# Delete old blocks to prevent them accumulating
|
|
try:
|
|
del_height = self.min_undo_height(height) - 1
|
|
os.remove(self.raw_block_path(del_height))
|
|
except FileNotFoundError:
|
|
pass
|
|
|
|
def clear_excess_undo_info(self):
|
|
'''Clear excess undo info. Only most recent N are kept.'''
|
|
prefix = b'U'
|
|
min_height = self.min_undo_height(self.db_height)
|
|
keys = []
|
|
for key, hist in self.utxo_db.iterator(prefix=prefix):
|
|
height, = unpack('>I', key[-4:])
|
|
if height >= min_height:
|
|
break
|
|
keys.append(key)
|
|
|
|
if keys:
|
|
with self.utxo_db.write_batch() as batch:
|
|
for key in keys:
|
|
batch.delete(key)
|
|
self.logger.info(f'deleted {len(keys):,d} stale undo entries')
|
|
|
|
# delete old block files
|
|
prefix = self.raw_block_prefix()
|
|
paths = [path for path in glob(f'{prefix}[0-9]*')
|
|
if len(path) > len(prefix)
|
|
and int(path[len(prefix):]) < min_height]
|
|
if paths:
|
|
for path in paths:
|
|
try:
|
|
os.remove(path)
|
|
except FileNotFoundError:
|
|
pass
|
|
self.logger.info(f'deleted {len(paths):,d} stale block files')
|
|
|
|
# -- UTXO database
|
|
|
|
def read_utxo_state(self):
|
|
state = self.utxo_db.get(b'state')
|
|
if not state:
|
|
self.db_height = -1
|
|
self.db_tx_count = 0
|
|
self.db_tip = b'\0' * 32
|
|
self.db_version = max(self.DB_VERSIONS)
|
|
self.utxo_flush_count = 0
|
|
self.wall_time = 0
|
|
self.first_sync = True
|
|
else:
|
|
state = ast.literal_eval(state.decode())
|
|
if not isinstance(state, dict):
|
|
raise self.DBError('failed reading state from DB')
|
|
self.db_version = state['db_version']
|
|
if self.db_version not in self.DB_VERSIONS:
|
|
raise self.DBError('your UTXO DB version is {} but this '
|
|
'software only handles versions {}'
|
|
.format(self.db_version, self.DB_VERSIONS))
|
|
# backwards compat
|
|
genesis_hash = state['genesis']
|
|
if isinstance(genesis_hash, bytes):
|
|
genesis_hash = genesis_hash.decode()
|
|
if genesis_hash != self.coin.GENESIS_HASH:
|
|
raise self.DBError('DB genesis hash {} does not match coin {}'
|
|
.format(genesis_hash,
|
|
self.coin.GENESIS_HASH))
|
|
self.db_height = state['height']
|
|
self.db_tx_count = state['tx_count']
|
|
self.db_tip = state['tip']
|
|
self.utxo_flush_count = state['utxo_flush_count']
|
|
self.wall_time = state['wall_time']
|
|
self.first_sync = state['first_sync']
|
|
|
|
# Log some stats
|
|
self.logger.info('DB version: {:d}'.format(self.db_version))
|
|
self.logger.info('coin: {}'.format(self.coin.NAME))
|
|
self.logger.info('network: {}'.format(self.coin.NET))
|
|
self.logger.info('height: {:,d}'.format(self.db_height))
|
|
self.logger.info('tip: {}'.format(hash_to_hex_str(self.db_tip)))
|
|
self.logger.info('tx count: {:,d}'.format(self.db_tx_count))
|
|
if self.first_sync:
|
|
self.logger.info('sync time so far: {}'
|
|
.format(util.formatted_time(self.wall_time)))
|
|
|
|
def write_utxo_state(self, batch):
|
|
'''Write (UTXO) state to the batch.'''
|
|
state = {
|
|
'genesis': self.coin.GENESIS_HASH,
|
|
'height': self.db_height,
|
|
'tx_count': self.db_tx_count,
|
|
'tip': self.db_tip,
|
|
'utxo_flush_count': self.utxo_flush_count,
|
|
'wall_time': self.wall_time,
|
|
'first_sync': self.first_sync,
|
|
'db_version': self.db_version,
|
|
}
|
|
batch.put(b'state', repr(state).encode())
|
|
|
|
def set_flush_count(self, count):
|
|
self.utxo_flush_count = count
|
|
with self.utxo_db.write_batch() as batch:
|
|
self.write_utxo_state(batch)
|
|
|
|
def get_balance(self, hashX):
|
|
'''Returns the confirmed balance of an address.'''
|
|
return sum(utxo.value for utxo in self.get_utxos(hashX, limit=None))
|
|
|
|
def get_utxos(self, hashX, limit=1000):
|
|
'''Generator that yields all UTXOs for an address sorted in no
|
|
particular order. By default yields at most 1000 entries.
|
|
Set limit to None to get them all.
|
|
'''
|
|
limit = util.resolve_limit(limit)
|
|
s_unpack = unpack
|
|
# Key: b'u' + address_hashX + tx_idx + tx_num
|
|
# Value: the UTXO value as a 64-bit unsigned integer
|
|
prefix = b'u' + hashX
|
|
for db_key, db_value in self.utxo_db.iterator(prefix=prefix):
|
|
if limit == 0:
|
|
return
|
|
limit -= 1
|
|
tx_pos, tx_num = s_unpack('<HI', db_key[-6:])
|
|
value, = unpack('<Q', db_value)
|
|
tx_hash, height = self.fs_tx_hash(tx_num)
|
|
yield UTXO(tx_num, tx_pos, tx_hash, height, value)
|
|
|
|
async def lookup_utxos(self, prevouts):
|
|
'''For each prevout, lookup it up in the DB and return a (hashX,
|
|
value) pair or None if not found.
|
|
|
|
Used by the mempool code.
|
|
'''
|
|
def lookup_hashXs():
|
|
'''Return (hashX, suffix) pairs, or None if not found,
|
|
for each prevout.
|
|
'''
|
|
def lookup_hashX(tx_hash, tx_idx):
|
|
idx_packed = pack('<H', tx_idx)
|
|
|
|
# Key: b'h' + compressed_tx_hash + tx_idx + tx_num
|
|
# Value: hashX
|
|
prefix = b'h' + tx_hash[:4] + idx_packed
|
|
|
|
# Find which entry, if any, the TX_HASH matches.
|
|
for db_key, hashX in self.utxo_db.iterator(prefix=prefix):
|
|
tx_num_packed = db_key[-4:]
|
|
tx_num, = unpack('<I', tx_num_packed)
|
|
hash, height = self.fs_tx_hash(tx_num)
|
|
if hash == tx_hash:
|
|
return hashX, idx_packed + tx_num_packed
|
|
return None, None
|
|
return [lookup_hashX(*prevout) for prevout in prevouts]
|
|
|
|
def lookup_utxos(hashX_pairs):
|
|
def lookup_utxo(hashX, suffix):
|
|
if not hashX:
|
|
# This can happen when the daemon is a block ahead
|
|
# of us and has mempool txs spending outputs from
|
|
# that new block
|
|
return None
|
|
# Key: b'u' + address_hashX + tx_idx + tx_num
|
|
# Value: the UTXO value as a 64-bit unsigned integer
|
|
key = b'u' + hashX + suffix
|
|
db_value = self.utxo_db.get(key)
|
|
if not db_value:
|
|
# This can happen if the DB was updated between
|
|
# getting the hashXs and getting the UTXOs
|
|
return None
|
|
value, = unpack('<Q', db_value)
|
|
return hashX, value
|
|
return [lookup_utxo(*hashX_pair) for hashX_pair in hashX_pairs]
|
|
|
|
run_in_thread = self.tasks.run_in_thread
|
|
hashX_pairs = await run_in_thread(lookup_hashXs)
|
|
return await run_in_thread(lookup_utxos, hashX_pairs)
|