From 17034ac7a7e40e582fa8e1aa9c3fb0f422eb175e Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sat, 8 Oct 2016 23:42:01 +0900 Subject: [PATCH 1/3] Make get_utxos() and get_history() generators The also take a limit. --- server/db.py | 59 ++++++++++++++++++++++++++++++++++------------------ 1 file changed, 39 insertions(+), 20 deletions(-) diff --git a/server/db.py b/server/db.py index 223a887..7f0efde 100644 --- a/server/db.py +++ b/server/db.py @@ -433,27 +433,41 @@ class DB(object): return tx_hash, height + @staticmethod + def resolve_limit(limit): + if limit is None: + return -1 + assert isinstance(limit, int) and limit >= 0 + return limit + + def get_history(self, hash160, limit=1000): + '''Generator that returns an unpruned, sorted list of (tx_hash, + height) tuples of transactions that touched the address, + earliest in the blockchain first. Includes both spending and + receiving transactions. By default yields at most 1000 entries. + Set limit to None to get them all. + ''' + limit = self.resolve_limit(limit) + prefix = b'H' + hash160 + for key, hist in self.db.iterator(prefix=prefix): + a = array.array('I') + a.frombytes(hist) + for tx_num in a: + if limit == 0: + return + yield self.get_tx_hash(tx_num) + limit -= 1 + def get_balance(self, hash160): '''Returns the confirmed balance of an address.''' - utxos = self.get_utxos(hash_160) - return sum(utxo.value for utxo in utxos) + return sum(utxo.value for utxo in self.get_utxos(hash_160, limit=None)) - def get_history(self, hash160): - '''Returns an unpruned, sorted list of (tx_hash, height) tuples of - transactions that touched the address, earliest in the - blockchain first. Includes both spending and receiving - transactions. - ''' - prefix = b'H' + hash160 - a = array.array('I') - for key, hist in self.db.iterator(prefix=prefix): - a.frombytes(hist) - return [self.get_tx_hash(tx_num) for tx_num in a] - - def get_utxos(self, hash160): - '''Returns all UTXOs for an address sorted such that the earliest - in the blockchain comes first. + def get_utxos(self, hash160, limit=1000): + '''Generator that yields all UTXOs for an address sorted in no + particular order. By default yields at most 1000 entries. + Set limit to None to get them all. ''' + limit = self.resolve_limit(limit) unpack = struct.unpack prefix = b'u' + hash160 utxos = [] @@ -461,10 +475,15 @@ class DB(object): (tx_pos, ) = unpack(' Date: Sun, 9 Oct 2016 08:26:46 +0900 Subject: [PATCH 2/3] Add a simple query for debugging --- query.py | 33 +++++++++++++++++++++++++++++++++ server/server.py | 29 ----------------------------- 2 files changed, 33 insertions(+), 29 deletions(-) create mode 100644 query.py diff --git a/query.py b/query.py new file mode 100644 index 0000000..f445d06 --- /dev/null +++ b/query.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 + +# See the file "LICENSE" for information about the copyright +# and warranty status of this software. + +import asyncio +import os +import sys + +from server.env import Env +from server.server import Server + + +def main(): + env = Env() + os.chdir(env.db_dir) + loop = asyncio.get_event_loop() + server = Server(env, loop) + db = server.db + coin = db.coin + for addr in sys.argv[1:]: + print('Address: ', addr) + hash160 = coin.address_to_hash160(addr) + for n, (tx_hash, height) in enumerate(db.get_history(hash160)): + print('History #{:d}: hash: {} height: {:d}' + .format(n + 1, bytes(reversed(tx_hash)).hex(), height)) + for n, utxo in enumerate(db.get_utxos(hash160)): + print('UTXOs #{:d}: hash: {} pos: {:d} height: {:d} value: {:d}' + .format(n, bytes(reversed(utxo.tx_hash)).hex(), + utxo.tx_pos, utxo.height, utxo.value)) + +if __name__ == '__main__': + main() diff --git a/server/server.py b/server/server.py index 2618d2b..e6a7bbc 100644 --- a/server/server.py +++ b/server/server.py @@ -201,32 +201,3 @@ class RPC(object): self.logger.info('sleeping 1 second and trying again...') await asyncio.sleep(1) - - # for addr in [ - # # '1dice8EMZmqKvrGE4Qc9bUFf9PX3xaYDp', - # # '1HYBcza9tVquCCvCN1hUZkYT9RcM6GfLot', - # # '1BNwxHGaFbeUBitpjy2AsKpJ29Ybxntqvb', - # # '1ARanTkswPiVM6tUEYvbskyqDsZpweiciu', - # # '1VayNert3x1KzbpzMGt2qdqrAThiRovi8', - # # '1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa', - # # '1XPTgDRhN8RFnzniWCddobD9iKZatrvH4', - # # '153h6eE6xRhXuN3pE53gWVfXacAtfyBF8g', - # ]: - # print('Address: ', addr) - # hash160 = coin.address_to_hash160(addr) - # utxos = self.db.get_utxos(hash160) - # for n, utxo in enumerate(utxos): - # print('UTXOs #{:d}: hash: {} pos: {:d} height: {:d} value: {:d}' - # .format(n, bytes(reversed(utxo.tx_hash)).hex(), - # utxo.tx_pos, utxo.height, utxo.value)) - - # for addr in [ - # '19k8nToWwMGuF4HkNpzgoVAYk4viBnEs5D', - # '1HaHTfmvoUW6i6nhJf8jJs6tU4cHNmBQHQ', - # '1XPTgDRhN8RFnzniWCddobD9iKZatrvH4', - # ]: - # print('Address: ', addr) - # hash160 = coin.address_to_hash160(addr) - # for n, (tx_hash, height) in enumerate(self.db.get_history(hash160)): - # print('History #{:d}: hash: {} height: {:d}' - # .format(n + 1, bytes(reversed(tx_hash)).hex(), height)) From 6ccdce2c7794bb99990530f526906f59c7326ffd Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sun, 9 Oct 2016 08:41:42 +0900 Subject: [PATCH 3/3] Print if nothing. Clean up loop. --- query.py | 25 +++++++++++++++++-------- server/server.py | 8 +++++--- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/query.py b/query.py index f445d06..1cae0ae 100644 --- a/query.py +++ b/query.py @@ -3,31 +3,40 @@ # See the file "LICENSE" for information about the copyright # and warranty status of this software. -import asyncio import os import sys from server.env import Env -from server.server import Server +from server.db import DB def main(): env = Env() os.chdir(env.db_dir) - loop = asyncio.get_event_loop() - server = Server(env, loop) - db = server.db + db = DB(env) coin = db.coin - for addr in sys.argv[1:]: + argc = 1 + try: + limit = int(sys.argv[argc]) + argc += 1 + except: + limit = 10 + for addr in sys.argv[argc:]: print('Address: ', addr) hash160 = coin.address_to_hash160(addr) - for n, (tx_hash, height) in enumerate(db.get_history(hash160)): + n = None + for n, (tx_hash, height) in enumerate(db.get_history(hash160, limit)): print('History #{:d}: hash: {} height: {:d}' .format(n + 1, bytes(reversed(tx_hash)).hex(), height)) - for n, utxo in enumerate(db.get_utxos(hash160)): + if n is None: + print('No history') + n = None + for n, utxo in enumerate(db.get_utxos(hash160, limit)): print('UTXOs #{:d}: hash: {} pos: {:d} height: {:d} value: {:d}' .format(n, bytes(reversed(utxo.tx_hash)).hex(), utxo.tx_pos, utxo.height, utxo.value)) + if n is None: + print('No UTXOs') if __name__ == '__main__': main() diff --git a/server/server.py b/server/server.py index e6a7bbc..ca6b786 100644 --- a/server/server.py +++ b/server/server.py @@ -15,11 +15,11 @@ from server.db import DB class Server(object): - def __init__(self, env, loop): + def __init__(self, env): self.env = env self.db = DB(env) self.rpc = RPC(env) - self.block_cache = BlockCache(env, self.db, self.rpc, loop) + self.block_cache = BlockCache(env, self.db, self.rpc) def async_tasks(self): return [ @@ -32,7 +32,7 @@ class BlockCache(object): '''Requests blocks ahead of time from the daemon. Serves them to the blockchain processor.''' - def __init__(self, env, db, rpc, loop): + def __init__(self, env, db, rpc): self.logger = logging.getLogger('BlockCache') self.logger.setLevel(logging.INFO) @@ -47,6 +47,8 @@ class BlockCache(object): self.blocks = [] self.recent_sizes = [] self.ave_size = 0 + + loop = asyncio.get_event_loop() for signame in ('SIGINT', 'SIGTERM'): loop.add_signal_handler(getattr(signal, signame), partial(self.on_signal, signame))