diff --git a/pybtc/connector.py b/pybtc/connector.py index 5954a39..500f519 100644 --- a/pybtc/connector.py +++ b/pybtc/connector.py @@ -3,10 +3,10 @@ from pybtc.functions.tools import var_int_to_int, var_int_len from pybtc.functions.tools import read_var_int from pybtc.functions.hash import double_sha256 from pybtc.transaction import Transaction -from pybtc.block import Block from pybtc import int_to_c_int, c_int_to_int, c_int_len, int_to_bytes from pybtc.functions.block import bits_to_target, target_to_difficulty from struct import unpack, pack +import sys import traceback import aiojsonrpc import zmq @@ -78,11 +78,11 @@ class Connector: # cache and system self.preload = preload - self.block_preload = Cache(max_size=50000) - self.block_hashes = Cache(max_size=100000) + self.block_preload = Cache(max_size=200 * 1000000) + self.block_hashes = Cache(max_size=20 * 100000) self.block_hashes_preload_mutex = False - self.tx_cache = Cache(max_size=50000) - self.block_cache = Cache(max_size=10000) + self.tx_cache = Cache(max_size=100 * 1000000) + self.block_cache = Cache(max_size=20 * 1000000) self.block_txs_request = None @@ -296,11 +296,13 @@ class Connector: self.log.warning("Normal synchronization mode") # clear preload caches self.deep_synchronization = False - q = time.time() + if self.deep_synchronization: raw_block = self.block_preload.pop(self.last_block_height + 1) if raw_block: + q = time.time() block = decode_block_tx(raw_block) + self.blocks_decode_time += time.time() - q else: h = self.block_hashes.pop(self.last_block_height + 1) if h is None: @@ -311,7 +313,7 @@ class Connector: else: h = await self.rpc.getblockhash(self.last_block_height + 1) block = await self._get_block_by_hash(h) - self.blocks_download_time += time.time() - q + self.loop.create_task(self._new_block(block)) except Exception as err: @@ -620,8 +622,6 @@ class Connector: return try: self.block_hashes_preload_mutex = True - - self.block_hashes = Cache(max_size=50000) max_height = self.node_last_block - self.deep_synchronization height = self.last_block_height + 1 processed_height = self.last_block_height @@ -666,8 +666,8 @@ class Connector: for i in range(processed_height, self.last_block_height): self.block_preload.remove(i) processed_height = self.last_block_height - if self.block_preload.len() < 40000 and \ - height < self.last_block_height + 40000: + if self.block_preload._store_size < 190 * 1000000 and \ + height < self.last_block_height + self.batch_limit: continue # self.log.critical(str((processed_height, self.last_block_height))) @@ -932,8 +932,9 @@ class DependsTransaction(Exception): class Cache(): - def __init__(self, max_size=1000): + def __init__(self, max_size=1000000): self._store = OrderedDict() + self._store_size = 0 self._max_size = max_size self.clear_tail = False self._requests = 0 @@ -942,12 +943,13 @@ class Cache(): def set(self, key, value): self._check_limit() self._store[key] = value + self._store_size += sys.getsizeof(value) + sys.getsizeof(key) def _check_limit(self): - if len(self._store) >= self._max_size: + if self._store_size >= self._max_size: self.clear_tail = True if self.clear_tail: - if len(self._store) >= int(self._max_size * 0.75): + if self._store_size >= int(self._max_size * 0.75): [self._store.popitem(last=False) for i in range(20)] else: self.clear_tail = False @@ -964,8 +966,8 @@ class Cache(): def pop(self, key): self._requests += 1 try: - data = self._store[key] - del self._store[key] + data = self._store.pop(key) + self._store_size -= sys.getsizeof(data) + sys.getsizeof(key) self._hit += 1 return data except: @@ -973,7 +975,8 @@ class Cache(): def remove(self, key): try: - del self._store[key] + data = self._store.pop(key) + self._store_size -= sys.getsizeof(data) + sys.getsizeof(key) except: pass @@ -982,6 +985,7 @@ class Cache(): i = next(reversed(self._store)) data = self._store[i] del self._store[i] + self._store_size -= sys.getsizeof(data) + sys.getsizeof(i) return data except: return None