RanchiMall/pyflo
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

connector

Browse Source
This commit is contained in:
4tochka 2019-05-02 17:54:45 +04:00
parent a81bce41d0
commit bcd51fbf3c
1 changed files with 21 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
pybtc/connector.py
View File

@ -3,10 +3,10 @@ from pybtc.functions.tools import var_int_to_int, var_int_len
from pybtc.functions.tools import read_var_int from pybtc.functions.tools import read_var_int
from pybtc.functions.hash import double_sha256 from pybtc.functions.hash import double_sha256
from pybtc.transaction import Transaction from pybtc.transaction import Transaction
from pybtc.block import Block
from pybtc import int_to_c_int, c_int_to_int, c_int_len, int_to_bytes from pybtc import int_to_c_int, c_int_to_int, c_int_len, int_to_bytes
from pybtc.functions.block import bits_to_target, target_to_difficulty from pybtc.functions.block import bits_to_target, target_to_difficulty
from struct import unpack, pack from struct import unpack, pack
import sys
import traceback import traceback
import aiojsonrpc import aiojsonrpc
import zmq import zmq
@ -78,11 +78,11 @@ class Connector:
# cache and system # cache and system
self.preload = preload self.preload = preload
self.block_preload = Cache(max_size=50000) self.block_preload = Cache(max_size=200 * 1000000)
self.block_hashes = Cache(max_size=100000) self.block_hashes = Cache(max_size=20 * 100000)
self.block_hashes_preload_mutex = False self.block_hashes_preload_mutex = False
self.tx_cache = Cache(max_size=50000) self.tx_cache = Cache(max_size=100 * 1000000)
self.block_cache = Cache(max_size=10000) self.block_cache = Cache(max_size=20 * 1000000)
self.block_txs_request = None self.block_txs_request = None
@ -296,11 +296,13 @@ class Connector:
self.log.warning("Normal synchronization mode") self.log.warning("Normal synchronization mode")
# clear preload caches # clear preload caches
self.deep_synchronization = False self.deep_synchronization = False
q = time.time()
if self.deep_synchronization: if self.deep_synchronization:
raw_block = self.block_preload.pop(self.last_block_height + 1) raw_block = self.block_preload.pop(self.last_block_height + 1)
if raw_block: if raw_block:
q = time.time()
block = decode_block_tx(raw_block) block = decode_block_tx(raw_block)
self.blocks_decode_time += time.time() - q
else: else:
h = self.block_hashes.pop(self.last_block_height + 1) h = self.block_hashes.pop(self.last_block_height + 1)
if h is None: if h is None:
@ -311,7 +313,7 @@ class Connector:
else: else:
h = await self.rpc.getblockhash(self.last_block_height + 1) h = await self.rpc.getblockhash(self.last_block_height + 1)
block = await self._get_block_by_hash(h) block = await self._get_block_by_hash(h)
self.blocks_download_time += time.time() - q
self.loop.create_task(self._new_block(block)) self.loop.create_task(self._new_block(block))
except Exception as err: except Exception as err:
@ -620,8 +622,6 @@ class Connector:
return return
try: try:
self.block_hashes_preload_mutex = True self.block_hashes_preload_mutex = True
self.block_hashes = Cache(max_size=50000)
max_height = self.node_last_block - self.deep_synchronization max_height = self.node_last_block - self.deep_synchronization
height = self.last_block_height + 1 height = self.last_block_height + 1
processed_height = self.last_block_height processed_height = self.last_block_height
@ -666,8 +666,8 @@ class Connector:
for i in range(processed_height, self.last_block_height): for i in range(processed_height, self.last_block_height):
self.block_preload.remove(i) self.block_preload.remove(i)
processed_height = self.last_block_height processed_height = self.last_block_height
if self.block_preload.len() < 40000 and \ if self.block_preload._store_size < 190 * 1000000 and \
height < self.last_block_height + 40000: height < self.last_block_height + self.batch_limit:
continue continue
# self.log.critical(str((processed_height, self.last_block_height))) # self.log.critical(str((processed_height, self.last_block_height)))
@ -932,8 +932,9 @@ class DependsTransaction(Exception):
class Cache(): class Cache():
def __init__(self, max_size=1000): def __init__(self, max_size=1000000):
self._store = OrderedDict() self._store = OrderedDict()
self._store_size = 0
self._max_size = max_size self._max_size = max_size
self.clear_tail = False self.clear_tail = False
self._requests = 0 self._requests = 0
@ -942,12 +943,13 @@ class Cache():
def set(self, key, value): def set(self, key, value):
self._check_limit() self._check_limit()
self._store[key] = value self._store[key] = value
self._store_size += sys.getsizeof(value) + sys.getsizeof(key)
def _check_limit(self): def _check_limit(self):
if len(self._store) >= self._max_size: if self._store_size >= self._max_size:
self.clear_tail = True self.clear_tail = True
if self.clear_tail: if self.clear_tail:
if len(self._store) >= int(self._max_size * 0.75): if self._store_size >= int(self._max_size * 0.75):
[self._store.popitem(last=False) for i in range(20)] [self._store.popitem(last=False) for i in range(20)]
else: else:
self.clear_tail = False self.clear_tail = False
@ -964,8 +966,8 @@ class Cache():
def pop(self, key): def pop(self, key):
self._requests += 1 self._requests += 1
try: try:
data = self._store[key] data = self._store.pop(key)
del self._store[key] self._store_size -= sys.getsizeof(data) + sys.getsizeof(key)
self._hit += 1 self._hit += 1
return data return data
except: except:
@ -973,7 +975,8 @@ class Cache():
def remove(self, key): def remove(self, key):
try: try:
del self._store[key] data = self._store.pop(key)
self._store_size -= sys.getsizeof(data) + sys.getsizeof(key)
except: except:
pass pass
@ -982,6 +985,7 @@ class Cache():
i = next(reversed(self._store)) i = next(reversed(self._store))
data = self._store[i] data = self._store[i]
del self._store[i] del self._store[i]
self._store_size -= sys.getsizeof(data) + sys.getsizeof(i)
return data return data
except: except:
return None return None