connector

pybtc/connector/block_loader.py

@@ -10,7 +10,7 @@ import sys
 import aiojsonrpc
 import traceback
 from pybtc.connector.utils import decode_block_tx
-import pickle
+import msgpack
 from lru import LRU
 
 
@@ -162,7 +162,7 @@ class BlockLoader:
 
             if msg_type == b'result':
                 self.worker_busy[index] = False
-                blocks = pickle.loads(msg)
+                blocks = msgpack.loads(msg)
 
                 for i in blocks:
                     self.parent.block_preload.set(i, blocks[i])
@@ -256,9 +256,9 @@ class Worker:
                             blocks[x]["rawTx"][y]["vOut"][i]["_s_"] = self.destroyed_coins[pointer]
                         except: pass
 
-                blocks[x] = pickle.dumps(blocks[x])
+                blocks[x] = msgpack.dumps(blocks[x])
 
-            self.pipe_sent_msg(b'result', pickle.dumps(blocks))
+            self.pipe_sent_msg(b'result', msgpack.dumps(blocks))
         except:
             self.log.critical(str(traceback.format_exc()))
 

pybtc/connector/connector.py

@@ -12,7 +12,7 @@ import zmq
 import zmq.asyncio
 import asyncio
 import time
-import pickle
+import msgpack
 
 class Connector:
     def __init__(self, node_rpc_url, node_zerromq_url, logger,
@@ -304,7 +304,7 @@ class Connector:
                     raw_block = self.block_preload.pop(self.last_block_height + 1)
                     if raw_block:
                         q = time.time()
-                        block = pickle.loads(raw_block)
+                        block = msgpack.loads(raw_block)
                         self.blocks_decode_time += time.time() - q
 
                 if not block:

pybtc/connector/utxo.py

@@ -2,7 +2,7 @@ from pybtc import int_to_c_int, c_int_to_int, c_int_len
 import asyncio
 from collections import OrderedDict, deque
 from lru import LRU
-import pickle
+
 
 class UTXO():
     def __init__(self, db_pool, loop, log, cache_size):
@@ -34,7 +34,7 @@ class UTXO():
         self.outs_total = 0
 
     def set(self, outpoint, pointer, amount, address):
-        self.cached[outpoint] = pickle.dumps((pointer, amount, address))
+        self.cached[outpoint] = (pointer, amount, address)
         self.outs_total += 1
         if pointer:
             self.last_cached_block = pointer >> 42
@@ -144,10 +144,10 @@ class UTXO():
         finally:
             self.save_process = False
 
-    def get(self, key):
+    def get(self, key, block_height):
         self._requests += 1
         try:
-            i = pickle.loads(self.cached[key])
+            i = self.cached[key]
             self.destroyed.append(key)
             # try:
             #     self.destroyed[block_height].add(key)

pybtc/test/block.py

@@ -604,6 +604,13 @@ class BlockDeserializeTests(unittest.TestCase):
             Block(fc[:-1], format="raw", keep_raw_tx=False),
               )
         print("decoded block", time.time() - qt )
+        import msgpack
+        qt = time.time()
+        k = msgpack.dumps(bt)
+        print("decoded block dump", time.time() - qt)
+        qt = time.time()
+        p = msgpack.loads(k)
+        print("decoded block load", time.time() - qt)
         import pickle
         qt = time.time()
         k = pickle.dumps(bt)

setup.py

@@ -140,7 +140,7 @@ setup(name='pybtc',
       package_data={
           'pybtc': ['bip39_word_list/*.txt', 'test/*.txt'],
       },
-      install_requires=['lru-dict'],
+      install_requires=['lru-dict', 'msgpack'],
       cmdclass={
         'build_clib': build_clib,
         'build_ext': build_ext,