RanchiMall/pyflo
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

connector

Browse Source
This commit is contained in:
4tochka 2019-05-12 16:40:26 +04:00
parent 24a8fda01f
commit 7a222adbfd
4 changed files with 125 additions and 232 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

341
pybtc/_lru/lru.c
View File

@ -62,8 +62,7 @@ typedef struct _Node {
struct _Node * next; struct _Node * next;
} Node; } Node;
static void static void node_dealloc(Node* self)
node_dealloc(Node* self)
{ {
Py_DECREF(self->key); Py_DECREF(self->key);
Py_DECREF(self->value); Py_DECREF(self->value);
@ -72,11 +71,7 @@ node_dealloc(Node* self)
PyObject_Del((PyObject*)self); PyObject_Del((PyObject*)self);
} }
static PyObject* static PyObject*node_repr(Node* self) { return PyObject_Repr(self->value);}
node_repr(Node* self)
{
return PyObject_Repr(self->value);
}
static PyTypeObject NodeType = { static PyTypeObject NodeType = {
PyVarObject_HEAD_INIT(NULL, 0) PyVarObject_HEAD_INIT(NULL, 0)
@ -125,37 +120,10 @@ typedef struct {
Node * first; Node * first;
Node * last; Node * last;
Py_ssize_t size; Py_ssize_t size;
Py_ssize_t hits;
Py_ssize_t misses;
PyObject *callback;
} LRU; } LRU;
static PyObject * static void lru_remove_node(LRU *self, Node* node)
set_callback(LRU *self, PyObject *args)
{
PyObject *result = NULL;
PyObject *temp;
if (PyArg_ParseTuple(args, "O:set_callback", &temp)) {
if (temp == Py_None) {
Py_XDECREF(self->callback);
self->callback = NULL;
} else if (!PyCallable_Check(temp)) {
PyErr_SetString(PyExc_TypeError, "parameter must be callable");
return NULL;
} else {
Py_XINCREF(temp); /* Add a reference to new callback */
Py_XDECREF(self->callback); /* Dispose of previous callback */
self->callback = temp; /* Remember new callback */
}
Py_RETURN_NONE;
}
return result;
}
static void
lru_remove_node(LRU *self, Node* node)
{ {
if (self->first == node) { if (self->first == node) {
self->first = node->next; self->first = node->next;
@ -172,8 +140,7 @@ lru_remove_node(LRU *self, Node* node)
node->next = node->prev = NULL; node->next = node->prev = NULL;
} }
static void static void lru_add_node_at_head(LRU *self, Node* node)
lru_add_node_at_head(LRU *self, Node* node)
{ {
node->prev = NULL; node->prev = NULL;
if (!self->first) { if (!self->first) {
@ -188,36 +155,32 @@ lru_add_node_at_head(LRU *self, Node* node)
} }
} }
static void static void lru_add_node_at_tail(LRU *self, Node* node)
lru_delete_last(LRU *self)
{ {
PyObject *arglist; node->next = NULL;
PyObject *result; if (!self->first) {
Node* n = self->last; self->first = self->last = node;
node->prev = NULL;
if (!self->last) } else {
return; node->prev = self->last;
if (node->prev) {
if (self->callback) { node->prev->next = node;
}
arglist = Py_BuildValue("OO", n->key, n->value); self->last = node;
result = PyObject_CallObject(self->callback, arglist);
Py_XDECREF(result);
Py_DECREF(arglist);
} }
}
static void lru_delete_last(LRU *self)
{
Node* n = self->last;
if (!self->last) return;
lru_remove_node(self, n); lru_remove_node(self, n);
PUT_NODE(self->dict, n->key, NULL); PUT_NODE(self->dict, n->key, NULL);
} }
static Py_ssize_t static Py_ssize_t lru_length(LRU *self) { return PyDict_Size(self->dict); }
lru_length(LRU *self)
{
return PyDict_Size(self->dict);
}
static PyObject * static PyObject *LRU_contains_key(LRU *self, PyObject *key)
LRU_contains_key(LRU *self, PyObject *key)
{ {
if (PyDict_Contains(self->dict, key)) { if (PyDict_Contains(self->dict, key)) {
Py_RETURN_TRUE; Py_RETURN_TRUE;
@ -226,38 +189,27 @@ LRU_contains_key(LRU *self, PyObject *key)
} }
} }
static PyObject * static PyObject *LRU_contains(LRU *self, PyObject *args)
LRU_contains(LRU *self, PyObject *args)
{ {
PyObject *key; PyObject *key;
if (!PyArg_ParseTuple(args, "O", &key)) if (!PyArg_ParseTuple(args, "O", &key)) return NULL;
return NULL;
return LRU_contains_key(self, key); return LRU_contains_key(self, key);
} }
static int static int LRU_seq_contains(LRU *self, PyObject *key) { return PyDict_Contains(self->dict, key); }
LRU_seq_contains(LRU *self, PyObject *key)
{
return PyDict_Contains(self->dict, key);
}
static PyObject * static PyObject *lru_subscript(LRU *self, register PyObject *key)
lru_subscript(LRU *self, register PyObject *key)
{ {
Node *node = GET_NODE(self->dict, key); Node *node = GET_NODE(self->dict, key);
if (!node) { if (!node) return NULL;
self->misses++;
return NULL;
}
self->hits++;
Py_INCREF(node->value); Py_INCREF(node->value);
Py_DECREF(node); Py_DECREF(node);
return node->value; return node->value;
} }
static PyObject * static PyObject *LRU_pop(LRU *self)
LRU_pop(LRU *self)
{ {
if (self->last) { if (self->last) {
@ -275,31 +227,51 @@ LRU_pop(LRU *self)
} }
static PyObject * static PyObject *LRU_get(LRU *self, PyObject *args)
LRU_get(LRU *self, PyObject *args)
{ {
PyObject *key; PyObject *key;
PyObject *instead = NULL; PyObject *instead = NULL;
PyObject *result; PyObject *result;
if (!PyArg_ParseTuple(args, "O|O", &key, &instead)) if (!PyArg_ParseTuple(args, "O|O", &key, &instead)) return NULL;
return NULL;
result = lru_subscript(self, key); result = lru_subscript(self, key);
PyErr_Clear(); /* GET_NODE sets an exception on miss. Shut it up. */ PyErr_Clear(); /* GET_NODE sets an exception on miss. Shut it up. */
if (result) if (result) return result;
return result;
if (!instead) { if (!instead) { Py_RETURN_NONE; }
Py_RETURN_NONE;
}
Py_INCREF(instead); Py_INCREF(instead);
return instead; return instead;
} }
static int
lru_ass_sub(LRU *self, PyObject *key, PyObject *value) static int lru_put(LRU *self, PyObject *key, PyObject *value)
{
int res = 0;
Node *node = PyObject_NEW(Node, &NodeType);
node->key = key;
node->value = value;
node->next = node->prev = NULL;
Py_INCREF(key);
Py_INCREF(value);
res = PUT_NODE(self->dict, key, node);
if (res == 0) {
if (self->size > 0 && lru_length(self) > self->size) lru_delete_last(self);
lru_add_node_at_head(self, node);
}
Py_XDECREF(node);
return res;
}
static int lru_ass_sub(LRU *self, PyObject *key, PyObject *value)
{ {
int res = 0; int res = 0;
Node *node = GET_NODE(self->dict, key); Node *node = GET_NODE(self->dict, key);
@ -310,10 +282,6 @@ lru_ass_sub(LRU *self, PyObject *key, PyObject *value)
Py_INCREF(value); Py_INCREF(value);
Py_DECREF(node->value); Py_DECREF(node->value);
node->value = value; node->value = value;
lru_remove_node(self, node);
lru_add_node_at_head(self, node);
res = 0; res = 0;
} else { } else {
node = PyObject_NEW(Node, &NodeType); node = PyObject_NEW(Node, &NodeType);
@ -326,19 +294,12 @@ lru_ass_sub(LRU *self, PyObject *key, PyObject *value)
res = PUT_NODE(self->dict, key, node); res = PUT_NODE(self->dict, key, node);
if (res == 0) { if (res == 0) {
if (lru_length(self) > self->size) { if (self->size > 0 && lru_length(self) > self->size) lru_delete_last(self);
lru_delete_last(self);
}
lru_add_node_at_head(self, node); lru_add_node_at_head(self, node);
} }
} }
} else { } else {
res = PUT_NODE(self->dict, key, NULL); if (PUT_NODE(self->dict, key, NULL) == 0) lru_remove_node(self, node);
if (res == 0) {
assert(node && PyObject_TypeCheck(node, &NodeType));
lru_remove_node(self, node);
}
} }
Py_XDECREF(node); Py_XDECREF(node);
@ -351,8 +312,7 @@ static PyMappingMethods LRU_as_mapping = {
(objobjargproc)lru_ass_sub, /*mp_ass_subscript*/ (objobjargproc)lru_ass_sub, /*mp_ass_subscript*/
}; };
static PyObject * static PyObject *collect(LRU *self, PyObject * (*getterfunc)(Node *))
collect(LRU *self, PyObject * (*getterfunc)(Node *))
{ {
register PyObject *v; register PyObject *v;
Node *curr; Node *curr;
@ -371,15 +331,34 @@ collect(LRU *self, PyObject * (*getterfunc)(Node *))
return v; return v;
} }
static PyObject * static PyObject *get_key(Node *node)
get_key(Node *node)
{ {
Py_INCREF(node->key); Py_INCREF(node->key);
return node->key; return node->key;
} }
static PyObject * static PyObject *LRU_put(LRU *self, PyObject *args, PyObject *kwargs)
LRU_update(LRU *self, PyObject *args, PyObject *kwargs) {
PyObject *key, *value;
PyObject *arg = NULL;
Py_ssize_t pos = 0;
if ((PyArg_ParseTuple(args, "|O", &arg))) {
if (arg && PyDict_Check(arg)) {
while (PyDict_Next(arg, &pos, &key, &value))
lru_put(self, key, value);
}
}
if (kwargs != NULL && PyDict_Check(kwargs)) {
while (PyDict_Next(kwargs, &pos, &key, &value))
lru_put(self, key, value);
}
Py_RETURN_NONE;
}
static PyObject *LRU_update(LRU *self, PyObject *args, PyObject *kwargs)
{ {
PyObject *key, *value; PyObject *key, *value;
PyObject *arg = NULL; PyObject *arg = NULL;
@ -400,8 +379,7 @@ LRU_update(LRU *self, PyObject *args, PyObject *kwargs)
Py_RETURN_NONE; Py_RETURN_NONE;
} }
static PyObject * static PyObject *LRU_peek_first_item(LRU *self)
LRU_peek_first_item(LRU *self)
{ {
if (self->first) { if (self->first) {
PyObject *tuple = PyTuple_New(2); PyObject *tuple = PyTuple_New(2);
@ -414,8 +392,7 @@ LRU_peek_first_item(LRU *self)
else Py_RETURN_NONE; else Py_RETURN_NONE;
} }
static PyObject * static PyObject *LRU_peek_last_item(LRU *self)
LRU_peek_last_item(LRU *self)
{ {
if (self->last) { if (self->last) {
PyObject *tuple = PyTuple_New(2); PyObject *tuple = PyTuple_New(2);
@ -428,32 +405,18 @@ LRU_peek_last_item(LRU *self)
else Py_RETURN_NONE; else Py_RETURN_NONE;
} }
static PyObject * static PyObject *LRU_keys(LRU *self) { return collect(self, get_key); }
LRU_keys(LRU *self) {
return collect(self, get_key);
}
static PyObject * static PyObject *get_value(Node *node)
get_value(Node *node)
{ {
Py_INCREF(node->value); Py_INCREF(node->value);
return node->value; return node->value;
} }
static PyObject * static PyObject *LRU_values(LRU *self) { return collect(self, get_value); }
LRU_values(LRU *self)
{
return collect(self, get_value);
}
static PyObject *
LRU_set_callback(LRU *self, PyObject *args)
{
return set_callback(self, args);
}
static PyObject * static PyObject *get_item(Node *node)
get_item(Node *node)
{ {
PyObject *tuple = PyTuple_New(2); PyObject *tuple = PyTuple_New(2);
Py_INCREF(node->key); Py_INCREF(node->key);
@ -463,32 +426,24 @@ get_item(Node *node)
return tuple; return tuple;
} }
static PyObject * static PyObject *LRU_items(LRU *self) { return collect(self, get_item); }
LRU_items(LRU *self)
{
return collect(self, get_item);
}
static PyObject * static PyObject *LRU_set_size(LRU *self, PyObject *args, PyObject *kwds)
LRU_set_size(LRU *self, PyObject *args, PyObject *kwds)
{ {
Py_ssize_t newSize; Py_ssize_t newSize;
if (!PyArg_ParseTuple(args, "n", &newSize)) { if (!PyArg_ParseTuple(args, "n", &newSize)) return NULL;
return NULL;
}
if (newSize <= 0) { if (newSize <= 0) {
PyErr_SetString(PyExc_ValueError, "Size should be a positive number"); PyErr_SetString(PyExc_ValueError, "Size should be a positive number");
return NULL; return NULL;
} }
while (lru_length(self) > newSize) { while (lru_length(self) > newSize) lru_delete_last(self);
lru_delete_last(self);
}
self->size = newSize; self->size = newSize;
Py_RETURN_NONE; Py_RETURN_NONE;
} }
static PyObject * static PyObject *LRU_clear(LRU *self)
LRU_clear(LRU *self)
{ {
Node *c = self->first; Node *c = self->first;
@ -499,23 +454,12 @@ LRU_clear(LRU *self)
} }
PyDict_Clear(self->dict); PyDict_Clear(self->dict);
self->hits = 0;
self->misses = 0;
Py_RETURN_NONE; Py_RETURN_NONE;
} }
static PyObject * static PyObject *LRU_get_size(LRU *self) { return Py_BuildValue("i", self->size); }
LRU_get_size(LRU *self)
{
return Py_BuildValue("i", self->size);
}
static PyObject *
LRU_get_stats(LRU *self)
{
return Py_BuildValue("nn", self->hits, self->misses);
}
/* Hack to implement "key in lru" */ /* Hack to implement "key in lru" */
@ -551,8 +495,6 @@ static PyMethodDef LRU_methods[] = {
PyDoc_STR("L.get_size() -> get size of LRU")}, PyDoc_STR("L.get_size() -> get size of LRU")},
{"clear", (PyCFunction)LRU_clear, METH_NOARGS, {"clear", (PyCFunction)LRU_clear, METH_NOARGS,
PyDoc_STR("L.clear() -> clear LRU")}, PyDoc_STR("L.clear() -> clear LRU")},
{"get_stats", (PyCFunction)LRU_get_stats, METH_NOARGS,
PyDoc_STR("L.get_stats() -> returns a tuple with cache hits and misses")},
{"peek_first_item", (PyCFunction)LRU_peek_first_item, METH_NOARGS, {"peek_first_item", (PyCFunction)LRU_peek_first_item, METH_NOARGS,
PyDoc_STR("L.peek_first_item() -> returns the MRU item (key,value) without changing key order")}, PyDoc_STR("L.peek_first_item() -> returns the MRU item (key,value) without changing key order")},
{"peek_last_item", (PyCFunction)LRU_peek_last_item, METH_NOARGS, {"peek_last_item", (PyCFunction)LRU_peek_last_item, METH_NOARGS,
@ -561,44 +503,28 @@ static PyMethodDef LRU_methods[] = {
PyDoc_STR("L.pop() -> returns the LRU item (key,value) without changing key order")}, PyDoc_STR("L.pop() -> returns the LRU item (key,value) without changing key order")},
{"update", (PyCFunction)LRU_update, METH_VARARGS | METH_KEYWORDS, {"update", (PyCFunction)LRU_update, METH_VARARGS | METH_KEYWORDS,
PyDoc_STR("L.update() -> update value for key in LRU")}, PyDoc_STR("L.update() -> update value for key in LRU")},
{"set_callback", (PyCFunction)LRU_set_callback, METH_VARARGS, {"put", (PyCFunction)LRU_put, METH_VARARGS | METH_KEYWORDS,
PyDoc_STR("L.set_callback(callback) -> set a callback to call when an item is evicted.")}, PyDoc_STR("L.update() -> update value for key in LRU")},
{NULL, NULL}, {NULL, NULL},
}; };
static PyObject* static PyObject*LRU_repr(LRU* self) { return PyObject_Repr(self->dict); }
LRU_repr(LRU* self)
{
return PyObject_Repr(self->dict);
}
static int static int
LRU_init(LRU *self, PyObject *args, PyObject *kwds) LRU_init(LRU *self, PyObject *args, PyObject *kwds)
{ {
static char *kwlist[] = {"size", "callback", NULL}; static char *kwlist[] = {"size", NULL};
PyObject *callback = NULL;
self->callback = NULL;
if (!PyArg_ParseTupleAndKeywords(args, kwds, "n|O", kwlist, &self->size, &callback)) {
return -1;
}
if (callback && callback != Py_None) { if (!PyArg_ParseTupleAndKeywords(args, kwds, "n|O", kwlist, &self->size)) { return -1; }
if (!PyCallable_Check(callback)) {
PyErr_SetString(PyExc_TypeError, "parameter must be callable");
return -1;
}
Py_XINCREF(callback);
self->callback = callback;
}
if ((Py_ssize_t)self->size <= 0) {
if ((Py_ssize_t)self->size < 0) {
PyErr_SetString(PyExc_ValueError, "Size should be a positive number"); PyErr_SetString(PyExc_ValueError, "Size should be a positive number");
return -1; return -1;
} }
self->dict = PyDict_New(); self->dict = PyDict_New();
self->first = self->last = NULL; self->first = self->last = NULL;
self->hits = 0;
self->misses = 0;
return 0; return 0;
} }
@ -608,17 +534,15 @@ LRU_dealloc(LRU *self)
if (self->dict) { if (self->dict) {
LRU_clear(self); LRU_clear(self);
Py_DECREF(self->dict); Py_DECREF(self->dict);
Py_XDECREF(self->callback);
} }
PyObject_Del((PyObject*)self); PyObject_Del((PyObject*)self);
} }
PyDoc_STRVAR(lru_doc, PyDoc_STRVAR(lru_doc,
"LRU(size, callback=None) -> new LRU dict that can store up to size elements\n" "LRU(size) -> new LRU dict that can store up to size elements\n"
"An LRU dict behaves like a standard dict, except that it stores only fixed\n" "An LRU dict behaves like a standard dict, except that it stores only fixed\n"
"set of elements. Once the size overflows, it evicts least recently used\n" "set of elements. Once the size overflows, it evicts least recently used\n"
"items. If a callback is set it will call the callback with the evicted key\n" "items. \n\n"
" and item.\n\n"
"Eg:\n" "Eg:\n"
">>> l = LRU(3)\n" ">>> l = LRU(3)\n"
">>> for i in range(5):\n" ">>> for i in range(5):\n"
@ -669,8 +593,8 @@ static PyTypeObject LRUType = {
0, /* tp_new */ 0, /* tp_new */
}; };
#if PY_MAJOR_VERSION >= 3
static struct PyModuleDef moduledef = { static struct PyModuleDef moduledef = {
PyModuleDef_HEAD_INIT, PyModuleDef_HEAD_INIT,
"_lru", /* m_name */ "_lru", /* m_name */
lru_doc, /* m_doc */ lru_doc, /* m_doc */
@ -681,47 +605,20 @@ static PyTypeObject LRUType = {
NULL, /* m_clear */ NULL, /* m_clear */
NULL, /* m_free */ NULL, /* m_free */
}; };
#endif
static PyObject *
moduleinit(void) static PyObject *moduleinit(void) {
{
PyObject *m; PyObject *m;
NodeType.tp_new = PyType_GenericNew; NodeType.tp_new = PyType_GenericNew;
if (PyType_Ready(&NodeType) < 0) if (PyType_Ready(&NodeType) < 0) return NULL;
return NULL;
LRUType.tp_new = PyType_GenericNew; LRUType.tp_new = PyType_GenericNew;
if (PyType_Ready(&LRUType) < 0) if (PyType_Ready(&LRUType) < 0) return NULL;
return NULL; m = PyModule_Create(&moduledef);
if (m == NULL) return NULL;
#if PY_MAJOR_VERSION >= 3
m = PyModule_Create(&moduledef);
#else
m = Py_InitModule3("_lru", NULL, lru_doc);
#endif
if (m == NULL)
return NULL;
Py_INCREF(&NodeType); Py_INCREF(&NodeType);
Py_INCREF(&LRUType); Py_INCREF(&LRUType);
PyModule_AddObject(m, "LRU", (PyObject *) &LRUType); PyModule_AddObject(m, "LRU", (PyObject *) &LRUType);
return m; return m;
} }
#if PY_MAJOR_VERSION < 3 PyMODINIT_FUNC PyInit__lru(void) { return moduleinit(); }
PyMODINIT_FUNC
initlru(void)
{
moduleinit();
}
#else
PyMODINIT_FUNC
PyInit__lru(void)
{
return moduleinit();
}
#endif

6
pybtc/connector/block_loader.py
View File

@ -178,9 +178,7 @@ class BlockLoader:
for i in blocks: for i in blocks:
self.parent.block_preload.set(i, blocks[i]) self.parent.block_preload.set(i, blocks[i])
if blocks: if blocks:
last_tx = blocks[i]["rawTx"][len(blocks[i]["rawTx"]) - 1] self.parent.checkpoints.append(i)
pointer = (i << 42) + ((len(blocks[i]["rawTx"]) - 1) << 21) + (en(last_tx["vOut"]) - 1)
self.parent.checkpoints.append(pointer)
# def disconnect(self,ip): # def disconnect(self,ip):
@ -263,6 +261,8 @@ class Worker:
block["rawTx"][z]["vOut"][i]["addressHash"])) block["rawTx"][z]["vOut"][i]["addressHash"]))
self.coins[o] = (pointer, block["rawTx"][z]["vOut"][i]["value"], address) self.coins[o] = (pointer, block["rawTx"][z]["vOut"][i]["value"], address)
blocks[x] = block blocks[x] = block
if blocks:
blocks[x]["checkpoint"] = x
for x in blocks: for x in blocks:
for y in blocks[x]["rawTx"]: for y in blocks[x]["rawTx"]:
for i in blocks[x]["rawTx"][y]["vOut"]: for i in blocks[x]["rawTx"][y]["vOut"]:

2
pybtc/connector/connector.py
View File

@ -375,6 +375,8 @@ class Connector:
self.last_block_height = block["height"] self.last_block_height = block["height"]
if self.utxo_data: if self.utxo_data:
self.utxo.destroy_utxo() self.utxo.destroy_utxo()
try: self.checkpoints.append(block["checkpoint"])
except: pass
self.blocks_processed_count += 1 self.blocks_processed_count += 1

8
pybtc/connector/utxo.py
View File

@ -37,7 +37,7 @@ class UTXO():
self.outs_total = 0 self.outs_total = 0
def set(self, outpoint, pointer, amount, address): def set(self, outpoint, pointer, amount, address):
self.cached[outpoint] = (pointer, amount, address) self.cached.put({outpoint: (pointer, amount, address)})
def remove(self, outpoint): def remove(self, outpoint):
@ -57,12 +57,6 @@ class UTXO():
self.destroyed_utxo += 1 self.destroyed_utxo += 1
pass pass
if len(self.cached) > self._cache_hard_limit:
await self.save_utxo()
elif len(self.cached) > self._cache_soft_limit and self.save_future.done():
self.loop.create_task(self.save_utxo())
async def save_utxo(self): async def save_utxo(self):
# save to db tail from cache # save to db tail from cache