diff --git a/pybtc/__init__.py b/pybtc/__init__.py index 76cc530..847b3d3 100644 --- a/pybtc/__init__.py +++ b/pybtc/__init__.py @@ -7,6 +7,6 @@ from .block import * from .address import * from .wallet import * from .crypto import * -from pybtc.cache_strategies import PLE from pybtc.connector import Connector +from cache_strategies import LRU, MRU diff --git a/pybtc/cache_strategies/cache.c b/pybtc/cache_strategies/cache.c new file mode 100644 index 0000000..3be0cf6 --- /dev/null +++ b/pybtc/cache_strategies/cache.c @@ -0,0 +1,741 @@ +#include + +#ifndef Py_TYPE + #define Py_TYPE(ob) (((PyObject*)(ob))->ob_type) +#endif + +#define GET_NODE(d, key) (Node *) Py_TYPE(d)->tp_as_mapping->mp_subscript((d), (key)) +#define PUT_NODE(d, key, node) Py_TYPE(d)->tp_as_mapping->mp_ass_subscript((d), (key), ((PyObject *)node)) + + +typedef struct _Node { + PyObject_HEAD + PyObject * value; + PyObject * key; + struct _Node * prev; + struct _Node * next; +} Node; + +static void node_dealloc(Node* self) { + Py_DECREF(self->key); + Py_DECREF(self->value); + PyObject_Del((PyObject*)self); +} + +static PyObject*node_repr(Node* self) {return PyObject_Repr(self->value);} + +static PyTypeObject NodeType = { + PyVarObject_HEAD_INIT(NULL, 0) + "cache.Node", /* tp_name */ + sizeof(Node), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)node_dealloc,/* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + (reprfunc)node_repr, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + "Linked List Node", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + 0, /* tp_new */ +}; + +typedef struct { + PyObject_HEAD + PyObject * dict; + Node * first; + Node * last; + Py_ssize_t size; +} CACHE; + +static Py_ssize_t cache_length(CACHE *self) {return PyDict_Size(self->dict);} + + +static void cache_remove_node(CACHE *self, Node* node) { + if (self->first == node) { + self->first = node->next; + } + if (self->last == node) { + self->last = node->prev; + } + if (node->prev) { + node->prev->next = node->next; + } + if (node->next) { + node->next->prev = node->prev; + } + node->next = node->prev = NULL; +} + +static void cache_add_node_at_head(CACHE *self, Node* node) { + node->prev = NULL; + if (!self->first) { + self->first = self->last = node; + node->next = NULL; + } else { + node->next = self->first; + if (node->next) { + node->next->prev = node; + } + self->first = node; + } +} + +static void cache_add_node_at_tail(CACHE *self, Node* node) { + node->next = NULL; + if (!self->first) { + self->first = self->last = node; + node->prev = NULL; + } else { + node->prev = self->last; + if (node->prev) { + node->prev->next = node; + } + self->last = node; + } +} + +static void cache_delete_last(CACHE *self) { + Node* n = self->last; + if (!self->last) return; + cache_remove_node(self, n); + PyDict_DelItem(self->dict, n->key); +} + +static int cache_append(CACHE *self, PyObject *key, PyObject *value) { + int res = 0; + Node *node = GET_NODE(self->dict, key); + PyErr_Clear(); + if (value) { + if (node) { + Py_INCREF(value); + Py_DECREF(node->value); + node->value = value; + res = 0; + } else { + node = PyObject_NEW(Node, &NodeType); + node->key = key; + node->value = value; + node->next = node->prev = NULL; + + Py_INCREF(key); + Py_INCREF(value); + + res = PUT_NODE(self->dict, key, node); + if (res == 0) { + if (self->size > 0 && cache_length(self) > self->size) cache_delete_last(self); + cache_add_node_at_tail(self, node); + } + } + } else { + + if (PUT_NODE(self->dict, key, NULL) == 0) cache_remove_node(self, node); + } + + Py_XDECREF(node); + return res; +} + +static PyObject *cache_contains_key(CACHE *self, PyObject *key) { + if (PyDict_Contains(self->dict, key)) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } +} + +static int cache_seq_contains(CACHE *self, PyObject *key) {return PyDict_Contains(self->dict, key);} + +static PyObject *collect(CACHE *self, PyObject * (*getterfunc)(Node *)) { + register PyObject *v; + Node *curr; + int i; + v = PyList_New(cache_length(self)); + if (v == NULL) + return NULL; + curr = self->first; + i = 0; + + while (curr) { + PyList_SET_ITEM(v, i++, getterfunc(curr)); + curr = curr->next; + } + + return v; +} + +static PyObject *get_key(Node *node) { + Py_INCREF(node->key); + return node->key; +} + +static PyObject *get_value(Node *node) { + Py_INCREF(node->value); + return node->value; +} + +static PyObject *get_item(Node *node) { + PyObject *tuple = PyTuple_New(2); + Py_INCREF(node->key); + PyTuple_SET_ITEM(tuple, 0, node->key); + Py_INCREF(node->value); + PyTuple_SET_ITEM(tuple, 1, node->value); + return tuple; +} + + + +/* cache methods */ + +static PyObject *CACHE_keys(CACHE *self) {return collect(self, get_key);} + +static PyObject *CACHE_values(CACHE *self) {return collect(self, get_value);} + +static PyObject *CACHE_items(CACHE *self) {return collect(self, get_item);} + +static PyObject *CACHE_contains(CACHE *self, PyObject *args) { + PyObject *key; + if (!PyArg_ParseTuple(args, "O", &key)) return NULL; + + return cache_contains_key(self, key); +} + +static PyObject *CACHE_delete(CACHE *self, PyObject *args) { + PyObject *key; + PyObject *instead = NULL; + if (!PyArg_ParseTuple(args, "O|O", &key, &instead)) return NULL; + Node *node = GET_NODE(self->dict, key); + + if (!node) { + if (!instead) { + Py_XDECREF(node); + Py_RETURN_NONE; + } + + Py_INCREF(instead); + Py_XDECREF(node); + return instead; + } + + Py_INCREF(node->value); + cache_remove_node(self, node); + PyDict_DelItem(self->dict, node->key); + Py_XDECREF(node); + + return node->value; +} + +static PyObject *CACHE_set_size(CACHE *self, PyObject *args, PyObject *kwds) { + Py_ssize_t newSize; + if (!PyArg_ParseTuple(args, "n", &newSize)) return NULL; + + if (newSize < 0) { + PyErr_SetString(PyExc_ValueError, "Size should be a positive number"); + return NULL; + } + while (cache_length(self) > newSize) cache_delete_last(self); + + self->size = newSize; + Py_RETURN_NONE; +} + +static PyObject *CACHE_get_size(CACHE *self) {return Py_BuildValue("i", self->size);} + +static PyObject *CACHE_clear(CACHE *self) { + Node *c = self->first; + + while (c) { + Node* n = c; + c = c->next; + cache_remove_node(self, n); + } + PyDict_Clear(self->dict); + + Py_RETURN_NONE; +} + +static PyObject *CACHE_peek_first_item(CACHE *self) { + if (self->first) { + PyObject *tuple = PyTuple_New(2); + Py_INCREF(self->first->key); + PyTuple_SET_ITEM(tuple, 0, self->first->key); + Py_INCREF(self->first->value); + PyTuple_SET_ITEM(tuple, 1, self->first->value); + return tuple; + } + else Py_RETURN_NONE; +} + +static PyObject *CACHE_peek_last_item(CACHE *self) { + if (self->last) { + PyObject *tuple = PyTuple_New(2); + Py_INCREF(self->last->key); + PyTuple_SET_ITEM(tuple, 0, self->last->key); + Py_INCREF(self->last->value); + PyTuple_SET_ITEM(tuple, 1, self->last->value); + return tuple; + } + else Py_RETURN_NONE; +} + +static PyObject *CACHE_pop(CACHE *self) { + if (self->last) { + PyObject *tuple = PyTuple_New(2); + Py_INCREF(self->last->key); + PyTuple_SET_ITEM(tuple, 0, self->last->key); + Py_INCREF(self->last->value); + PyTuple_SET_ITEM(tuple, 1, self->last->value); + Node* n = self->last; + cache_remove_node(self, n); + PyDict_DelItem(self->dict, n->key); + return tuple; + } + else Py_RETURN_NONE; +} + +static PyObject *CACHE_append(CACHE *self, PyObject *args, PyObject *kwargs) { + PyObject *key, *value; + PyObject *arg = NULL; + Py_ssize_t pos = 0; + + if ((PyArg_ParseTuple(args, "|O", &arg))) { + if (arg && PyDict_Check(arg)) { + while (PyDict_Next(arg, &pos, &key, &value)) + cache_append(self, key, value); + } + } + + if (kwargs != NULL && PyDict_Check(kwargs)) { + while (PyDict_Next(kwargs, &pos, &key, &value)) + cache_append(self, key, value); + } + + Py_RETURN_NONE; +} + + + +/* MRU - Most Recently Used */ + +static PyObject *mru_subscript(CACHE *self, register PyObject *key) { + Node *node = GET_NODE(self->dict, key); + if (!node) return NULL; + + Py_INCREF(node->value); + Py_DECREF(node); + return node->value; +} + +static int mru_ass_sub(CACHE *self, PyObject *key, PyObject *value) { + int res = 0; + Node *node = GET_NODE(self->dict, key); + PyErr_Clear(); + if (value) { + if (node) { + Py_INCREF(value); + Py_DECREF(node->value); + node->value = value; + res = 0; + } else { + node = PyObject_NEW(Node, &NodeType); + node->key = key; + node->value = value; + node->next = node->prev = NULL; + + Py_INCREF(key); + Py_INCREF(value); + + res = PUT_NODE(self->dict, key, node); + if (res == 0) { + if (self->size > 0 && cache_length(self) > self->size) cache_delete_last(self); + cache_add_node_at_head(self, node); + } + } + } else { + if (PUT_NODE(self->dict, key, NULL) == 0) cache_remove_node(self, node); + } + + Py_XDECREF(node); + return res; +} + +static PyMappingMethods MRU_as_mapping = { + (lenfunc)cache_length, /*mp_length*/ + (binaryfunc)mru_subscript, /*mp_subscript*/ + (objobjargproc)mru_ass_sub, /*mp_ass_subscript*/ +}; + + + +static PyObject *MRU_get(CACHE *self, PyObject *args) { + PyObject *key; + PyObject *instead = NULL; + PyObject *result; + + if (!PyArg_ParseTuple(args, "O|O", &key, &instead)) return NULL; + + result = mru_subscript(self, key); + PyErr_Clear(); /* GET_NODE sets an exception on miss. Shut it up. */ + if (result) return result; + + if (!instead) { Py_RETURN_NONE; } + + Py_INCREF(instead); + return instead; +} + +static PyObject *MRU_update(CACHE *self, PyObject *args, PyObject *kwargs) { + PyObject *key, *value; + PyObject *arg = NULL; + Py_ssize_t pos = 0; + + if ((PyArg_ParseTuple(args, "|O", &arg))) { + if (arg && PyDict_Check(arg)) { + while (PyDict_Next(arg, &pos, &key, &value)) + mru_ass_sub(self, key, value); + } + } + + if (kwargs != NULL && PyDict_Check(kwargs)) { + while (PyDict_Next(kwargs, &pos, &key, &value)) + mru_ass_sub(self, key, value); + } + + Py_RETURN_NONE; +} + + + + +/* LRU - Last Recently Used */ + + + +static PyObject * lru_subscript(CACHE *self, register PyObject *key) { + Node *node = GET_NODE(self->dict, key); + if (!node) return NULL; + + /* We don't need to move the node when it's already self->first. */ + if (node != self->first) { + cache_remove_node(self, node); + cache_add_node_at_head(self, node); + } + + Py_INCREF(node->value); + Py_DECREF(node); + return node->value; +} + +static PyObject *LRU_get(CACHE *self, PyObject *args) { + PyObject *key; + PyObject *instead = NULL; + PyObject *result; + + if (!PyArg_ParseTuple(args, "O|O", &key, &instead)) return NULL; + + result = lru_subscript(self, key); + PyErr_Clear(); /* GET_NODE sets an exception on miss. Shut it up. */ + if (result) return result; + + if (!instead) { Py_RETURN_NONE; } + + Py_INCREF(instead); + return instead; +} + +static int lru_ass_sub(CACHE *self, PyObject *key, PyObject *value) { + int res = 0; + Node *node = GET_NODE(self->dict, key); + PyErr_Clear(); + if (value) { + if (node) { + Py_INCREF(value); + Py_DECREF(node->value); + node->value = value; + + cache_remove_node(self, node); + cache_add_node_at_head(self, node); + res = 0; + } else { + node = PyObject_NEW(Node, &NodeType); + node->key = key; + node->value = value; + node->next = node->prev = NULL; + + Py_INCREF(key); + Py_INCREF(value); + + res = PUT_NODE(self->dict, key, node); + if (res == 0) { + if (self->size > 0 && cache_length(self) > self->size) cache_delete_last(self); + cache_add_node_at_head(self, node); + } + } + } else { + if (PUT_NODE(self->dict, key, NULL) == 0) cache_remove_node(self, node); + } + + Py_XDECREF(node); + return res; +} + +static PyMappingMethods LRU_as_mapping = { + (lenfunc)cache_length, /*mp_length*/ + (binaryfunc)lru_subscript, /*mp_subscript*/ + (objobjargproc)lru_ass_sub, /*mp_ass_subscript*/ +}; + +static PyObject *LRU_update(CACHE *self, PyObject *args, PyObject *kwargs) { + PyObject *key, *value; + PyObject *arg = NULL; + Py_ssize_t pos = 0; + + if ((PyArg_ParseTuple(args, "|O", &arg))) { + if (arg && PyDict_Check(arg)) { + while (PyDict_Next(arg, &pos, &key, &value)) + lru_ass_sub(self, key, value); + } + } + + if (kwargs != NULL && PyDict_Check(kwargs)) { + while (PyDict_Next(kwargs, &pos, &key, &value)) + lru_ass_sub(self, key, value); + } + + Py_RETURN_NONE; +} + + + + + + + + + +static int cache_init(CACHE *self, PyObject *args, PyObject *kwds) { + static char *kwlist[] = {"size", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|n", kwlist, &self->size)) self->size = 0; + + if ((Py_ssize_t)self->size < 0) { + PyErr_SetString(PyExc_ValueError, "Size should be a positive number"); + return -1; + } + self->dict = PyDict_New(); + self->first = self->last = NULL; + return 0; +} + +static void cache_dealloc(CACHE *self) { + if (self->dict) { + CACHE_clear(self); + Py_DECREF(self->dict); + } + PyObject_Del((PyObject*)self); +} + +static PySequenceMethods cache_as_sequence = { + 0, /* sq_length */ + 0, /* sq_concat */ + 0, /* sq_repeat */ + 0, /* sq_item */ + 0, /* sq_slice */ + 0, /* sq_ass_item */ + 0, /* sq_ass_slice */ + (objobjproc) cache_seq_contains,/* sq_contains */ + 0, /* sq_inplace_concat */ + 0, /* sq_inplace_repeat */ +}; + +static PyObject*cache_repr(CACHE* self) {return PyObject_Repr(self->dict);} + + + + + + + + +static PyMethodDef MRU_methods[] = { + {"__contains__", (PyCFunction)cache_contains_key, METH_O | METH_COEXIST, "L.__contains__(key) -> Check if key is there in L"}, + {"keys", (PyCFunction)CACHE_keys, METH_NOARGS, "L.keys() -> list of L's keys in MRU order"}, + {"values", (PyCFunction)CACHE_values, METH_NOARGS, "L.values() -> list of L's values in MRU order"}, + {"items", (PyCFunction)CACHE_items, METH_NOARGS, "L.items() -> list of L's items (key,value) in MRU order"}, + {"has_key", (PyCFunction)CACHE_contains, METH_VARARGS, "L.has_key(key) -> Check if key is there in L"}, + {"get", (PyCFunction)MRU_get, METH_VARARGS, "L.get(key, instead) -> If L has key return its value, otherwise instead"}, + {"delete", (PyCFunction)CACHE_delete, METH_VARARGS, "L.delete(key, instead) -> If L has key return its value, otherwise instead"}, + {"set_size", (PyCFunction)CACHE_set_size, METH_VARARGS, "L.set_size() -> set size of MRU"}, + {"get_size", (PyCFunction)CACHE_get_size, METH_NOARGS, "L.get_size() -> get size of MRU"}, + {"clear", (PyCFunction)CACHE_clear, METH_NOARGS, "L.clear() -> clear MRU"}, + {"peek_first_item", (PyCFunction)CACHE_peek_first_item, METH_NOARGS, "L.peek_first_item() -> returns the MRU item (key,value) without changing key order"}, + {"peek_last_item", (PyCFunction)CACHE_peek_last_item, METH_NOARGS, "L.peek_last_item() -> returns the MRU item (key,value) without changing key order"}, + {"pop", (PyCFunction)CACHE_pop, METH_NOARGS, "L.pop() -> returns the MRU item (key,value) without changing key order"}, + {"update", (PyCFunction)MRU_update, METH_VARARGS | METH_KEYWORDS, "L.update() -> update value for key in MRU"}, + {"append", (PyCFunction)CACHE_append, METH_VARARGS | METH_KEYWORDS, "L.append() -> append value for key in MRU"}, + {NULL, NULL}, +}; + +static PyMethodDef LRU_methods[] = { + {"__contains__", (PyCFunction)cache_contains_key, METH_O | METH_COEXIST, "L.__contains__(key) -> Check if key is there in L"}, + {"keys", (PyCFunction)CACHE_keys, METH_NOARGS, "L.keys() -> list of L's keys in LRU order"}, + {"values", (PyCFunction)CACHE_values, METH_NOARGS, "L.values() -> list of L's values in LRU order"}, + {"items", (PyCFunction)CACHE_items, METH_NOARGS, "L.items() -> list of L's items (key,value) in LRU order"}, + {"has_key", (PyCFunction)CACHE_contains, METH_VARARGS, "L.has_key(key) -> Check if key is there in L"}, + {"get", (PyCFunction)LRU_get, METH_VARARGS, "L.get(key, instead) -> If L has key return its value, otherwise instead"}, + {"delete", (PyCFunction)CACHE_delete, METH_VARARGS, "L.delete(key, instead) -> If L has key return its value, otherwise instead"}, + {"set_size", (PyCFunction)CACHE_set_size, METH_VARARGS, "L.set_size() -> set size of LRU"}, + {"get_size", (PyCFunction)CACHE_get_size, METH_NOARGS, "L.get_size() -> get size of LRU"}, + {"clear", (PyCFunction)CACHE_clear, METH_NOARGS, "L.clear() -> clear LRU"}, + {"peek_first_item", (PyCFunction)CACHE_peek_first_item, METH_NOARGS, "L.peek_first_item() -> returns the LRU item (key,value) without changing key order"}, + {"peek_last_item", (PyCFunction)CACHE_peek_last_item, METH_NOARGS, "L.peek_last_item() -> returns the LRU item (key,value) without changing key order"}, + {"pop", (PyCFunction)CACHE_pop, METH_NOARGS, "L.pop() -> returns the LRU item (key,value) without changing key order"}, + {"update", (PyCFunction)LRU_update, METH_VARARGS | METH_KEYWORDS, "L.update() -> update value for key in LRU"}, + {"append", (PyCFunction)CACHE_append, METH_VARARGS | METH_KEYWORDS, "L.append() -> append value for key in LRU"}, + {NULL, NULL}, +}; + + +static PyTypeObject MRUType = { + PyVarObject_HEAD_INIT(NULL, 0) + "cache_strategies.MRU", /* tp_name */ + sizeof(CACHE), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)cache_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + (reprfunc)cache_repr, /* tp_repr */ + 0, /* tp_as_number */ + &cache_as_sequence, /* tp_as_sequence */ + &MRU_as_mapping, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + 0, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + MRU_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)cache_init, /* tp_init */ + 0, /* tp_alloc */ + 0, /* tp_new */ +}; + +static PyTypeObject LRUType = { + PyVarObject_HEAD_INIT(NULL, 0) + "cache_strategies.LRU", /* tp_name */ + sizeof(CACHE), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)cache_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + (reprfunc)cache_repr, /* tp_repr */ + 0, /* tp_as_number */ + &cache_as_sequence, /* tp_as_sequence */ + &LRU_as_mapping, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + 0, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + LRU_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)cache_init, /* tp_init */ + 0, /* tp_alloc */ + 0, /* tp_new */ +}; + + + + + +static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + "cache_strategies", /* m_name */ + "cache strategies", /* m_doc */ + -1, /* m_size */ + NULL, /* m_methods */ + NULL, /* m_reload */ + NULL, /* m_traverse */ + NULL, /* m_clear */ + NULL, /* m_free */ + }; + + +static PyObject *moduleinit(void) { + PyObject *m; + NodeType.tp_new = PyType_GenericNew; + if (PyType_Ready(&NodeType) < 0) return NULL; + MRUType.tp_new = PyType_GenericNew; + if (PyType_Ready(&MRUType) < 0) return NULL; + LRUType.tp_new = PyType_GenericNew; + if (PyType_Ready(&LRUType) < 0) return NULL; + m = PyModule_Create(&moduledef); + if (m == NULL) return NULL; + Py_INCREF(&NodeType); + Py_INCREF(&MRUType); + Py_INCREF(&LRUType); + PyModule_AddObject(m, "MRU", (PyObject *) &MRUType); + PyModule_AddObject(m, "LRU", (PyObject *) &LRUType); + return m; +} + +PyMODINIT_FUNC PyInit_cache_strategies(void) {return moduleinit();} diff --git a/pybtc/cache_strategies/ple.c b/pybtc/cache_strategies/ple.c deleted file mode 100644 index 505d500..0000000 --- a/pybtc/cache_strategies/ple.c +++ /dev/null @@ -1,674 +0,0 @@ -#include - - - -#ifndef Py_TYPE - #define Py_TYPE(ob) (((PyObject*)(ob))->ob_type) -#endif - -#define GET_NODE(d, key) (Node *) Py_TYPE(d)->tp_as_mapping->mp_subscript((d), (key)) -#define PUT_NODE(d, key, node) Py_TYPE(d)->tp_as_mapping->mp_ass_subscript((d), (key), ((PyObject *)node)) - - -typedef struct _Node { - PyObject_HEAD - PyObject * value; - PyObject * key; - struct _Node * prev; - struct _Node * next; -} Node; - -static void node_dealloc(Node* self) -{ - Py_DECREF(self->key); - Py_DECREF(self->value); - assert(self->prev == NULL); - assert(self->next == NULL); - PyObject_Del((PyObject*)self); -} - -static PyObject*node_repr(Node* self) { return PyObject_Repr(self->value);} - -static PyTypeObject NodeType = { - PyVarObject_HEAD_INIT(NULL, 0) - "ple.Node", /* tp_name */ - sizeof(Node), /* tp_basicsize */ - 0, /* tp_itemsize */ - (destructor)node_dealloc,/* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_compare */ - (reprfunc)node_repr, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - "Linked List Node", /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - 0, /* tp_new */ -}; - -typedef struct { - PyObject_HEAD - PyObject * dict; - Node * first; - Node * last; - Py_ssize_t size; -} PLE; - - -static void ple_remove_node(PLE *self, Node* node) -{ - if (self->first == node) { - self->first = node->next; - } - if (self->last == node) { - self->last = node->prev; - } - if (node->prev) { - node->prev->next = node->next; - } - if (node->next) { - node->next->prev = node->prev; - } - node->next = node->prev = NULL; -} - -static void ple_add_node_at_head(PLE *self, Node* node) -{ - node->prev = NULL; - if (!self->first) { - self->first = self->last = node; - node->next = NULL; - } else { - node->next = self->first; - if (node->next) { - node->next->prev = node; - } - self->first = node; - } -} - -static void ple_add_node_at_tail(PLE *self, Node* node) -{ - node->next = NULL; - if (!self->first) { - self->first = self->last = node; - node->prev = NULL; - } else { - node->prev = self->last; - if (node->prev) { - node->prev->next = node; - } - self->last = node; - } -} - -static void ple_delete_last(PLE *self) -{ - Node* n = self->last; - if (!self->last) return; - ple_remove_node(self, n); - PyDict_DelItem(self->dict, n->key); -} - - -static Py_ssize_t ple_length(PLE *self) {return PyDict_Size(self->dict);} - -static PyObject *PLE_contains_key(PLE *self, PyObject *key) -{ - if (PyDict_Contains(self->dict, key)) { - Py_RETURN_TRUE; - } else { - Py_RETURN_FALSE; - } -} - -static PyObject *PLE_contains(PLE *self, PyObject *args) -{ - PyObject *key; - if (!PyArg_ParseTuple(args, "O", &key)) return NULL; - - return PLE_contains_key(self, key); -} - -static int PLE_seq_contains(PLE *self, PyObject *key) {return PyDict_Contains(self->dict, key);} - -static PyObject *ple_subscript(PLE *self, register PyObject *key) -{ - Node *node = GET_NODE(self->dict, key); - if (!node) return NULL; - - Py_INCREF(node->value); - Py_DECREF(node); - return node->value; -} - -static PyObject *PLE_pop(PLE *self) -{ - - if (self->last) { - PyObject *tuple = PyTuple_New(2); - Py_INCREF(self->last->key); - PyTuple_SET_ITEM(tuple, 0, self->last->key); - Py_INCREF(self->last->value); - PyTuple_SET_ITEM(tuple, 1, self->last->value); - Node* n = self->last; - ple_remove_node(self, n); - PyDict_DelItem(self->dict, n->key); - return tuple; - } - else Py_RETURN_NONE; -} - - -static PyObject *PLE_get(PLE *self, PyObject *args) -{ - PyObject *key; - PyObject *instead = NULL; - PyObject *result; - - if (!PyArg_ParseTuple(args, "O|O", &key, &instead)) return NULL; - - result = ple_subscript(self, key); - PyErr_Clear(); /* GET_NODE sets an exception on miss. Shut it up. */ - if (result) return result; - - if (!instead) { Py_RETURN_NONE; } - - Py_INCREF(instead); - return instead; -} - - -static PyObject *PLE_delete(PLE *self, PyObject *args) -{ - - PyObject *key; - PyObject *instead = NULL; - if (!PyArg_ParseTuple(args, "O|O", &key, &instead)) return NULL; - Node *node = GET_NODE(self->dict, key); - - if (!node) { - if (!instead) { - Py_XDECREF(node); - Py_RETURN_NONE; } - - Py_INCREF(instead); - Py_XDECREF(node); - return instead; - } - - - - - PyObject *tuple = PyTuple_New(2); - - Py_INCREF(node->key); - PyTuple_SET_ITEM(tuple, 0, node->key); - Py_INCREF(node->value); - PyTuple_SET_ITEM(tuple, 1, node->value); - ple_remove_node(self, node); - PyDict_DelItem(self->dict, node->key); - Py_XDECREF(node); - return tuple; -} - - - - -static int ple_append(PLE *self, PyObject *key, PyObject *value) -{ - int res = 0; - Node *node = GET_NODE(self->dict, key); - PyErr_Clear(); /* GET_NODE sets an exception on miss. Shut it up. */ - - if (value) { - if (node) { - Py_INCREF(value); - Py_DECREF(node->value); - node->value = value; - res = 0; - } else { - node = PyObject_NEW(Node, &NodeType); - node->key = key; - node->value = value; - node->next = node->prev = NULL; - - Py_INCREF(key); - Py_INCREF(value); - - res = PUT_NODE(self->dict, key, node); - if (res == 0) { - if (self->size > 0 && ple_length(self) > self->size) ple_delete_last(self); - ple_add_node_at_tail(self, node); - } - } - } else { - - if (PUT_NODE(self->dict, key, NULL) == 0) ple_remove_node(self, node); - } - - Py_XDECREF(node); - return res; -} - - -static int ple_ass_sub(PLE *self, PyObject *key, PyObject *value) -{ - int res = 0; - Node *node = GET_NODE(self->dict, key); - PyErr_Clear(); /* GET_NODE sets an exception on miss. Shut it up. */ - - if (value) { - if (node) { - Py_INCREF(value); - Py_DECREF(node->value); - node->value = value; - res = 0; - } else { - node = PyObject_NEW(Node, &NodeType); - node->key = key; - node->value = value; - node->next = node->prev = NULL; - - Py_INCREF(key); - Py_INCREF(value); - - res = PUT_NODE(self->dict, key, node); - if (res == 0) { - if (self->size > 0 && ple_length(self) > self->size) ple_delete_last(self); - ple_add_node_at_head(self, node); - } - } - } else { - if (PUT_NODE(self->dict, key, NULL) == 0) ple_remove_node(self, node); - } - - Py_XDECREF(node); - return res; -} - -static int ple_put(PLE *self, PyObject *key, PyObject *value) -{ - int res = 0; - - Node *node = PyObject_NEW(Node, &NodeType); - node->key = key; - node->value = value; - node->next = node->prev = NULL; - - Py_INCREF(key); - Py_INCREF(value); - - res = PUT_NODE(self->dict, key, node); - if (res == 0) { - if (self->size > 0 && ple_length(self) > self->size) ple_delete_last(self); - ple_add_node_at_head(self, node); - } - - Py_XDECREF(node); - return res; -} - -static PyMappingMethods PLE_as_mapping = { - (lenfunc)ple_length, /*mp_length*/ - (binaryfunc)ple_subscript, /*mp_subscript*/ - (objobjargproc)ple_ass_sub, /*mp_ass_subscript*/ -}; - -static PyObject *collect(PLE *self, PyObject * (*getterfunc)(Node *)) -{ - register PyObject *v; - Node *curr; - int i; - v = PyList_New(ple_length(self)); - if (v == NULL) - return NULL; - curr = self->first; - i = 0; - - while (curr) { - PyList_SET_ITEM(v, i++, getterfunc(curr)); - curr = curr->next; - } - assert(i == ple_length(self)); - return v; -} - -static PyObject *get_key(Node *node) -{ - Py_INCREF(node->key); - return node->key; -} - -static PyObject *PLE_append(PLE *self, PyObject *args, PyObject *kwargs) -{ - PyObject *key, *value; - PyObject *arg = NULL; - Py_ssize_t pos = 0; - - if ((PyArg_ParseTuple(args, "|O", &arg))) { - if (arg && PyDict_Check(arg)) { - while (PyDict_Next(arg, &pos, &key, &value)) - ple_append(self, key, value); - } - } - - if (kwargs != NULL && PyDict_Check(kwargs)) { - while (PyDict_Next(kwargs, &pos, &key, &value)) - ple_append(self, key, value); - } - - Py_RETURN_NONE; -} - -static PyObject *PLE_update(PLE *self, PyObject *args, PyObject *kwargs) -{ - PyObject *key, *value; - PyObject *arg = NULL; - Py_ssize_t pos = 0; - - if ((PyArg_ParseTuple(args, "|O", &arg))) { - if (arg && PyDict_Check(arg)) { - while (PyDict_Next(arg, &pos, &key, &value)) - ple_ass_sub(self, key, value); - } - } - - if (kwargs != NULL && PyDict_Check(kwargs)) { - while (PyDict_Next(kwargs, &pos, &key, &value)) - ple_ass_sub(self, key, value); - } - - Py_RETURN_NONE; -} - -static PyObject *PLE_put(PLE *self, PyObject *args, PyObject *kwargs) -{ - PyObject *key, *value; - PyObject *arg = NULL; - Py_ssize_t pos = 0; - - if ((PyArg_ParseTuple(args, "|O", &arg))) { - if (arg && PyDict_Check(arg)) { - while (PyDict_Next(arg, &pos, &key, &value)) - ple_put(self, key, value); - } - } - - if (kwargs != NULL && PyDict_Check(kwargs)) { - while (PyDict_Next(kwargs, &pos, &key, &value)) - ple_put(self, key, value); - } - - Py_RETURN_NONE; -} - -static PyObject *PLE_peek_first_item(PLE *self) -{ - if (self->first) { - PyObject *tuple = PyTuple_New(2); - Py_INCREF(self->first->key); - PyTuple_SET_ITEM(tuple, 0, self->first->key); - Py_INCREF(self->first->value); - PyTuple_SET_ITEM(tuple, 1, self->first->value); - return tuple; - } - else Py_RETURN_NONE; -} - -static PyObject *PLE_peek_last_item(PLE *self) -{ - if (self->last) { - PyObject *tuple = PyTuple_New(2); - Py_INCREF(self->last->key); - PyTuple_SET_ITEM(tuple, 0, self->last->key); - Py_INCREF(self->last->value); - PyTuple_SET_ITEM(tuple, 1, self->last->value); - return tuple; - } - else Py_RETURN_NONE; -} - -static PyObject *PLE_keys(PLE *self) {return collect(self, get_key);} - -static PyObject *get_value(Node *node) -{ - Py_INCREF(node->value); - return node->value; -} - -static PyObject *PLE_values(PLE *self) {return collect(self, get_value);} - - -static PyObject *get_item(Node *node) -{ - PyObject *tuple = PyTuple_New(2); - Py_INCREF(node->key); - PyTuple_SET_ITEM(tuple, 0, node->key); - Py_INCREF(node->value); - PyTuple_SET_ITEM(tuple, 1, node->value); - return tuple; -} - -static PyObject *PLE_items(PLE *self) {return collect(self, get_item);} - -static PyObject *PLE_set_size(PLE *self, PyObject *args, PyObject *kwds) -{ - Py_ssize_t newSize; - if (!PyArg_ParseTuple(args, "n", &newSize)) return NULL; - - if (newSize < 0) { - PyErr_SetString(PyExc_ValueError, "Size should be a positive number"); - return NULL; - } - while (ple_length(self) > newSize) ple_delete_last(self); - - self->size = newSize; - Py_RETURN_NONE; -} - -static PyObject *PLE_clear(PLE *self) -{ - Node *c = self->first; - - while (c) { - Node* n = c; - c = c->next; - ple_remove_node(self, n); - } - PyDict_Clear(self->dict); - - Py_RETURN_NONE; -} - - -static PyObject *PLE_get_size(PLE *self) {return Py_BuildValue("i", self->size);} - - - -/* Hack to implement "key in ple" */ -static PySequenceMethods ple_as_sequence = { - 0, /* sq_length */ - 0, /* sq_concat */ - 0, /* sq_repeat */ - 0, /* sq_item */ - 0, /* sq_slice */ - 0, /* sq_ass_item */ - 0, /* sq_ass_slice */ - (objobjproc) PLE_seq_contains, /* sq_contains */ - 0, /* sq_inplace_concat */ - 0, /* sq_inplace_repeat */ -}; - -static PyMethodDef PLE_methods[] = { - {"__contains__", (PyCFunction)PLE_contains_key, METH_O | METH_COEXIST, - PyDoc_STR("L.__contains__(key) -> Check if key is there in L")}, - {"keys", (PyCFunction)PLE_keys, METH_NOARGS, - PyDoc_STR("L.keys() -> list of L's keys in MRU order")}, - {"values", (PyCFunction)PLE_values, METH_NOARGS, - PyDoc_STR("L.values() -> list of L's values in MRU order")}, - {"items", (PyCFunction)PLE_items, METH_NOARGS, - PyDoc_STR("L.items() -> list of L's items (key,value) in MRU order")}, - {"has_key", (PyCFunction)PLE_contains, METH_VARARGS, - PyDoc_STR("L.has_key(key) -> Check if key is there in L")}, - {"get", (PyCFunction)PLE_get, METH_VARARGS, - PyDoc_STR("L.get(key, instead) -> If L has key return its value, otherwise instead")}, - {"delete", (PyCFunction)PLE_delete, METH_VARARGS, - PyDoc_STR("L.get(key, instead) -> If L has key return its value, otherwise instead")}, - {"set_size", (PyCFunction)PLE_set_size, METH_VARARGS, - PyDoc_STR("L.set_size() -> set size of LRU")}, - {"get_size", (PyCFunction)PLE_get_size, METH_NOARGS, - PyDoc_STR("L.get_size() -> get size of LRU")}, - {"clear", (PyCFunction)PLE_clear, METH_NOARGS, - PyDoc_STR("L.clear() -> clear LRU")}, - {"peek_first_item", (PyCFunction)PLE_peek_first_item, METH_NOARGS, - PyDoc_STR("L.peek_first_item() -> returns the MRU item (key,value) without changing key order")}, - {"peek_last_item", (PyCFunction)PLE_peek_last_item, METH_NOARGS, - PyDoc_STR("L.peek_last_item() -> returns the LRU item (key,value) without changing key order")}, - {"pop", (PyCFunction)PLE_pop, METH_NOARGS, - PyDoc_STR("L.pop() -> returns the LRU item (key,value) without changing key order")}, - {"update", (PyCFunction)PLE_update, METH_VARARGS | METH_KEYWORDS, - PyDoc_STR("L.update() -> update value for key in LRU")}, - {"put", (PyCFunction)PLE_put, METH_VARARGS | METH_KEYWORDS, - PyDoc_STR("L.append() -> append value for key in LRU")}, - {"append", (PyCFunction)PLE_append, METH_VARARGS | METH_KEYWORDS, - PyDoc_STR("L.append() -> append value for key in LRU")}, - - {NULL, NULL}, -}; - -static PyObject*PLE_repr(PLE* self) {return PyObject_Repr(self->dict);} - -static int PLE_init(PLE *self, PyObject *args, PyObject *kwds) -{ - static char *kwlist[] = {"size", NULL}; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|n", kwlist, &self->size)) self->size = 0; - - - if ((Py_ssize_t)self->size < 0) { - PyErr_SetString(PyExc_ValueError, "Size should be a positive number"); - return -1; - } - self->dict = PyDict_New(); - self->first = self->last = NULL; - return 0; -} - -static void PLE_dealloc(PLE *self) -{ - if (self->dict) { - PLE_clear(self); - Py_DECREF(self->dict); - } - PyObject_Del((PyObject*)self); -} - -PyDoc_STRVAR(ple_doc, -"LRU(size) -> new LRU dict that can store up to size elements\n" -"An LRU dict behaves like a standard dict, except that it stores only fixed\n" -"set of elements. Once the size overflows, it evicts least recently used\n" -"items. \n\n" -"Eg:\n" -">>> l = LRU(3)\n" -">>> for i in range(5):\n" -">>> l[i] = str(i)\n" -">>> l.keys()\n" -"[2,3,4]\n\n" -"Note: An LRU(n) can be thought of as a dict that will have the most\n" -"recently accessed n items.\n"); - -static PyTypeObject PLEType = { - PyVarObject_HEAD_INIT(NULL, 0) - "_ple.PLE", /* tp_name */ - sizeof(PLE), /* tp_basicsize */ - 0, /* tp_itemsize */ - (destructor)PLE_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_compare */ - (reprfunc)PLE_repr, /* tp_repr */ - 0, /* tp_as_number */ - &ple_as_sequence, /* tp_as_sequence */ - &PLE_as_mapping, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - ple_doc, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - PLE_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - (initproc)PLE_init, /* tp_init */ - 0, /* tp_alloc */ - 0, /* tp_new */ -}; - - -static struct PyModuleDef moduledef = { - PyModuleDef_HEAD_INIT, - "_ple", /* m_name */ - ple_doc, /* m_doc */ - -1, /* m_size */ - NULL, /* m_methods */ - NULL, /* m_reload */ - NULL, /* m_traverse */ - NULL, /* m_clear */ - NULL, /* m_free */ - }; - - -static PyObject *moduleinit(void) { - PyObject *m; - NodeType.tp_new = PyType_GenericNew; - if (PyType_Ready(&NodeType) < 0) return NULL; - PLEType.tp_new = PyType_GenericNew; - if (PyType_Ready(&PLEType) < 0) return NULL; - m = PyModule_Create(&moduledef); - if (m == NULL) return NULL; - Py_INCREF(&NodeType); - Py_INCREF(&PLEType); - PyModule_AddObject(m, "PLE", (PyObject *) &PLEType); - return m; -} - -PyMODINIT_FUNC PyInit__ple(void) {return moduleinit();} diff --git a/pybtc/connector/block_loader.py b/pybtc/connector/block_loader.py index ce09755..83152aa 100644 --- a/pybtc/connector/block_loader.py +++ b/pybtc/connector/block_loader.py @@ -12,7 +12,7 @@ import traceback from pybtc.connector.utils import decode_block_tx import _pickle as pickle -from pybtc.cache_strategies import PLE +from pybtc import MRU class BlockLoader: @@ -209,8 +209,8 @@ class Worker: self.loop.set_default_executor(ThreadPoolExecutor(20)) self.out_writer = out_writer self.in_reader = in_reader - self.coins = PLE(100000) - self.destroyed_coins = PLE(100000) + self.coins = MRU(100000) + self.destroyed_coins = MRU(100000) signal.signal(signal.SIGTERM, self.terminate) self.loop.create_task(self.message_loop()) self.loop.run_forever() @@ -246,9 +246,9 @@ class Worker: outpoint = b"".join((inp["txId"], int_to_bytes(inp["vOut"]))) try: r = self.coins.delete(outpoint) - block["rawTx"][z]["vIn"][i]["_c_"] = r[1] + block["rawTx"][z]["vIn"][i]["_c_"] = r t += 1 - self.destroyed_coins[r[1][0]] = True + self.destroyed_coins[r[0]] = True except: pass for i in block["rawTx"][z]["vOut"]: @@ -269,7 +269,7 @@ class Worker: try: pointer = (x << 42) + (y << 21) + i r = self.destroyed_coins.delete(pointer) - blocks[x]["rawTx"][y]["vOut"][i]["_s_"] = r[1] + blocks[x]["rawTx"][y]["vOut"][i]["_s_"] = r except: pass blocks[x] = pickle.dumps(blocks[x]) diff --git a/pybtc/connector/connector.py b/pybtc/connector/connector.py index 55435f8..09b4483 100644 --- a/pybtc/connector/connector.py +++ b/pybtc/connector/connector.py @@ -598,34 +598,42 @@ class Connector: async def _new_transaction(self, tx, block_time = None, block_height = None, block_index = None): if not(tx["txId"] in self.tx_in_process or self.tx_cache.get(tx["txId"])): try: - stxo = None + c = 0 self.tx_in_process.add(tx["txId"]) if not tx["coinbase"]: if block_height is not None: await self.wait_block_dependences(tx) if self.utxo: - stxo, missed = set(), set() + stxo, missed = dict(), set() for i in tx["vIn"]: inp = tx["vIn"][i] outpoint = b"".join((inp["txId"], int_to_bytes(inp["vOut"]))) try: - stxo.add((outpoint, inp["_c_"][0], inp["_c_"][1], inp["_c_"][2])) + tx["vIn"][i]["outpoint"] = outpoint + tx["vIn"][i]["coin"] = inp["_c_"] + c += 1 self.yy += 1 - except Exception as err: + except: r = self.utxo.get(outpoint) - stxo.add(r) if r else missed.add((outpoint, (block_height << 42) + (block_index << 21) + i)) + if r: + tx["vIn"][i]["coin"] = r + c += 1 + else: + missed.add((outpoint, (block_height << 42) + (block_index << 21) + i, i)) if missed: await self.utxo.load_utxo() - [stxo.add(self.utxo.get_loaded(o)) for o, s in missed] + for o, s, i in missed: + tx["vIn"][i]["coin"] = self.utxo.get_loaded(o) + c += 1 - if len(stxo) != len(tx["vIn"]) and not self.cache_loading: + + if c != len(tx["vIn"]) and not self.cache_loading: self.log.critical("utxo get failed " + rh2s(tx["txId"])) - self.log.critical(str(stxo)) - raise Exception("utxo get failed " + str("")) + raise Exception("utxo get failed ") if self.tx_handler and not self.cache_loading: - await self.tx_handler(tx, stxo, block_time, block_height, block_index) + await self.tx_handler(tx, block_time, block_height, block_index) if self.utxo: for i in tx["vOut"]: @@ -633,11 +641,11 @@ class Connector: self.tt += 1 else: out = tx["vOut"][i] - # if self.skip_opreturn and out["nType"] in (3, 8): - # continue + if self.skip_opreturn and out["nType"] in (3, 8): + continue pointer = (block_height << 42) + (block_index << 21) + i try: - address = out["scriptPubKey"] + address = b"".join((bytes([out["nType"]]), out["scriptPubKey"])) except: address = b"".join((bytes([out["nType"]]), out["addressHash"])) self.utxo.set(b"".join((tx["txId"], int_to_bytes(i))), pointer, out["value"], address) @@ -663,70 +671,6 @@ class Connector: finally: self.tx_in_process.remove(tx["txId"]) - # async def _new_transaction(self, tx, block_time = None, block_height = None, block_index = None): - # if not(tx["txId"] in self.tx_in_process or self.tx_cache.get(tx["txId"])): - # try: - # stxo = None - # self.tx_in_process.add(tx["txId"]) - # if not tx["coinbase"]: - # if block_height is not None: - # await self.wait_block_dependences(tx) - # if self.utxo: - # # stxo = await self.get_stxo(tx, block_height, block_index) - # stxo, missed = set(), set() - # for i in tx["vIn"]: - # inp = tx["vIn"][i] - # outpoint = b"".join((inp["txId"], int_to_bytes(inp["vOut"]))) - # r = self.utxo.get(outpoint, block_height) - # stxo.add(r) if r else missed.add((outpoint, - # (block_height << 42) + (block_index << 21) + i)) - # - # if missed: - # await self.utxo.load_utxo() - # [stxo.add(self.utxo.get_loaded(o, block_height)) for o, s in missed] - # - # if len(stxo) != len(tx["vIn"]) and not self.cache_loading: - # self.log.critical("utxo get failed " + rh2s(tx["txId"])) - # self.log.critical(str(stxo)) - # raise Exception("utxo get failed ") - # - # if self.tx_handler and not self.cache_loading: - # await self.tx_handler(tx, stxo, block_time, block_height, block_index) - # - # if self.utxo: - # for i in tx["vOut"]: - # out = tx["vOut"][i] - # if out["nType"] in (3, 8): - # continue - # pointer = (block_height << 42) + (block_index << 21) + i - # try: - # address = out["scriptPubKey"] - # except: - # address = b"".join((bytes([out["nType"]]), out["addressHash"])) - # outpoint = b"".join((tx["txId"], int_to_bytes(i))) - # self.utxo.set(outpoint, pointer, out["value"], address) - # - # self.tx_cache.set(tx["txId"], True) - # try: - # self.await_tx.remove(tx["txId"]) - # if not self.await_tx_future[tx["txId"]].done(): - # self.await_tx_future[tx["txId"]].set_result(True) - # if not self.await_tx: - # self.block_txs_request.set_result(True) - # except: - # pass - # except Exception as err: - # if tx["txId"] in self.await_tx: - # self.await_tx = set() - # self.block_txs_request.cancel() - # for i in self.await_tx_future: - # if not self.await_tx_future[i].done(): - # self.await_tx_future[i].cancel() - # self.log.debug("new transaction error %s " % err) - # self.log.debug(str(traceback.format_exc())) - # finally: - # self.tx_in_process.remove(tx["txId"]) - async def get_stxo(self, tx, block_height, block_index): stxo, missed = set(), set() diff --git a/pybtc/connector/utxo.py b/pybtc/connector/utxo.py index 8339108..7dd680c 100644 --- a/pybtc/connector/utxo.py +++ b/pybtc/connector/utxo.py @@ -1,17 +1,17 @@ from pybtc import int_to_c_int, c_int_to_int, c_int_len import asyncio -from collections import OrderedDict, deque -from pybtc.cache_strategies import PLE +from collections import OrderedDict +from pybtc import MRU class UTXO(): def __init__(self, db_pool, loop, log, cache_size): - self.cached = PLE() + self.cached = MRU() self.missed = set() self.deleted = set() self.pending_deleted = set() self.checkpoints = list() self.log = log - self.loaded = PLE() + self.loaded = MRU() self.pending_saved = OrderedDict() self.maturity = 100 self.size_limit = cache_size diff --git a/setup.py b/setup.py index ec9a087..7026e00 100644 --- a/setup.py +++ b/setup.py @@ -148,7 +148,7 @@ setup(name='pybtc', 'bdist_wheel': bdist_wheel }, distclass=Distribution, - ext_modules=[Extension("cache_strategies", ["pybtc/cache_strategies/ple.c"]), + ext_modules=[Extension("cache_strategies", ["pybtc/cache_strategies/cache.c"]), Extension("_secp256k1", ["pybtc/_secp256k1/module_secp256k1.c"], include_dirs=["libsecp256k1/include/", "libsecp256k1/src/"]), Extension("_crypto",