From ebb1e24e273bca5a9633f60f351bf81752ddd81c Mon Sep 17 00:00:00 2001 From: 4tochka Date: Thu, 16 May 2019 17:52:50 +0300 Subject: [PATCH] connector --- .../cache_strategies.cpython-37m-darwin.so | 0 .../pybtc/cache_strategies/cache.o | 0 pybtc/__init__.py | 4 +- .../{_lru/lru.c => cache_strategies/cache.c} | 871 +++++++++--------- pybtc/connector/block_loader.py | 26 +- pybtc/connector/utxo.py | 2 +- setup.py | 2 +- 7 files changed, 457 insertions(+), 448 deletions(-) create mode 100644 build/lib.macosx-10.9-x86_64-3.7/cache_strategies.cpython-37m-darwin.so create mode 100644 build/temp.macosx-10.9-x86_64-3.7/pybtc/cache_strategies/cache.o rename pybtc/{_lru/lru.c => cache_strategies/cache.c} (51%) diff --git a/build/lib.macosx-10.9-x86_64-3.7/cache_strategies.cpython-37m-darwin.so b/build/lib.macosx-10.9-x86_64-3.7/cache_strategies.cpython-37m-darwin.so new file mode 100644 index 0000000..e69de29 diff --git a/build/temp.macosx-10.9-x86_64-3.7/pybtc/cache_strategies/cache.o b/build/temp.macosx-10.9-x86_64-3.7/pybtc/cache_strategies/cache.o new file mode 100644 index 0000000..e69de29 diff --git a/pybtc/__init__.py b/pybtc/__init__.py index f475436..9eacda4 100644 --- a/pybtc/__init__.py +++ b/pybtc/__init__.py @@ -7,8 +7,8 @@ from .block import * from .address import * from .wallet import * from .crypto import * -# from cache_strategies import LRU -from _lru import LRU +from cache_strategies import LRU +from cache_strategies import MRU from pybtc.connector import Connector diff --git a/pybtc/_lru/lru.c b/pybtc/cache_strategies/cache.c similarity index 51% rename from pybtc/_lru/lru.c rename to pybtc/cache_strategies/cache.c index 5b57358..0ee65bd 100644 --- a/pybtc/_lru/lru.c +++ b/pybtc/cache_strategies/cache.c @@ -1,37 +1,5 @@ #include -/* - * This is a simple implementation of LRU Dict that uses a Python dict and an associated doubly linked - * list to keep track of recently inserted/accessed items. - * - * Dict will store: key -> Node mapping, where Node is a linked list node. - * The Node itself will contain the value as well as the key. - * - * For eg: - * - * >>> l = LRU(2) - * >>> l[0] = 'foo' - * >>> l[1] = 'bar' - * - * can be visualised as: - * - * ---+--(hash(0)--+--hash(1)--+ - * self->dict ...| | | - * ---+-----|------+---------|-+ - * | | - * +-----v------+ +-----v------+ - * self->first--->|<'foo'>, <0>|-->|<'bar'>, <1>|<---self->last - * +--| |<--| |--+ - * | +------------+ +------------+ | - * v v - * NULL NULL - * - * The invariant is to maintain the list to reflect the LRU order of items in the dict. - * self->first will point to the MRU item and self-last to LRU item. Size of list will not - * grow beyond size of LRU dict. - * - */ - #ifndef Py_TYPE #define Py_TYPE(ob) (((PyObject*)(ob))->ob_type) #endif @@ -39,20 +7,6 @@ #define GET_NODE(d, key) (Node *) Py_TYPE(d)->tp_as_mapping->mp_subscript((d), (key)) #define PUT_NODE(d, key, node) Py_TYPE(d)->tp_as_mapping->mp_ass_subscript((d), (key), ((PyObject *)node)) -/* If someone figures out how to enable debug builds with setuptools, you can delete this */ -#if 0 -#undef assert -#define str(s) #s -#define assert(v) \ - do { \ - if (!(v)) { \ - fprintf(stderr, "Assertion failed: %s on %s:%d\n", \ - str(v), __FILE__, __LINE__); \ - fflush(stderr); \ - abort(); \ - } \ - } while(0) -#endif typedef struct _Node { PyObject_HEAD @@ -62,20 +16,17 @@ typedef struct _Node { struct _Node * next; } Node; -static void node_dealloc(Node* self) -{ +static void node_dealloc(Node* self) { Py_DECREF(self->key); Py_DECREF(self->value); - assert(self->prev == NULL); - assert(self->next == NULL); PyObject_Del((PyObject*)self); } -static PyObject*node_repr(Node* self) { return PyObject_Repr(self->value);} +static PyObject*node_repr(Node* self) {return PyObject_Repr(self->value);} static PyTypeObject NodeType = { PyVarObject_HEAD_INIT(NULL, 0) - "lru.Node", /* tp_name */ + "cache.Node", /* tp_name */ sizeof(Node), /* tp_basicsize */ 0, /* tp_itemsize */ (destructor)node_dealloc,/* tp_dealloc */ @@ -120,11 +71,12 @@ typedef struct { Node * first; Node * last; Py_ssize_t size; -} LRU; +} CACHE; + +static Py_ssize_t cache_length(CACHE *self) {return PyDict_Size(self->dict);} -static void lru_remove_node(LRU *self, Node* node) -{ +static void cache_remove_node(CACHE *self, Node* node) { if (self->first == node) { self->first = node->next; } @@ -140,8 +92,7 @@ static void lru_remove_node(LRU *self, Node* node) node->next = node->prev = NULL; } -static void lru_add_node_at_head(LRU *self, Node* node) -{ +static void cache_add_node_at_head(CACHE *self, Node* node) { node->prev = NULL; if (!self->first) { self->first = self->last = node; @@ -155,8 +106,7 @@ static void lru_add_node_at_head(LRU *self, Node* node) } } -static void lru_add_node_at_tail(LRU *self, Node* node) -{ +static void cache_add_node_at_tail(CACHE *self, Node* node) { node->next = NULL; if (!self->first) { self->first = self->last = node; @@ -170,20 +120,48 @@ static void lru_add_node_at_tail(LRU *self, Node* node) } } -static void lru_delete_last(LRU *self) -{ +static void cache_delete_last(CACHE *self) { Node* n = self->last; if (!self->last) return; - lru_remove_node(self, n); + cache_remove_node(self, n); PyDict_DelItem(self->dict, n->key); -// PUT_NODE(self->dict, n->key, NULL); } +static int cache_append(CACHE *self, PyObject *key, PyObject *value) { + int res = 0; + Node *node = GET_NODE(self->dict, key); + PyErr_Clear(); + if (value) { + if (node) { + Py_INCREF(value); + Py_DECREF(node->value); + node->value = value; + res = 0; + } else { + node = PyObject_NEW(Node, &NodeType); + node->key = key; + node->value = value; + node->next = node->prev = NULL; -static Py_ssize_t lru_length(LRU *self) { return PyDict_Size(self->dict); } + Py_INCREF(key); + Py_INCREF(value); -static PyObject *LRU_contains_key(LRU *self, PyObject *key) -{ + res = PUT_NODE(self->dict, key, node); + if (res == 0) { + if (self->size > 0 && cache_length(self) > self->size) cache_delete_last(self); + cache_add_node_at_tail(self, node); + } + } + } else { + + if (PUT_NODE(self->dict, key, NULL) == 0) cache_remove_node(self, node); + } + + Py_XDECREF(node); + return res; +} + +static PyObject *cache_contains_key(CACHE *self, PyObject *key) { if (PyDict_Contains(self->dict, key)) { Py_RETURN_TRUE; } else { @@ -191,18 +169,180 @@ static PyObject *LRU_contains_key(LRU *self, PyObject *key) } } -static PyObject *LRU_contains(LRU *self, PyObject *args) -{ +static int cache_seq_contains(CACHE *self, PyObject *key) {return PyDict_Contains(self->dict, key);} + +static PyObject *collect(CACHE *self, PyObject * (*getterfunc)(Node *)) { + register PyObject *v; + Node *curr; + int i; + v = PyList_New(cache_length(self)); + if (v == NULL) + return NULL; + curr = self->first; + i = 0; + + while (curr) { + PyList_SET_ITEM(v, i++, getterfunc(curr)); + curr = curr->next; + } + + return v; +} + +static PyObject *get_key(Node *node) { + Py_INCREF(node->key); + return node->key; +} + +static PyObject *get_value(Node *node) { + Py_INCREF(node->value); + return node->value; +} + +static PyObject *get_item(Node *node) { + PyObject *tuple = PyTuple_New(2); + Py_INCREF(node->key); + PyTuple_SET_ITEM(tuple, 0, node->key); + Py_INCREF(node->value); + PyTuple_SET_ITEM(tuple, 1, node->value); + return tuple; +} + + + +/* cache methods */ + +static PyObject *CACHE_keys(CACHE *self) {return collect(self, get_key);} + +static PyObject *CACHE_values(CACHE *self) {return collect(self, get_value);} + +static PyObject *CACHE_items(CACHE *self) {return collect(self, get_item);} + +static PyObject *CACHE_contains(CACHE *self, PyObject *args) { PyObject *key; if (!PyArg_ParseTuple(args, "O", &key)) return NULL; - return LRU_contains_key(self, key); + return cache_contains_key(self, key); } -static int LRU_seq_contains(LRU *self, PyObject *key) { return PyDict_Contains(self->dict, key); } +static PyObject *CACHE_delete(CACHE *self, PyObject *args) { + PyObject *key; + PyObject *instead = NULL; + if (!PyArg_ParseTuple(args, "O|O", &key, &instead)) return NULL; + Node *node = GET_NODE(self->dict, key); -static PyObject *lru_subscript(LRU *self, register PyObject *key) -{ + if (!node) { + if (!instead) { + Py_XDECREF(node); + Py_RETURN_NONE; + } + + Py_INCREF(instead); + Py_XDECREF(node); + return instead; + } + + cache_remove_node(self, node); + PyDict_DelItem(self->dict, node->key); + Py_XDECREF(node); + + PyObject *return_value = Py_BuildValue("O", node->value); + return return_value; +} + +static PyObject *CACHE_set_size(CACHE *self, PyObject *args, PyObject *kwds) { + Py_ssize_t newSize; + if (!PyArg_ParseTuple(args, "n", &newSize)) return NULL; + + if (newSize < 0) { + PyErr_SetString(PyExc_ValueError, "Size should be a positive number"); + return NULL; + } + while (cache_length(self) > newSize) cache_delete_last(self); + + self->size = newSize; + Py_RETURN_NONE; +} + +static PyObject *CACHE_get_size(CACHE *self) {return Py_BuildValue("i", self->size);} + +static PyObject *CACHE_clear(CACHE *self) { + Node *c = self->first; + + while (c) { + Node* n = c; + c = c->next; + cache_remove_node(self, n); + } + PyDict_Clear(self->dict); + + Py_RETURN_NONE; +} + +static PyObject *CACHE_peek_first_item(CACHE *self) { + if (self->first) { + PyObject *tuple = PyTuple_New(2); + Py_INCREF(self->first->key); + PyTuple_SET_ITEM(tuple, 0, self->first->key); + Py_INCREF(self->first->value); + PyTuple_SET_ITEM(tuple, 1, self->first->value); + return tuple; + } + else Py_RETURN_NONE; +} + +static PyObject *CACHE_peek_last_item(CACHE *self) { + if (self->last) { + PyObject *tuple = PyTuple_New(2); + Py_INCREF(self->last->key); + PyTuple_SET_ITEM(tuple, 0, self->last->key); + Py_INCREF(self->last->value); + PyTuple_SET_ITEM(tuple, 1, self->last->value); + return tuple; + } + else Py_RETURN_NONE; +} + +static PyObject *CACHE_pop(CACHE *self) { + if (self->last) { + PyObject *tuple = PyTuple_New(2); + Py_INCREF(self->last->key); + PyTuple_SET_ITEM(tuple, 0, self->last->key); + Py_INCREF(self->last->value); + PyTuple_SET_ITEM(tuple, 1, self->last->value); + Node* n = self->last; + cache_remove_node(self, n); + PyDict_DelItem(self->dict, n->key); + return tuple; + } + else Py_RETURN_NONE; +} + +static PyObject *CACHE_append(CACHE *self, PyObject *args, PyObject *kwargs) { + PyObject *key, *value; + PyObject *arg = NULL; + Py_ssize_t pos = 0; + + if ((PyArg_ParseTuple(args, "|O", &arg))) { + if (arg && PyDict_Check(arg)) { + while (PyDict_Next(arg, &pos, &key, &value)) + cache_append(self, key, value); + } + } + + if (kwargs != NULL && PyDict_Check(kwargs)) { + while (PyDict_Next(kwargs, &pos, &key, &value)) + cache_append(self, key, value); + } + + Py_RETURN_NONE; +} + + + +/* MRU - Most Recently Used */ + +static PyObject *mru_subscript(CACHE *self, register PyObject *key) { Node *node = GET_NODE(self->dict, key); if (!node) return NULL; @@ -211,27 +351,107 @@ static PyObject *lru_subscript(LRU *self, register PyObject *key) return node->value; } -static PyObject *LRU_pop(LRU *self) -{ +static int mru_ass_sub(CACHE *self, PyObject *key, PyObject *value) { + int res = 0; + Node *node = GET_NODE(self->dict, key); + PyErr_Clear(); + if (value) { + if (node) { + Py_INCREF(value); + Py_DECREF(node->value); + node->value = value; + res = 0; + } else { + node = PyObject_NEW(Node, &NodeType); + node->key = key; + node->value = value; + node->next = node->prev = NULL; - if (self->last) { - PyObject *tuple = PyTuple_New(2); - Py_INCREF(self->last->key); - PyTuple_SET_ITEM(tuple, 0, self->last->key); - Py_INCREF(self->last->value); - PyTuple_SET_ITEM(tuple, 1, self->last->value); - Node* n = self->last; - lru_remove_node(self, n); - PyDict_DelItem(self->dict, n->key); -// PUT_NODE(self->dict, n->key, NULL); - return tuple; + Py_INCREF(key); + Py_INCREF(value); + + res = PUT_NODE(self->dict, key, node); + if (res == 0) { + if (self->size > 0 && cache_length(self) > self->size) cache_delete_last(self); + cache_add_node_at_head(self, node); + } + } + } else { + if (PUT_NODE(self->dict, key, NULL) == 0) cache_remove_node(self, node); } - else Py_RETURN_NONE; + + Py_XDECREF(node); + return res; +} + +static PyMappingMethods MRU_as_mapping = { + (lenfunc)cache_length, /*mp_length*/ + (binaryfunc)mru_subscript, /*mp_subscript*/ + (objobjargproc)mru_ass_sub, /*mp_ass_subscript*/ +}; + + + +static PyObject *MRU_get(CACHE *self, PyObject *args) { + PyObject *key; + PyObject *instead = NULL; + PyObject *result; + + if (!PyArg_ParseTuple(args, "O|O", &key, &instead)) return NULL; + + result = mru_subscript(self, key); + PyErr_Clear(); /* GET_NODE sets an exception on miss. Shut it up. */ + if (result) return result; + + if (!instead) { Py_RETURN_NONE; } + + Py_INCREF(instead); + return instead; +} + +static PyObject *MRU_update(CACHE *self, PyObject *args, PyObject *kwargs) { + PyObject *key, *value; + PyObject *arg = NULL; + Py_ssize_t pos = 0; + + if ((PyArg_ParseTuple(args, "|O", &arg))) { + if (arg && PyDict_Check(arg)) { + while (PyDict_Next(arg, &pos, &key, &value)) + mru_ass_sub(self, key, value); + } + } + + if (kwargs != NULL && PyDict_Check(kwargs)) { + while (PyDict_Next(kwargs, &pos, &key, &value)) + mru_ass_sub(self, key, value); + } + + Py_RETURN_NONE; } -static PyObject *LRU_get(LRU *self, PyObject *args) -{ + + +/* LRU - Last Recently Used */ + + + +static PyObject * lru_subscript(CACHE *self, register PyObject *key) { + Node *node = GET_NODE(self->dict, key); + if (!node) return NULL; + + /* We don't need to move the node when it's already self->first. */ + if (node != self->first) { + cache_remove_node(self, node); + cache_add_node_at_head(self, node); + } + + Py_INCREF(node->value); + Py_DECREF(node); + return node->value; +} + +static PyObject *LRU_get(CACHE *self, PyObject *args) { PyObject *key; PyObject *instead = NULL; PyObject *result; @@ -248,61 +468,18 @@ static PyObject *LRU_get(LRU *self, PyObject *args) return instead; } - -static PyObject *LRU_delete(LRU *self, PyObject *args) -{ - - PyObject *key; - PyObject *instead = NULL; - if (!PyArg_ParseTuple(args, "O|O", &key, &instead)) return NULL; - Node *node = GET_NODE(self->dict, key); - - if (!node) { - if (!instead) { - Py_XDECREF(node); - Py_RETURN_NONE; } - - Py_INCREF(instead); - Py_XDECREF(node); - return instead; - } - - - - - PyObject *tuple = PyTuple_New(2); - - Py_INCREF(node->key); - PyTuple_SET_ITEM(tuple, 0, node->key); - Py_INCREF(node->value); - PyTuple_SET_ITEM(tuple, 1, node->value); -// lru_delete_last(self); - lru_remove_node(self, node); - PyDict_DelItem(self->dict, node->key); -// PUT_NODE(self->dict, node->key, NULL); - Py_XDECREF(node); - return tuple; -// Py_RETURN_NONE; - - - - -} - - - - -static int lru_append(LRU *self, PyObject *key, PyObject *value) -{ +static int lru_ass_sub(CACHE *self, PyObject *key, PyObject *value) { int res = 0; Node *node = GET_NODE(self->dict, key); - PyErr_Clear(); /* GET_NODE sets an exception on miss. Shut it up. */ - + PyErr_Clear(); if (value) { if (node) { Py_INCREF(value); Py_DECREF(node->value); node->value = value; + + cache_remove_node(self, node); + cache_add_node_at_head(self, node); res = 0; } else { node = PyObject_NEW(Node, &NodeType); @@ -315,71 +492,12 @@ static int lru_append(LRU *self, PyObject *key, PyObject *value) res = PUT_NODE(self->dict, key, node); if (res == 0) { - if (self->size > 0 && lru_length(self) > self->size) lru_delete_last(self); - lru_add_node_at_tail(self, node); + if (self->size > 0 && cache_length(self) > self->size) cache_delete_last(self); + cache_add_node_at_head(self, node); } } } else { - - if (PUT_NODE(self->dict, key, NULL) == 0) lru_remove_node(self, node); - } - - Py_XDECREF(node); - return res; -} - - -static int lru_ass_sub(LRU *self, PyObject *key, PyObject *value) -{ - int res = 0; - Node *node = GET_NODE(self->dict, key); - PyErr_Clear(); /* GET_NODE sets an exception on miss. Shut it up. */ - - if (value) { - if (node) { - Py_INCREF(value); - Py_DECREF(node->value); - node->value = value; - res = 0; - } else { - node = PyObject_NEW(Node, &NodeType); - node->key = key; - node->value = value; - node->next = node->prev = NULL; - - Py_INCREF(key); - Py_INCREF(value); - - res = PUT_NODE(self->dict, key, node); - if (res == 0) { - if (self->size > 0 && lru_length(self) > self->size) lru_delete_last(self); - lru_add_node_at_head(self, node); - } - } - } else { - if (PUT_NODE(self->dict, key, NULL) == 0) lru_remove_node(self, node); - } - - Py_XDECREF(node); - return res; -} - -static int lru_put(LRU *self, PyObject *key, PyObject *value) -{ - int res = 0; - - Node *node = PyObject_NEW(Node, &NodeType); - node->key = key; - node->value = value; - node->next = node->prev = NULL; - - Py_INCREF(key); - Py_INCREF(value); - - res = PUT_NODE(self->dict, key, node); - if (res == 0) { - if (self->size > 0 && lru_length(self) > self->size) lru_delete_last(self); - lru_add_node_at_head(self, node); + if (PUT_NODE(self->dict, key, NULL) == 0) cache_remove_node(self, node); } Py_XDECREF(node); @@ -387,59 +505,12 @@ static int lru_put(LRU *self, PyObject *key, PyObject *value) } static PyMappingMethods LRU_as_mapping = { - (lenfunc)lru_length, /*mp_length*/ + (lenfunc)cache_length, /*mp_length*/ (binaryfunc)lru_subscript, /*mp_subscript*/ (objobjargproc)lru_ass_sub, /*mp_ass_subscript*/ }; -static PyObject *collect(LRU *self, PyObject * (*getterfunc)(Node *)) -{ - register PyObject *v; - Node *curr; - int i; - v = PyList_New(lru_length(self)); - if (v == NULL) - return NULL; - curr = self->first; - i = 0; - - while (curr) { - PyList_SET_ITEM(v, i++, getterfunc(curr)); - curr = curr->next; - } - assert(i == lru_length(self)); - return v; -} - -static PyObject *get_key(Node *node) -{ - Py_INCREF(node->key); - return node->key; -} - -static PyObject *LRU_append(LRU *self, PyObject *args, PyObject *kwargs) -{ - PyObject *key, *value; - PyObject *arg = NULL; - Py_ssize_t pos = 0; - - if ((PyArg_ParseTuple(args, "|O", &arg))) { - if (arg && PyDict_Check(arg)) { - while (PyDict_Next(arg, &pos, &key, &value)) - lru_append(self, key, value); - } - } - - if (kwargs != NULL && PyDict_Check(kwargs)) { - while (PyDict_Next(kwargs, &pos, &key, &value)) - lru_append(self, key, value); - } - - Py_RETURN_NONE; -} - -static PyObject *LRU_update(LRU *self, PyObject *args, PyObject *kwargs) -{ +static PyObject *LRU_update(CACHE *self, PyObject *args, PyObject *kwargs) { PyObject *key, *value; PyObject *arg = NULL; Py_ssize_t pos = 0; @@ -459,170 +530,19 @@ static PyObject *LRU_update(LRU *self, PyObject *args, PyObject *kwargs) Py_RETURN_NONE; } -static PyObject *LRU_put(LRU *self, PyObject *args, PyObject *kwargs) -{ - PyObject *key, *value; - PyObject *arg = NULL; - Py_ssize_t pos = 0; - - if ((PyArg_ParseTuple(args, "|O", &arg))) { - if (arg && PyDict_Check(arg)) { - while (PyDict_Next(arg, &pos, &key, &value)) - lru_put(self, key, value); - } - } - - if (kwargs != NULL && PyDict_Check(kwargs)) { - while (PyDict_Next(kwargs, &pos, &key, &value)) - lru_put(self, key, value); - } - - Py_RETURN_NONE; -} - -static PyObject *LRU_peek_first_item(LRU *self) -{ - if (self->first) { - PyObject *tuple = PyTuple_New(2); - Py_INCREF(self->first->key); - PyTuple_SET_ITEM(tuple, 0, self->first->key); - Py_INCREF(self->first->value); - PyTuple_SET_ITEM(tuple, 1, self->first->value); - return tuple; - } - else Py_RETURN_NONE; -} - -static PyObject *LRU_peek_last_item(LRU *self) -{ - if (self->last) { - PyObject *tuple = PyTuple_New(2); - Py_INCREF(self->last->key); - PyTuple_SET_ITEM(tuple, 0, self->last->key); - Py_INCREF(self->last->value); - PyTuple_SET_ITEM(tuple, 1, self->last->value); - return tuple; - } - else Py_RETURN_NONE; -} - -static PyObject *LRU_keys(LRU *self) { return collect(self, get_key); } - -static PyObject *get_value(Node *node) -{ - Py_INCREF(node->value); - return node->value; -} - -static PyObject *LRU_values(LRU *self) { return collect(self, get_value); } - - -static PyObject *get_item(Node *node) -{ - PyObject *tuple = PyTuple_New(2); - Py_INCREF(node->key); - PyTuple_SET_ITEM(tuple, 0, node->key); - Py_INCREF(node->value); - PyTuple_SET_ITEM(tuple, 1, node->value); - return tuple; -} - -static PyObject *LRU_items(LRU *self) { return collect(self, get_item); } - -static PyObject *LRU_set_size(LRU *self, PyObject *args, PyObject *kwds) -{ - Py_ssize_t newSize; - if (!PyArg_ParseTuple(args, "n", &newSize)) return NULL; - - if (newSize < 0) { - PyErr_SetString(PyExc_ValueError, "Size should be a positive number"); - return NULL; - } - while (lru_length(self) > newSize) lru_delete_last(self); - - self->size = newSize; - Py_RETURN_NONE; -} - -static PyObject *LRU_clear(LRU *self) -{ - Node *c = self->first; - - while (c) { - Node* n = c; - c = c->next; - lru_remove_node(self, n); - } - PyDict_Clear(self->dict); - - Py_RETURN_NONE; -} - - -static PyObject *LRU_get_size(LRU *self) { return Py_BuildValue("i", self->size); } -/* Hack to implement "key in lru" */ -static PySequenceMethods lru_as_sequence = { - 0, /* sq_length */ - 0, /* sq_concat */ - 0, /* sq_repeat */ - 0, /* sq_item */ - 0, /* sq_slice */ - 0, /* sq_ass_item */ - 0, /* sq_ass_slice */ - (objobjproc) LRU_seq_contains, /* sq_contains */ - 0, /* sq_inplace_concat */ - 0, /* sq_inplace_repeat */ -}; -static PyMethodDef LRU_methods[] = { - {"__contains__", (PyCFunction)LRU_contains_key, METH_O | METH_COEXIST, - PyDoc_STR("L.__contains__(key) -> Check if key is there in L")}, - {"keys", (PyCFunction)LRU_keys, METH_NOARGS, - PyDoc_STR("L.keys() -> list of L's keys in MRU order")}, - {"values", (PyCFunction)LRU_values, METH_NOARGS, - PyDoc_STR("L.values() -> list of L's values in MRU order")}, - {"items", (PyCFunction)LRU_items, METH_NOARGS, - PyDoc_STR("L.items() -> list of L's items (key,value) in MRU order")}, - {"has_key", (PyCFunction)LRU_contains, METH_VARARGS, - PyDoc_STR("L.has_key(key) -> Check if key is there in L")}, - {"get", (PyCFunction)LRU_get, METH_VARARGS, - PyDoc_STR("L.get(key, instead) -> If L has key return its value, otherwise instead")}, - {"delete", (PyCFunction)LRU_delete, METH_VARARGS, - PyDoc_STR("L.get(key, instead) -> If L has key return its value, otherwise instead")}, - {"set_size", (PyCFunction)LRU_set_size, METH_VARARGS, - PyDoc_STR("L.set_size() -> set size of LRU")}, - {"get_size", (PyCFunction)LRU_get_size, METH_NOARGS, - PyDoc_STR("L.get_size() -> get size of LRU")}, - {"clear", (PyCFunction)LRU_clear, METH_NOARGS, - PyDoc_STR("L.clear() -> clear LRU")}, - {"peek_first_item", (PyCFunction)LRU_peek_first_item, METH_NOARGS, - PyDoc_STR("L.peek_first_item() -> returns the MRU item (key,value) without changing key order")}, - {"peek_last_item", (PyCFunction)LRU_peek_last_item, METH_NOARGS, - PyDoc_STR("L.peek_last_item() -> returns the LRU item (key,value) without changing key order")}, - {"pop", (PyCFunction)LRU_pop, METH_NOARGS, - PyDoc_STR("L.pop() -> returns the LRU item (key,value) without changing key order")}, - {"update", (PyCFunction)LRU_update, METH_VARARGS | METH_KEYWORDS, - PyDoc_STR("L.update() -> update value for key in LRU")}, - {"put", (PyCFunction)LRU_put, METH_VARARGS | METH_KEYWORDS, - PyDoc_STR("L.append() -> append value for key in LRU")}, - {"append", (PyCFunction)LRU_append, METH_VARARGS | METH_KEYWORDS, - PyDoc_STR("L.append() -> append value for key in LRU")}, - {NULL, NULL}, -}; -static PyObject*LRU_repr(LRU* self) { return PyObject_Repr(self->dict); } -static int LRU_init(LRU *self, PyObject *args, PyObject *kwds) -{ + +static int cache_init(CACHE *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"size", NULL}; if (!PyArg_ParseTupleAndKeywords(args, kwds, "|n", kwlist, &self->size)) self->size = 0; - if ((Py_ssize_t)self->size < 0) { PyErr_SetString(PyExc_ValueError, "Size should be a positive number"); return -1; @@ -632,43 +552,129 @@ static int LRU_init(LRU *self, PyObject *args, PyObject *kwds) return 0; } -static void -LRU_dealloc(LRU *self) -{ +static void cache_dealloc(CACHE *self) { if (self->dict) { - LRU_clear(self); + CACHE_clear(self); Py_DECREF(self->dict); } PyObject_Del((PyObject*)self); } -PyDoc_STRVAR(lru_doc, -"LRU(size) -> new LRU dict that can store up to size elements\n" -"An LRU dict behaves like a standard dict, except that it stores only fixed\n" -"set of elements. Once the size overflows, it evicts least recently used\n" -"items. \n\n" -"Eg:\n" -">>> l = LRU(3)\n" -">>> for i in range(5):\n" -">>> l[i] = str(i)\n" -">>> l.keys()\n" -"[2,3,4]\n\n" -"Note: An LRU(n) can be thought of as a dict that will have the most\n" -"recently accessed n items.\n"); +static PySequenceMethods cache_as_sequence = { + 0, /* sq_length */ + 0, /* sq_concat */ + 0, /* sq_repeat */ + 0, /* sq_item */ + 0, /* sq_slice */ + 0, /* sq_ass_item */ + 0, /* sq_ass_slice */ + (objobjproc) cache_seq_contains,/* sq_contains */ + 0, /* sq_inplace_concat */ + 0, /* sq_inplace_repeat */ +}; -static PyTypeObject LRUType = { +static PyObject*cache_repr(CACHE* self) {return PyObject_Repr(self->dict);} + + + + + + + + +static PyMethodDef MRU_methods[] = { + {"__contains__", (PyCFunction)cache_contains_key, METH_O | METH_COEXIST, "L.__contains__(key) -> Check if key is there in L"}, + {"keys", (PyCFunction)CACHE_keys, METH_NOARGS, "L.keys() -> list of L's keys in MRU order"}, + {"values", (PyCFunction)CACHE_values, METH_NOARGS, "L.values() -> list of L's values in MRU order"}, + {"items", (PyCFunction)CACHE_items, METH_NOARGS, "L.items() -> list of L's items (key,value) in MRU order"}, + {"has_key", (PyCFunction)CACHE_contains, METH_VARARGS, "L.has_key(key) -> Check if key is there in L"}, + {"get", (PyCFunction)MRU_get, METH_VARARGS, "L.get(key, instead) -> If L has key return its value, otherwise instead"}, + {"delete", (PyCFunction)CACHE_delete, METH_VARARGS, "L.delete(key, instead) -> If L has key return its value, otherwise instead"}, + {"set_size", (PyCFunction)CACHE_set_size, METH_VARARGS, "L.set_size() -> set size of MRU"}, + {"get_size", (PyCFunction)CACHE_get_size, METH_NOARGS, "L.get_size() -> get size of MRU"}, + {"clear", (PyCFunction)CACHE_clear, METH_NOARGS, "L.clear() -> clear MRU"}, + {"peek_first_item", (PyCFunction)CACHE_peek_first_item, METH_NOARGS, "L.peek_first_item() -> returns the MRU item (key,value) without changing key order"}, + {"peek_last_item", (PyCFunction)CACHE_peek_last_item, METH_NOARGS, "L.peek_last_item() -> returns the MRU item (key,value) without changing key order"}, + {"pop", (PyCFunction)CACHE_pop, METH_NOARGS, "L.pop() -> returns the MRU item (key,value) without changing key order"}, + {"update", (PyCFunction)MRU_update, METH_VARARGS | METH_KEYWORDS, "L.update() -> update value for key in MRU"}, + {"append", (PyCFunction)CACHE_append, METH_VARARGS | METH_KEYWORDS, "L.append() -> append value for key in MRU"}, + {NULL, NULL}, +}; + +static PyMethodDef LRU_methods[] = { + {"__contains__", (PyCFunction)cache_contains_key, METH_O | METH_COEXIST, "L.__contains__(key) -> Check if key is there in L"}, + {"keys", (PyCFunction)CACHE_keys, METH_NOARGS, "L.keys() -> list of L's keys in LRU order"}, + {"values", (PyCFunction)CACHE_values, METH_NOARGS, "L.values() -> list of L's values in LRU order"}, + {"items", (PyCFunction)CACHE_items, METH_NOARGS, "L.items() -> list of L's items (key,value) in LRU order"}, + {"has_key", (PyCFunction)CACHE_contains, METH_VARARGS, "L.has_key(key) -> Check if key is there in L"}, + {"get", (PyCFunction)LRU_get, METH_VARARGS, "L.get(key, instead) -> If L has key return its value, otherwise instead"}, + {"delete", (PyCFunction)CACHE_delete, METH_VARARGS, "L.delete(key, instead) -> If L has key return its value, otherwise instead"}, + {"set_size", (PyCFunction)CACHE_set_size, METH_VARARGS, "L.set_size() -> set size of LRU"}, + {"get_size", (PyCFunction)CACHE_get_size, METH_NOARGS, "L.get_size() -> get size of LRU"}, + {"clear", (PyCFunction)CACHE_clear, METH_NOARGS, "L.clear() -> clear LRU"}, + {"peek_first_item", (PyCFunction)CACHE_peek_first_item, METH_NOARGS, "L.peek_first_item() -> returns the LRU item (key,value) without changing key order"}, + {"peek_last_item", (PyCFunction)CACHE_peek_last_item, METH_NOARGS, "L.peek_last_item() -> returns the LRU item (key,value) without changing key order"}, + {"pop", (PyCFunction)CACHE_pop, METH_NOARGS, "L.pop() -> returns the LRU item (key,value) without changing key order"}, + {"update", (PyCFunction)LRU_update, METH_VARARGS | METH_KEYWORDS, "L.update() -> update value for key in LRU"}, + {"append", (PyCFunction)CACHE_append, METH_VARARGS | METH_KEYWORDS, "L.append() -> append value for key in LRU"}, + {NULL, NULL}, +}; + + +static PyTypeObject MRUType = { PyVarObject_HEAD_INIT(NULL, 0) - "_lru.LRU", /* tp_name */ - sizeof(LRU), /* tp_basicsize */ + "_cache.MRU", /* tp_name */ + sizeof(CACHE), /* tp_basicsize */ 0, /* tp_itemsize */ - (destructor)LRU_dealloc, /* tp_dealloc */ + (destructor)cache_dealloc, /* tp_dealloc */ 0, /* tp_print */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_compare */ - (reprfunc)LRU_repr, /* tp_repr */ + (reprfunc)cache_repr, /* tp_repr */ 0, /* tp_as_number */ - &lru_as_sequence, /* tp_as_sequence */ + &cache_as_sequence, /* tp_as_sequence */ + &MRU_as_mapping, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + 0, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + MRU_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)cache_init, /* tp_init */ + 0, /* tp_alloc */ + 0, /* tp_new */ +}; + +static PyTypeObject LRUType = { + PyVarObject_HEAD_INIT(NULL, 0) + "_cache.LRU", /* tp_name */ + sizeof(CACHE), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)cache_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + (reprfunc)cache_repr, /* tp_repr */ + 0, /* tp_as_number */ + &cache_as_sequence, /* tp_as_sequence */ &LRU_as_mapping, /* tp_as_mapping */ 0, /* tp_hash */ 0, /* tp_call */ @@ -677,7 +683,7 @@ static PyTypeObject LRUType = { 0, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT, /* tp_flags */ - lru_doc, /* tp_doc */ + 0, /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ @@ -692,16 +698,19 @@ static PyTypeObject LRUType = { 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ - (initproc)LRU_init, /* tp_init */ + (initproc)cache_init, /* tp_init */ 0, /* tp_alloc */ 0, /* tp_new */ }; + + + static struct PyModuleDef moduledef = { PyModuleDef_HEAD_INIT, - "_lru", /* m_name */ - lru_doc, /* m_doc */ + "cache_strategies", /* m_name */ + "cache strategies", /* m_doc */ -1, /* m_size */ NULL, /* m_methods */ NULL, /* m_reload */ @@ -715,14 +724,18 @@ static PyObject *moduleinit(void) { PyObject *m; NodeType.tp_new = PyType_GenericNew; if (PyType_Ready(&NodeType) < 0) return NULL; + MRUType.tp_new = PyType_GenericNew; + if (PyType_Ready(&MRUType) < 0) return NULL; LRUType.tp_new = PyType_GenericNew; - if (PyType_Ready(&LRUType) < 0) return NULL; + if (PyType_Ready(&MRUType) < 0) return NULL; m = PyModule_Create(&moduledef); if (m == NULL) return NULL; Py_INCREF(&NodeType); + Py_INCREF(&MRUType); Py_INCREF(&LRUType); +// PyModule_AddObject(m, "MRU", (PyObject *) &MRUType); PyModule_AddObject(m, "LRU", (PyObject *) &LRUType); return m; } -PyMODINIT_FUNC PyInit__lru(void) { return moduleinit(); } \ No newline at end of file +PyMODINIT_FUNC PyInit_cache_strategies(void) {return moduleinit();} diff --git a/pybtc/connector/block_loader.py b/pybtc/connector/block_loader.py index 8cd2499..83152aa 100644 --- a/pybtc/connector/block_loader.py +++ b/pybtc/connector/block_loader.py @@ -12,7 +12,7 @@ import traceback from pybtc.connector.utils import decode_block_tx import _pickle as pickle -from pybtc import LRU as MRU +from pybtc import MRU class BlockLoader: @@ -241,18 +241,16 @@ class Worker: if y["result"] is not None: block = decode_block_tx(y["result"]) for z in block["rawTx"]: - if not block["rawTx"][z]["coinbase"]: - for i in block["rawTx"][z]["vIn"]: - inp = block["rawTx"][z]["vIn"][i] - outpoint = b"".join((inp["txId"], int_to_bytes(inp["vOut"]))) - try: - r = self.coins.delete(outpoint) - self.log.critical(str(r)) - block["rawTx"][z]["vIn"][i]["_c_"] = r - t += 1 - self.destroyed_coins[r[0]] = True - except: - pass + for i in block["rawTx"][z]["vIn"]: + inp = block["rawTx"][z]["vIn"][i] + outpoint = b"".join((inp["txId"], int_to_bytes(inp["vOut"]))) + try: + r = self.coins.delete(outpoint) + block["rawTx"][z]["vIn"][i]["_c_"] = r + t += 1 + self.destroyed_coins[r[0]] = True + except: + pass for i in block["rawTx"][z]["vOut"]: o = b"".join((block["rawTx"][z]["txId"], int_to_bytes(i))) pointer = (x << 42) + (z << 21) + i @@ -262,7 +260,6 @@ class Worker: address = b"".join((bytes([block["rawTx"][z]["vOut"][i]["nType"]]), block["rawTx"][z]["vOut"][i]["addressHash"])) self.coins[o] = (pointer, block["rawTx"][z]["vOut"][i]["value"], address) - self.log.critical(str(o)) blocks[x] = block if blocks: blocks[x]["checkpoint"] = x @@ -272,7 +269,6 @@ class Worker: try: pointer = (x << 42) + (y << 21) + i r = self.destroyed_coins.delete(pointer) - self.log.critical(str(r)) blocks[x]["rawTx"][y]["vOut"][i]["_s_"] = r except: pass diff --git a/pybtc/connector/utxo.py b/pybtc/connector/utxo.py index 5ff272c..7dd680c 100644 --- a/pybtc/connector/utxo.py +++ b/pybtc/connector/utxo.py @@ -1,7 +1,7 @@ from pybtc import int_to_c_int, c_int_to_int, c_int_len import asyncio from collections import OrderedDict -from pybtc import LRU as MRU +from pybtc import MRU class UTXO(): def __init__(self, db_pool, loop, log, cache_size): diff --git a/setup.py b/setup.py index 3977a68..7026e00 100644 --- a/setup.py +++ b/setup.py @@ -148,7 +148,7 @@ setup(name='pybtc', 'bdist_wheel': bdist_wheel }, distclass=Distribution, - ext_modules=[Extension("_lru", ["pybtc/_lru/lru.c"]), + ext_modules=[Extension("cache_strategies", ["pybtc/cache_strategies/cache.c"]), Extension("_secp256k1", ["pybtc/_secp256k1/module_secp256k1.c"], include_dirs=["libsecp256k1/include/", "libsecp256k1/src/"]), Extension("_crypto",