From c958b3af494094aa9b99e7c153344444c0b51cb2 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Mon, 16 Jan 2017 22:03:45 +0900 Subject: [PATCH 1/5] Remove LMDB Performance of history writes was very poor even with the code rewritten to accommodate the environment concept (see db_abstraction branch) --- .travis.yml | 1 - docs/ENVIRONMENT.rst | 8 ++-- samples/systemd/electrumx.conf | 2 +- server/storage.py | 70 +--------------------------------- 4 files changed, 5 insertions(+), 76 deletions(-) diff --git a/.travis.yml b/.travis.yml index 54f6b7d..a519091 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,6 @@ python: # command to install dependencies install: - pip install aiohttp - - pip install lmdb - pip install plyvel - pip install pyrocksdb - pip install pytest-cov diff --git a/docs/ENVIRONMENT.rst b/docs/ENVIRONMENT.rst index 6aa695f..24e4993 100644 --- a/docs/ENVIRONMENT.rst +++ b/docs/ENVIRONMENT.rst @@ -66,11 +66,9 @@ These environment variables are optional: * **DB_ENGINE** Database engine for the UTXO and history database. The default is - `leveldb`. Supported alternatives are `rocksdb` and `lmdb`. You - will need to install the appropriate python package for your engine. - The value is not case sensitive. Note that the current way - ElectrumX uses LMDB results in poor performance. I intend to improve - this. + `leveldb`. The other alternative is `rocksdb`. You will need to + install the appropriate python package for your engine. The value + is not case sensitive. * **REORG_LIMIT** diff --git a/samples/systemd/electrumx.conf b/samples/systemd/electrumx.conf index c475301..64eba7a 100644 --- a/samples/systemd/electrumx.conf +++ b/samples/systemd/electrumx.conf @@ -44,7 +44,7 @@ #COIN = Bitcoin # lib/coins.py #NET = mainnet # lib/coins.py #DB_ENGINE = leveldb - #leveldb, rocksdb, lmdb (You'll need to install appropriate python packages) + #leveldb, rocksdb (You'll need to install appropriate python packages) #REORG_LIMIT = 200 #maximum number of blocks to be able to handle in a chain diff --git a/server/storage.py b/server/storage.py index a2b5d7f..40d74b6 100644 --- a/server/storage.py +++ b/server/storage.py @@ -5,10 +5,7 @@ # See the file "LICENCE" for information about the copyright # and warranty status of this software. -'''Backend database abstraction. - -The abstraction needs to be improved to not heavily penalise LMDB. -''' +'''Backend database abstraction.''' import os from functools import partial @@ -161,68 +158,3 @@ class RocksDB(Storage): def iterator(self, prefix=b'', reverse=False): return RocksDB.Iterator(self.db, prefix, reverse) - - -class LMDB(Storage): - '''RocksDB database engine.''' - - @classmethod - def import_module(cls): - import lmdb - cls.module = lmdb - - def open(self, name, create): - # I don't see anything equivalent to max_open_files for for_sync - self.env = LMDB.module.Environment('.', subdir=True, create=create, - max_dbs=32, map_size=5 * 10 ** 10) - self.db = self.env.open_db(create=create) - - def close(self): - self.env.close() - - def get(self, key): - with self.env.begin(db=self.db) as tx: - return tx.get(key) - - def put(self, key, value): - with self.env.begin(db=self.db, write=True) as tx: - tx.put(key, value) - - def write_batch(self): - return self.env.begin(db=self.db, write=True) - - def iterator(self, prefix=b'', reverse=False): - return LMDB.Iterator(self.db, self.env, prefix, reverse) - - class Iterator: - def __init__(self, db, env, prefix, reverse): - self.transaction = env.begin(db=db) - self.transaction.__enter__() - self.db = db - self.prefix = prefix - self.reverse = reverse - self._stop = False - - def __iter__(self): - self.iterator = LMDB.module.Cursor(self.db, self.transaction) - prefix = self.prefix - if self.reverse: - # Go to the first value after the prefix - prefix = increment_byte_string(prefix) - self.iterator.set_range(prefix) - if not self.iterator.key().startswith(self.prefix) and self.reverse: - # Go back to the first item starting with the prefix - self.iterator.prev() - return self - - def __next__(self): - k, v = self.iterator.item() - if not k.startswith(self.prefix) or self._stop: - # We're already ahead of the prefix - self.transaction.__exit__() - raise StopIteration - next = self.iterator.next \ - if not self.reverse else self.iterator.prev - # Stop after the next value if we're at the end of the DB - self._stop = not next() - return k, v From 5abe4faa8d986f119144cb204a6e78059814e52d Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Tue, 17 Jan 2017 07:23:58 +0900 Subject: [PATCH 2/5] JSON results are returned with HTTP status 500 --- server/controller.py | 2 +- server/daemon.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/server/controller.py b/server/controller.py index e14291a..76aa455 100644 --- a/server/controller.py +++ b/server/controller.py @@ -22,7 +22,7 @@ from lib.jsonrpc import JSONRPC, RPCError, RequestBase from lib.hash import sha256, double_sha256, hash_to_str, hex_str_to_hash import lib.util as util from server.block_processor import BlockProcessor -from server.daemon import Daemon +from server.daemon import Daemon, DaemonError from server.irc import IRC from server.session import LocalRPC, ElectrumX from server.mempool import MemPool diff --git a/server/daemon.py b/server/daemon.py index 4099958..8aacb9f 100644 --- a/server/daemon.py +++ b/server/daemon.py @@ -70,7 +70,10 @@ class Daemon(util.LoggedClass): async with self.workqueue_semaphore: url = self.urls[self.url_index] async with aiohttp.post(url, data=data) as resp: - if resp.status == 200: + # If bitcoind can't find a tx, for some reason + # it returns 500 but fills out the JSON. + # Should still return 200 IMO. + if resp.status in (200, 500): if self.prior_msg: self.logger.info('connection restored') result = processor(await resp.json()) From 5f9e33745760fddc8d9fe475e7752e1cd51b4506 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Tue, 17 Jan 2017 07:34:08 +0900 Subject: [PATCH 3/5] Improve RocksDB iterator --- lib/util.py | 21 ++++-------- server/storage.py | 87 +++++++++++++++++++++++++---------------------- 2 files changed, 54 insertions(+), 54 deletions(-) diff --git a/lib/util.py b/lib/util.py index c60a0d3..cc755b8 100644 --- a/lib/util.py +++ b/lib/util.py @@ -128,20 +128,13 @@ def int_to_bytes(value): def increment_byte_string(bs): - bs = bytearray(bs) - incremented = False - for i in reversed(range(len(bs))): - if bs[i] < 0xff: - # This is easy - bs[i] += 1 - incremented = True - break - # Otherwise we need to look at the previous character - bs[i] = 0 - if not incremented: - # This can only happen if all characters are 0xff - bs = bytes([1]) + bs - return bytes(bs) + '''Return the lexicographically next byte string of the same length. + + Return None if there is none (when the input is all 0xff bytes).''' + for n in range(1, len(bs) + 1): + if bs[-n] != 0xff: + return bs[:-n] + bytes([bs[-n] + 1]) + bytes(n - 1) + return None class LogicalFile(object): diff --git a/server/storage.py b/server/storage.py index 40d74b6..cd44746 100644 --- a/server/storage.py +++ b/server/storage.py @@ -115,46 +115,53 @@ class RocksDB(Storage): import gc gc.collect() - class WriteBatch(object): - def __init__(self, db): - self.batch = RocksDB.module.WriteBatch() - self.db = db - - def __enter__(self): - return self.batch - - def __exit__(self, exc_type, exc_val, exc_tb): - if not exc_val: - self.db.write(self.batch) - def write_batch(self): - return RocksDB.WriteBatch(self.db) - - class Iterator(object): - def __init__(self, db, prefix, reverse): - self.it = db.iteritems() - self.reverse = reverse - self.prefix = prefix - # Whether we are at the first item - self.first = True - - def __iter__(self): - prefix = self.prefix - if self.reverse: - prefix = increment_byte_string(prefix) - self.it = reversed(self.it) - self.it.seek(prefix) - return self - - def __next__(self): - k, v = self.it.__next__() - if self.first and self.reverse and not k.startswith(self.prefix): - k, v = self.it.__next__() - self.first = False - if not k.startswith(self.prefix): - # We're already ahead of the prefix - raise StopIteration - return k, v + return RocksDBWriteBatch(self.db) def iterator(self, prefix=b'', reverse=False): - return RocksDB.Iterator(self.db, prefix, reverse) + return RocksDBIterator(self.db, prefix, reverse) + + +class RocksDBWriteBatch(object): + '''A write batch for RocksDB.''' + + def __init__(self, db): + self.batch = RocksDB.module.WriteBatch() + self.db = db + + def __enter__(self): + return self.batch + + def __exit__(self, exc_type, exc_val, exc_tb): + if not exc_val: + self.db.write(self.batch) + + +class RocksDBIterator(object): + '''An iterator for RocksDB.''' + + def __init__(self, db, prefix, reverse): + self.prefix = prefix + if reverse: + self.iterator = reversed(db.iteritems()) + nxt_prefix = util.increment_byte_string(prefix) + if nxt_prefix: + self.iterator.seek(nxt_prefix) + try: + next(self.iterator) + except StopIteration: + self.iterator.seek(nxt_prefix) + else: + self.iterator.seek_to_last() + else: + self.iterator = db.iteritems() + self.iterator.seek(prefix) + + def __iter__(self): + return self + + def __next__(self): + k, v = next(self.iterator) + if not k.startswith(self.prefix): + raise StopIteration + return k, v From f05dd6c2a3f3075495b164af30e6f5caf90ba196 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Tue, 17 Jan 2017 07:37:10 +0900 Subject: [PATCH 4/5] Use snappy compression Seems to make ~4.5% faster (5281 tx/s -> 5516 tx/s) Shrinks UTXO DB by ~15% and hist DB by ~5% --- server/storage.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/server/storage.py b/server/storage.py index cd44746..fedcfe3 100644 --- a/server/storage.py +++ b/server/storage.py @@ -77,8 +77,9 @@ class LevelDB(Storage): def open(self, name, create): mof = 512 if self.for_sync else 128 + # Use snappy compression (the default) self.db = self.module.DB(name, create_if_missing=create, - max_open_files=mof, compression=None) + max_open_files=mof) self.close = self.db.close self.get = self.db.get self.put = self.db.put @@ -97,11 +98,8 @@ class RocksDB(Storage): def open(self, name, create): mof = 512 if self.for_sync else 128 - compression = "no" - compression = getattr(self.module.CompressionType, - compression + "_compression") + # Use snappy compression (the default) options = self.module.Options(create_if_missing=create, - compression=compression, use_fsync=True, target_file_size_base=33554432, max_open_files=mof) From 0dfdfaba7e270c6bf7bb9ba1a81286ec3f0255e7 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Tue, 17 Jan 2017 08:13:57 +0900 Subject: [PATCH 5/5] Prepare 0.10.7 --- README.rst | 9 +++++++++ server/version.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index a3f2a22..757de75 100644 --- a/README.rst +++ b/README.rst @@ -137,6 +137,15 @@ version prior to the release of 1.0. ChangeLog ========= +Version 0.10.7 +-------------- + +* remove LMDB +* turn on snappy compression for LevelDB and RocksDB; gives smaller DBs and + faster sync +* fix and speed up RocksDB iterator slightly +* expect JSON with bitcoind HTTP status code 500 + Version 0.10.6 -------------- diff --git a/server/version.py b/server/version.py index 83b18bf..b8afe92 100644 --- a/server/version.py +++ b/server/version.py @@ -1 +1 @@ -VERSION = "ElectrumX 0.10.6" +VERSION = "ElectrumX 0.10.7"