From aa101a16e8abda0e82fd4c9696cb37ea1603ebe8 Mon Sep 17 00:00:00 2001 From: Christopher Jeffrey Date: Mon, 7 Mar 2016 19:00:03 -0800 Subject: [PATCH] misc improvements. --- lib/bcoin.js | 15 ++++++++ lib/bcoin/blockdb.js | 16 ++++----- lib/bcoin/chain.js | 12 ++----- lib/bcoin/chaindb.js | 11 ++---- lib/bcoin/lru.js | 82 ++++++------------------------------------- lib/bcoin/node.js | 4 +-- lib/bcoin/profiler.js | 4 +++ lib/bcoin/utils.js | 9 +++-- lib/bcoin/walletdb.js | 10 +++--- 9 files changed, 55 insertions(+), 108 deletions(-) diff --git a/lib/bcoin.js b/lib/bcoin.js index 63e415a4..c450d2fd 100644 --- a/lib/bcoin.js +++ b/lib/bcoin.js @@ -15,6 +15,7 @@ bcoin.prefix = process.env.BCOIN_PREFIX || process.env.HOME + '/.bcoin'; bcoin.debug = +process.env.BCOIN_DEBUG === 1; bcoin.debugFile = +process.env.BCOIN_DEBUGFILE !== 0; bcoin.profile = +process.env.BCOIN_PROFILE === 1; +bcoin.fresh = +process.env.BCOIN_FRESH === 1; bcoin.ensurePrefix = function ensurePrefix() { if (!bcoin.fs) @@ -25,6 +26,9 @@ bcoin.ensurePrefix = function ensurePrefix() { bcoin._ensured = true; + if (bcoin.fresh && bcoin.prefix.indexOf('bcoin') !== -1) + bcoin.rimraf(bcoin.prefix); + try { bcoin.fs.statSync(bcoin.prefix); } catch (e) { @@ -32,6 +36,17 @@ bcoin.ensurePrefix = function ensurePrefix() { } }; +bcoin.rimraf = function rimraf(file) { + if (!bcoin.cp) + return; + + assert(typeof file === 'string'); + assert(file !== '/'); + assert(file !== process.env.HOME); + + bcoin.cp.execFileSync('rm', ['-rf', file], { stdio: 'ignore' }); +}; + bcoin.bn = require('bn.js'); bcoin.elliptic = require('elliptic'); diff --git a/lib/bcoin/blockdb.js b/lib/bcoin/blockdb.js index bc4be15b..ff4ac9a8 100644 --- a/lib/bcoin/blockdb.js +++ b/lib/bcoin/blockdb.js @@ -44,13 +44,10 @@ function BlockDB(node, options) { this.node = node; this.cache = { - unspent: new bcoin.lru(32 * 1024 * 1024, function() { return 80; }), - tx: new bcoin.lru(32 * 1024 * 1024, function(key, tx) { return tx.getSize(); }) + unspent: new bcoin.lru(200000), + tx: new bcoin.lru(50000) }; - if (+process.env.BCOIN_FRESH === 1 && bcoin.cp) - bcoin.cp.execFileSync('rm', ['-rf', this.file], { stdio: 'ignore' }); - this.db = new levelup(this.file, { keyEncoding: 'ascii', valueEncoding: 'binary', @@ -59,9 +56,7 @@ function BlockDB(node, options) { compression: true, cacheSize: 16 * 1024 * 1024, writeBufferSize: 8 * 1024 * 1024, - // blockSize: 4 * 1024, maxOpenFiles: 8192, - // blockRestartInterval: 16, db: bcoin.isBrowser ? require('level-js') : require('level' + 'down') @@ -804,7 +799,12 @@ BlockDB.prototype.getBlock = function getBlock(hash, callback) { return callback(err); } - block = bcoin.block.fromCompact(data); + try { + block = bcoin.block.fromCompact(data); + } catch (e) { + return callback(e); + } + block.txs = []; utils.forEach(block.hashes, function(hash, next, i) { diff --git a/lib/bcoin/chain.js b/lib/bcoin/chain.js index 36373803..b3025358 100644 --- a/lib/bcoin/chain.js +++ b/lib/bcoin/chain.js @@ -163,14 +163,6 @@ Chain.prototype._init = function _init() { self.tip = tip; self.height = tip.height; - // Start fsyncing writes once we're no - // longer dealing with historical data. - if (tip.height >= network.checkpoints.lastHeight) { - self.db.fsync = true; - if (self.blockdb) - self.blockdb.fsync = true; - } - self.loading = false; self.emit('load'); }); @@ -991,7 +983,7 @@ Chain.prototype._setBestChain = function _setBestChain(entry, block, callback) { // Start fsyncing writes once we're no // longer dealing with historical data. - if (entry.height >= network.checkpoints.lastHeight) { + if (self.isFull()) { self.db.fsync = true; if (self.blockdb) self.blockdb.fsync = true; @@ -1393,7 +1385,7 @@ Chain.prototype.add = function add(initial, peer, callback, force) { self.total += total; // Take heap snapshot for debugging. - if (self.total % 10 === 0) + if (self.total % 20 === 0) bcoin.profiler.snapshot(); utils.nextTick(function() { diff --git a/lib/bcoin/chaindb.js b/lib/bcoin/chaindb.js index cf44f5ca..da542968 100644 --- a/lib/bcoin/chaindb.js +++ b/lib/bcoin/chaindb.js @@ -65,10 +65,8 @@ function ChainDB(node, chain, options) { else this._cacheWindow = network.block.majorityWindow + 1; - this.cacheHash = new NullCache(this._cacheWindow * 200); // (not hashcash) - this.cacheHeight = new NullCache(this._cacheWindow * 200); - // this.cacheHash = new bcoin.lru(this._cacheWindow, function() { return 1; }); // (not hashcash) - // this.cacheHeight = new bcoin.lru(this._cacheWindow, function() { return 1; }); + this.cacheHash = new bcoin.lru(this._cacheWindow); + this.cacheHeight = new bcoin.lru(this._cacheWindow); this._init(); } @@ -80,9 +78,6 @@ ChainDB.prototype._init = function _init() { bcoin.ensurePrefix(); - if (+process.env.BCOIN_FRESH === 1 && bcoin.cp) - bcoin.cp.execFileSync('rm', ['-rf', this.file], { stdio: 'ignore' }); - this.db = new levelup(this.file, { keyEncoding: 'ascii', valueEncoding: 'binary', @@ -91,9 +86,7 @@ ChainDB.prototype._init = function _init() { compression: false, cacheSize: 16 * 1024 * 1024, writeBufferSize: 8 * 1024 * 1024, - // blockSize: 4 * 1024, maxOpenFiles: 8192, - // blockRestartInterval: 16, db: bcoin.isBrowser ? require('level-js') : require('level' + 'down') diff --git a/lib/bcoin/lru.js b/lib/bcoin/lru.js index 0b3a94ac..e6acae4b 100644 --- a/lib/bcoin/lru.js +++ b/lib/bcoin/lru.js @@ -37,22 +37,13 @@ LRU.prototype._getSize = function _getSize(item) { if (typeof item.value === 'number') return keySize + 4; - if (item.value._raw) - return keySize + item.value._raw.length; - - if (item.value.getSize) - return keySize + item.value.getSize(); - - if (typeof item.value._size === 'number') - return keySize + item.value._size; - if (typeof item.value === 'string') return keySize + item.value.length * 2; if (typeof item.value.length === 'number') return keySize + item.value.length; - return keySize + 1; + return 1; }; LRU.prototype._compact = function _compact() { @@ -220,7 +211,7 @@ LRU.prototype._removeList = function removeList(item) { item.next = null; }; -LRU.prototype._keys = function _keys() { +LRU.prototype.keys = function keys() { var keys = []; var item; @@ -237,66 +228,15 @@ LRU.prototype._keys = function _keys() { return keys; }; -var a1 = '1'; -var a2 = '2'; -var a3 = '3'; -var a4 = '4'; -var a5 = '5'; -var a6 = '6'; -var lru = new LRU(4, function() { return 1; }); -lru.set('a1', a1); -assert(lru.get('a1') === '1'); -assert(lru.size === 1); -assert(lru.head.key === 'a1' && lru.tail.key === 'a1' && !lru.head.prev && !lru.head.next); -lru._keys(); -// console.log(lru._keys()); -lru.set('a2', a2); -assert(lru.get('a2') === '2'); -assert(lru.size === 2); -assert(lru.head.key === 'a1' && lru.tail.key === 'a2' - && !lru.head.prev && lru.head.next.key === 'a2' - && !lru.tail.next && lru.tail.prev.key === 'a1'); -lru._keys(); -// console.log(lru._keys()); -lru.set('a3', a3); -assert(lru.get('a3') === '3'); -assert(lru.size === 3); -lru._keys(); -// console.log(lru._keys()); -lru.set('a3', a3); -assert(lru.get('a3') === '3'); -assert(lru.size === 3); -lru._keys(); -// console.log(lru._keys()); -lru.set('a4', a4); -assert(lru.get('a4') === '4'); -assert(lru.size === 4); -lru._keys(); -// console.log(lru._keys()); -assert(lru.get('a1')); -lru.remove('a1'); -lru._keys(); -// console.log(lru._keys()); -var _a3 = lru.head.next; -assert(_a3.key === 'a3'); -assert(_a3.prev === lru.head && _a3.next === lru.tail); -assert(lru.head.key === 'a2' && lru.tail.key === 'a4' - && !lru.head.prev && lru.head.next.key === 'a3' - && !lru.tail.next && lru.tail.prev.key === 'a3'); -lru.set('a5', a5); -assert(lru.get('a5') === '5'); -assert(lru.size === 4); -assert(!lru.get('a1')); -lru._keys(); -// console.log(lru._keys()); -lru.get('a2'); -lru._keys(); -lru.set('a6', a6); -assert(lru.get('a6') === '6'); -assert(lru.size === 4); -assert(!lru.get('a3')); -lru._keys(); -// console.log(lru._keys()); +LRU.prototype.iterator = function iterator() { + return { + item: { next: this.head }, + next: function() { + this.item = this.item.next; + return this.item; + } + }; +}; /** * Expose diff --git a/lib/bcoin/node.js b/lib/bcoin/node.js index 3fc7ac70..24ab1876 100644 --- a/lib/bcoin/node.js +++ b/lib/bcoin/node.js @@ -48,7 +48,7 @@ Fullnode.prototype._init = function _init() { // used for tx retrieval. this.blockdb = new bcoin.blockdb(this, { cache: false, - fsync: true + fsync: false }); // Mempool needs access to blockdb. @@ -59,7 +59,7 @@ Fullnode.prototype._init = function _init() { // Chain needs access to blockdb. this.chain = new bcoin.chain(this, { preload: false, - fsync: true + fsync: false }); // Pool needs access to the chain. diff --git a/lib/bcoin/profiler.js b/lib/bcoin/profiler.js index c1022960..b748e6c3 100644 --- a/lib/bcoin/profiler.js +++ b/lib/bcoin/profiler.js @@ -161,6 +161,7 @@ exports.takeSnapshot = function takeSnapshot(name) { }; exports.snapshot = function snapshot(name, callback) { + var mem = process.memoryUsage(); var snapshot; if (typeof name === 'function') { @@ -168,6 +169,9 @@ exports.snapshot = function snapshot(name, callback) { name = null; } + utils.debug('Memory: rss=%dmb, heap=%dmb', + utils.mb(mem.rss), utils.mb(mem.heapUsed)); + if (!profiler) return callback ? utils.nextTick(callback) : null; diff --git a/lib/bcoin/utils.js b/lib/bcoin/utils.js index c3fca621..e43288eb 100644 --- a/lib/bcoin/utils.js +++ b/lib/bcoin/utils.js @@ -1741,20 +1741,25 @@ function SyncBatch(db) { } SyncBatch.prototype.put = function put(key, value) { - this.ops.push({ type: 'put', key: key, value: value, sync: true }); + assert(this.ops, 'Batch was already written.'); + this.ops.push({ type: 'put', key: key, value: value }); }; SyncBatch.prototype.del = function del(key) { - this.ops.push({ type: 'del', key: key, sync: true }); + assert(this.ops, 'Batch was already written.'); + this.ops.push({ type: 'del', key: key }); }; SyncBatch.prototype.write = function write(callback) { + assert(this.ops, 'Batch was already written.'); this.db.batch(this.ops, { sync: true }, callback); this.ops.length = 0; delete this.ops; + delete this.db; }; SyncBatch.prototype.clear = function clear() { + assert(this.ops, 'Batch was already written.'); this.ops.length = 0; }; diff --git a/lib/bcoin/walletdb.js b/lib/bcoin/walletdb.js index 76d4efc8..03f7e120 100644 --- a/lib/bcoin/walletdb.js +++ b/lib/bcoin/walletdb.js @@ -104,11 +104,9 @@ WalletDB.prototype._init = function _init() { createIfMissing: true, errorIfExists: false, compression: true, - cacheSize: 1 * 1024 * 1024, - writeBufferSize: 1 * 1024 * 1024, - // blockSize: 4 * 1024, + cacheSize: 4 * 1024 * 1024, + writeBufferSize: 4 * 1024 * 1024, maxOpenFiles: 1024, - // blockRestartInterval: 16, db: bcoin.isBrowser ? require('level-js') : require('level' + 'down') @@ -576,8 +574,8 @@ WalletDB.prototype.removeBlock = function removeBlock(block, callback) { callback = utils.ensure(callback); - utils.forEach(block.txs, function(tx, next) { - self.tx.unconfirm(tx.hash('hex')); + utils.forEachSerial(block.txs, function(tx, next) { + self.tx.unconfirm(tx.hash('hex'), next); }, callback); };