diff --git a/lib/bcoin/blockdb.js b/lib/bcoin/blockdb.js index f7115379..b62ee87b 100644 --- a/lib/bcoin/blockdb.js +++ b/lib/bcoin/blockdb.js @@ -729,7 +729,7 @@ BlockDB.prototype._getTXByAddress = function _getTXByAddress(address, callback) } self.data.getAsync(record.size, record.offset, function(err, data) { - var tx, entry; + var tx; if (err) return callback(err); @@ -737,23 +737,36 @@ BlockDB.prototype._getTXByAddress = function _getTXByAddress(address, callback) if (data) { try { tx = bcoin.tx.fromRaw(data); - entry = bcoin.chain.global.db.getSync(record.height); } catch (e) { return callback(e); } - tx.height = record.height; - if (entry) { - tx.ts = entry.ts; - tx.block = entry.hash; - } - txs.push(tx); + return self._getEntry(record.height, function(err, entry) { + if (err) + return callback(err); - if (self.options.cache) - self.cache.tx.set(hash, tx); + tx.height = record.height; - if (self.options.paranoid && tx.hash('hex') !== hash) - return callback(new Error('BlockDB is corrupt. All is lost.')); + if (entry) { + tx.ts = entry.ts; + tx.block = entry.hash; + } + + txs.push(tx); + + if (self.options.cache) + self.cache.tx.set(hash, tx); + + if (self.options.paranoid && tx.hash('hex') !== hash) + return callback(new Error('BlockDB is corrupt. All is lost.')); + + pending--; + + if (done) { + if (!pending) + return callback(null, txs); + } + }); } pending--; @@ -790,7 +803,7 @@ BlockDB.prototype.getTX = function getTX(hash, callback) { record = self.parseOffset(record); self.data.getAsync(record.size, record.offset, function(err, data) { - var tx, entry; + var tx; if (err) return callback(err); @@ -798,18 +811,27 @@ BlockDB.prototype.getTX = function getTX(hash, callback) { if (data) { try { tx = bcoin.tx.fromRaw(data); - entry = bcoin.chain.global.db.getSync(record.height); } catch (e) { return callback(e); } - tx.height = record.height; - if (entry) { - tx.ts = entry.ts; - tx.block = entry.hash; - } - tx._fileOffset = record.offset; - if (self.options.paranoid && tx.hash('hex') !== hash) - return callback(new Error('BlockDB is corrupt. All is lost.')); + return self._getEntry(record.height, function(err, entry) { + if (err) + return callback(err); + + tx.height = record.height; + + if (entry) { + tx.ts = entry.ts; + tx.block = entry.hash; + } + + tx._fileOffset = record.offset; + + if (self.options.paranoid && tx.hash('hex') !== hash) + return callback(new Error('BlockDB is corrupt. All is lost.')); + + return callback(null, tx); + }); } return callback(null, tx); @@ -849,10 +871,19 @@ BlockDB.prototype.getBlock = function getBlock(hash, callback) { block.height = record.height; if (self.options.paranoid) { if (typeof hash === 'number') { - hash = bcoin.chain.global.db.getSync(hash); - if (!hash) + self._getEntry(hash, function(err, entry) { + if (err) + return callback(err); + + if (!entry) + return callback(null, block); + + if (block.hash('hex') !== entry.hash) + return callback(new Error('BlockDB is corrupt. All is lost.')); + return callback(null, block); - hash = hash.hash; + }); + return; } if (block.hash('hex') !== hash) return callback(new Error('BlockDB is corrupt. All is lost.')); @@ -1036,6 +1067,13 @@ BlockDB.prototype.resetHeight = function resetHeight(height, callback, emit) { }); }; +BlockDB.prototype._getEntry = function _getEntry(height, callback) { + if (!bcoin.chain.global) + return callback(); + + return bcoin.chain.global.db.get(height, callback); +}; + /** * BlockData */ diff --git a/lib/bcoin/chain.js b/lib/bcoin/chain.js index 4721a49a..ff66b32f 100644 --- a/lib/bcoin/chain.js +++ b/lib/bcoin/chain.js @@ -701,11 +701,7 @@ Chain.prototype.resetHeight = function resetHeight(height, force) { // Reset the orphan map completely. There may // have been some orphans on a forked chain we // no longer need. - this.emit('purge', this.orphan.count, this.orphan.size); - this.orphan.map = {}; - this.orphan.bmap = {}; - this.orphan.count = 0; - this.orphan.size = 0; + this.purgeOrphans(); unlock(); }; @@ -730,8 +726,17 @@ Chain.prototype._lock = function _lock(func, args, force) { this.pendingBlocks[block.hash('hex')] = true; this.pendingSize += block.getSize(); if (this.pendingSize > this.pendingLimit) { - utils.debug('Warning: %dmb of pending blocks.', + utils.debug('Warning: %dmb of pending blocks. Purging.', utils.mb(this.pendingSize)); + this.pending.forEach(function(block) { + delete self.pendingBlocks[block.hash('hex')]; + self.pendingSize -= block.getSize(); + }); + this.pending.length = 0; + this.jobs = this.jobs.filter(function(item) { + return item[0] !== Chain.prototype.add; + }); + return; } } this.jobs.push([func, args]); @@ -766,6 +771,59 @@ Chain.prototype._lock = function _lock(func, args, force) { }; }; +Chain.prototype.purgeOrphans = function purgeOrphans() { + this.emit('purge', this.orphan.count, this.orphan.size); + this.orphan.map = {}; + this.orphan.bmap = {}; + this.orphan.count = 0; + this.orphan.size = 0; +}; + +Chain.prototype.pruneOrphans = function pruneOrphans() { + var self = this; + var best, last; + + best = Object.keys(this.orphan.map).reduce(function(best, prevBlock, i) { + var orphan = self.orphan.map[prevBlock]; + var height = orphan.getCoinbaseHeight(); + + last = orphan; + + if (!best || height > best.getCoinbaseHeight()) + return orphan; + + return best; + }, null); + + // Save the best for last... or the + // last for the best in this case. + if (!best || best.getCoinbaseHeight() <= 0) + best = last; + + this.emit('purge', + this.orphan.count - (best ? 1 : 0), + this.orphan.size - (best ? best.getSize() : 0)); + + Object.keys(this.orphan.bmap).forEach(function(hash) { + var orphan = self.orphan.bmap[hash]; + if (orphan !== best) + self.emit('unresolved', orphan, peer); + }); + + this.orphan.map = {}; + this.orphan.bmap = {}; + this.orphan.count = 0; + this.orphan.size = 0; + + if (!best) + return; + + this.orphan.map[best.prevBlock] = best; + this.orphan.bmap[best.hash('hex')] = best; + this.orphan.count++; + this.orphan.size += best.getSize(); +}; + Chain.prototype.resetHeightAsync = function resetHeightAsync(height, callback, force) { var self = this; @@ -775,7 +833,8 @@ Chain.prototype.resetHeightAsync = function resetHeightAsync(height, callback, f function done(err, result) { unlock(); - callback(err, result); + if (callback) + callback(err, result); } if (height === this.db.getSize() - 1) @@ -788,11 +847,7 @@ Chain.prototype.resetHeightAsync = function resetHeightAsync(height, callback, f // Reset the orphan map completely. There may // have been some orphans on a forked chain we // no longer need. - self.emit('purge', self.orphan.count, self.orphan.size); - self.orphan.map = {}; - self.orphan.bmap = {}; - self.orphan.count = 0; - self.orphan.size = 0; + self.purgeOrphans(); return done(); }); @@ -955,17 +1010,22 @@ Chain.prototype.resetTimeAsync = function resetTimeAsync(ts, callback, force) { this.byTimeAsync(ts, function(err, entry) { if (err) { unlock(); - return callback(err); + if (callback) + callback(err); + return; } if (!entry) { unlock(); - return callback(); + if (callback) + callback(); + return; } self.resetHeightAsync(entry.height, function(err) { unlock(); - callback(err); + if (callback) + callback(err); }, true); }, true); }; @@ -1033,12 +1093,7 @@ Chain.prototype.add = function add(initial, peer, callback, force) { // If the orphan chain forked, simply // reset the orphans and find a new peer. if (orphan.hash('hex') !== hash) { - self.emit('purge', self.orphan.count, self.orphan.size, peer); - - self.orphan.map = {}; - self.orphan.bmap = {}; - self.orphan.count = 0; - self.orphan.size = 0; + self.purgeOrphans(); self.emit('fork', block, { height: -1, @@ -1260,16 +1315,8 @@ Chain.prototype.add = function add(initial, peer, callback, force) { // Failsafe for large orphan chains. Do not // allow more than 20mb stored in memory. - if (self.orphan.size > self.orphanLimit) { - self.emit('purge', self.orphan.count, self.orphan.size, peer); - Object.keys(self.orphan.bmap).forEach(function(hash) { - self.emit('unresolved', self.orphan.bmap[hash], peer); - }); - self.orphan.map = {}; - self.orphan.bmap = {}; - self.orphan.count = 0; - self.orphan.size = 0; - } + if (self.orphan.size > self.orphanLimit) + self.pruneOrphans(); // We intentionally did not asyncify the // callback so if it calls chain.add, it @@ -1278,6 +1325,7 @@ Chain.prototype.add = function add(initial, peer, callback, force) { // so we don't cause a stack overflow if // these end up being all sync chain.adds. utils.nextTick(function() { + // XXX Possibly put `unlock()` above callback! if (err) callback(err); else @@ -1663,8 +1711,7 @@ Chain.prototype.getLocatorAsync = function getLocatorAsync(start, callback, forc if (err) return done(err); - if (!existing) - return done(new Error('Potential reset.')); + assert(existing); hashes[i] = existing.hash; diff --git a/lib/bcoin/pool.js b/lib/bcoin/pool.js index f5f1e70f..b8b8acef 100644 --- a/lib/bcoin/pool.js +++ b/lib/bcoin/pool.js @@ -816,8 +816,10 @@ Pool.prototype._createPeer = function _createPeer(options) { peer.on('txs', function(txs) { self.emit('txs', txs, peer); - if (self.blockdb && !self.chain.isFull()) - return; + if (!self.options.spv) { + if (!self.chain.isFull()) + return; + } txs.forEach(function(hash) { hash = utils.toHex(hash); @@ -1357,7 +1359,10 @@ Pool.prototype.searchWallet = function(w, h) { if (!height || height === -1) height = this.chain.height - (7 * 24 * 6); - utils.nextTick(function() { + this.chain.resetHeightAsync(height, function(err) { + if (err) + throw err; + utils.debug('Wallet height: %s', height); utils.debug( 'Reverted chain to height=%d', @@ -1365,15 +1370,16 @@ Pool.prototype.searchWallet = function(w, h) { ); }); - this.chain.resetHeight(height); - return; } if (!ts) ts = utils.now() - 7 * 24 * 3600; - utils.nextTick(function() { + this.chain.resetTimeAsync(ts, function(err) { + if (err) + throw err; + utils.debug('Wallet time: %s', new Date(ts * 1000)); utils.debug( 'Reverted chain to height=%d (%s)', @@ -1381,8 +1387,6 @@ Pool.prototype.searchWallet = function(w, h) { new Date(self.chain.tip.ts * 1000) ); }); - - this.chain.resetTime(ts); }; Pool.prototype.search = function search(id, range, e) { @@ -1467,9 +1471,13 @@ Pool.prototype.search = function search(id, range, e) { timeout = setTimeout(done.bind(null, true), total); if (range.start < this.chain.tip.ts) { - this.chain.resetTime(range.start); - this.stopSync(); - this.startSync(); + this.chain.resetTimeAsync(range.start, function(err) { + if (err) + throw err; + + self.stopSync(); + self.startSync(); + }); } return e; @@ -1543,15 +1551,20 @@ Pool.prototype._startRequests = function _startRequests(peer) { if (peer._blockQueue.length === 0) return; - if (!this.blockdb) { + if (this.options.spv) { items = peer._blockQueue.slice(); peer._blockQueue.length = 0; } else { - // Blocks start getting big after 150k - if (this.chain.height <= 150000) + // Blocks start getting big after 150k. + if (this.chain.height <= 100000) size = 500; + else if (this.chain.height <= 150000) + size = 250; + else if (this.chain.height <= 170000) + size = this.blockdb ? 4 : 40; else - size = 1; + size = this.blockdb ? 1 : 10; + items = peer._blockQueue.slice(0, size); peer._blockQueue = peer._blockQueue.slice(size); }