diff --git a/lib/blockchain/chain.js b/lib/blockchain/chain.js index c7fb7fd2..87f55c5e 100644 --- a/lib/blockchain/chain.js +++ b/lib/blockchain/chain.js @@ -33,12 +33,11 @@ var VerifyResult = errors.VerifyResult; * @param {String?} options.name - Database name. * @param {String?} options.location - Database file location. * @param {String?} options.db - Database backend (`"leveldb"` by default). - * @param {Number?} options.orphanLimit + * @param {Number?} options.maxOrphans * @param {Boolean?} options.spv * @property {Boolean} loaded * @property {ChainDB} db - Note that Chain `options` will be passed * to the instantiated ChainDB. - * @property {Number} total * @property {Lock} locker * @property {Object} invalid * @property {ChainEntry?} tip @@ -80,13 +79,10 @@ function Chain(options) { this.tip = null; this.height = -1; this.synced = false; - this.total = 0; - this.startTime = util.hrtime(); this.orphanMap = {}; this.orphanPrev = {}; this.orphanCount = 0; - this.orphanSize = 0; this.db = new ChainDB(this); } @@ -224,6 +220,7 @@ Chain.prototype.isGenesis = function isGenesis(block) { */ Chain.prototype.verify = co(function* verify(block, prev, flags) { + var hash = block.hash('hex'); var ret = new VerifyResult(); var now = this.network.now(); var height = prev.height + 1; @@ -236,6 +233,14 @@ Chain.prototype.verify = co(function* verify(block, prev, flags) { if (block.prevBlock !== prev.hash) throw new VerifyError(block, 'invalid', 'bad-prevblk', 0); + // Verify a checkpoint if there is one. + if (!this.verifyCheckpoint(prev, hash)) { + throw new VerifyError(block, + 'checkpoint', + 'checkpoint mismatch', + 100); + } + // Skip everything when using checkpoints. // We can do this safely because every // block in between each checkpoint was @@ -842,6 +847,7 @@ Chain.prototype.disconnect = co(function* disconnect(entry) { this.height = prev.height; this.emit('tip', prev); + yield this.fire('disconnect', entry, block, view); }); @@ -891,6 +897,7 @@ Chain.prototype.reconnect = co(function* reconnect(entry) { this.emit('tip', entry); this.emit('reconnect', entry, block); + yield this.fire('connect', entry, block, result.view); }); @@ -958,6 +965,7 @@ Chain.prototype.setBestChain = co(function* setBestChain(entry, block, prev, fla this.emit('tip', entry); this.emit('block', block, entry); + yield this.fire('connect', entry, block, result.view); }); @@ -1173,15 +1181,16 @@ Chain.prototype.scan = co(function* scan(start, filter, iter) { * Add a block to the chain, perform all necessary verification. * @method * @param {Block} block - * @param {Number} flags + * @param {Number?} flags + * @param {Number?} id * @returns {Promise} */ -Chain.prototype.add = co(function* add(block, flags) { +Chain.prototype.add = co(function* add(block, flags, id) { var hash = block.hash('hex'); var unlock = yield this.locker.lock(hash); try { - return yield this._add(block, flags); + return yield this._add(block, flags, id); } finally { unlock(); } @@ -1192,146 +1201,181 @@ Chain.prototype.add = co(function* add(block, flags) { * @method * @private * @param {Block} block - * @param {Number} flags + * @param {Number?} flags + * @param {Number?} id * @returns {Promise} */ -Chain.prototype._add = co(function* add(block, flags) { - var initial = true; - var result = null; - var hash, entry, prev; +Chain.prototype._add = co(function* add(block, flags, id) { + var hash = block.hash('hex'); + var entry, prev; if (flags == null) flags = common.flags.DEFAULT_FLAGS; - assert(block); + if (id == null) + id = -1; - while (block) { - hash = block.hash('hex'); - - // Mark the start time. - this.mark(); - - // Special case for genesis block. - if (hash === this.network.genesis.hash) { - this.logger.debug('Saw genesis block: %s.', block.rhash()); - throw new VerifyError(block, 'duplicate', 'duplicate', 0); - } - - // Do we already have this block in the queue? - if (this.hasPending(hash)) { - this.logger.debug('Already have pending block: %s.', block.rhash()); - throw new VerifyError(block, 'duplicate', 'duplicate', 0); - } - - // If the block is already known to be - // an orphan, ignore it. - if (this.hasOrphan(hash)) { - this.logger.debug('Already have orphan block: %s.', block.rhash()); - throw new VerifyError(block, 'duplicate', 'duplicate', 0); - } - - // Do not revalidate known invalid blocks. - if (this.hasInvalid(hash, block)) { - this.logger.debug('Invalid ancestors for block: %s.', block.rhash()); - throw new VerifyError(block, 'duplicate', 'duplicate', 100); - } - - // Check the POW before doing anything. - if (flags & common.flags.VERIFY_POW) { - if (!block.verifyPOW()) - throw new VerifyError(block, 'invalid', 'high-hash', 50); - } - - // Do we already have this block? - if (yield this.db.hasEntry(hash)) { - this.logger.debug('Already have block: %s.', block.rhash()); - throw new VerifyError(block, 'duplicate', 'duplicate', 0); - } - - // Find the previous block entry. - prev = yield this.db.getEntry(block.prevBlock); - - // If previous block wasn't ever seen, - // add it current to orphans and break. - if (!prev) { - assert(initial); - assert(!result); - this.storeOrphan(block); - break; - } - - // Verify a checkpoint if there is one. - if (!this.verifyCheckpoint(prev, hash)) { - throw new VerifyError(block, - 'checkpoint', - 'checkpoint mismatch', - 100); - } - - // Explanation: we try to keep as much data - // off the javascript heap as possible. Blocks - // in the future may be 8mb or 20mb, who knows. - // In fullnode-mode we store the blocks in - // "compact" form (the headers plus the raw - // Buffer object) until they're ready to be - // fully validated here. They are deserialized, - // validated, and emitted. Hopefully the deserialized - // blocks get cleaned up by the GC quickly. - if (block.memory) { - try { - block = block.toBlock(); - } catch (e) { - this.logger.error(e); - throw new VerifyError(block, - 'malformed', - 'error parsing message', - 10); - } - } - - // Create a new chain entry. - entry = ChainEntry.fromBlock(this, block, prev); - - // The block is on a alternate chain if the - // chainwork is less than or equal to - // our tip's. Add the block but do _not_ - // connect the inputs. - if (entry.chainwork.cmp(this.tip.chainwork) <= 0) { - // Save block to an alternate chain. - yield this.saveAlternate(entry, block, prev, flags); - } else { - // Attempt to add block to the chain index. - yield this.setBestChain(entry, block, prev, flags); - } - - // Emit the resolved orphan. - if (!initial) { - this.logger.debug( - 'Orphan block was resolved: %s (%d).', - block.rhash(), entry.height); - this.emit('resolved', block, entry); - } - - // Keep track of stats. - this.finish(block, entry); - - // Try to resolve orphan chain. - block = this.resolveOrphan(hash); - initial = false; - - if (!result) - result = entry; + // Special case for genesis block. + if (hash === this.network.genesis.hash) { + this.logger.debug('Saw genesis block: %s.', block.rhash()); + throw new VerifyError(block, 'duplicate', 'duplicate', 0); } - // Failsafe for large orphan chains. Do not - // allow more than 20mb stored in memory. - this.pruneOrphans(); + // Do we already have this block in the queue? + if (this.hasPending(hash)) { + this.logger.debug('Already have pending block: %s.', block.rhash()); + throw new VerifyError(block, 'duplicate', 'duplicate', 0); + } + + // If the block is already known to be + // an orphan, ignore it. + if (this.hasOrphan(hash)) { + this.logger.debug('Already have orphan block: %s.', block.rhash()); + throw new VerifyError(block, 'duplicate', 'duplicate', 0); + } + + // Do not revalidate known invalid blocks. + if (this.hasInvalid(hash, block)) { + this.logger.debug('Invalid ancestors for block: %s.', block.rhash()); + throw new VerifyError(block, 'duplicate', 'duplicate', 100); + } + + // Check the POW before doing anything. + if (flags & common.flags.VERIFY_POW) { + if (!block.verifyPOW()) + throw new VerifyError(block, 'invalid', 'high-hash', 50); + } + + // Do we already have this block? + if (yield this.db.hasEntry(hash)) { + this.logger.debug('Already have block: %s.', block.rhash()); + throw new VerifyError(block, 'duplicate', 'duplicate', 0); + } + + // Find the previous block entry. + prev = yield this.db.getEntry(block.prevBlock); + + // If previous block wasn't ever seen, + // add it current to orphans and return. + if (!prev) { + this.storeOrphan(block, flags, id); + return null; + } + + // Connect the block. + entry = yield this.connect(prev, block, flags); + + // Handle any orphans. + if (this.hasNextOrphan(hash)) + yield this.handleOrphans(entry); + + return entry; +}); + +/** + * Connect block to chain. + * @method + * @private + * @param {ChainEntry} prev + * @param {Block} block + * @param {Number} flags + * @returns {Promise} + */ + +Chain.prototype.connect = co(function* connect(prev, block, flags) { + var start = util.hrtime(); + var entry; + + // Sanity check. + assert(block.prevBlock === prev.hash); + + // Explanation: we try to keep as much data + // off the javascript heap as possible. Blocks + // in the future may be 8mb or 20mb, who knows. + // In fullnode-mode we store the blocks in + // "compact" form (the headers plus the raw + // Buffer object) until they're ready to be + // fully validated here. They are deserialized, + // validated, and connected. Hopefully the + // deserialized blocks get cleaned up by the + // GC quickly. + if (block.memory) { + try { + block = block.toBlock(); + } catch (e) { + this.logger.error(e); + throw new VerifyError(block, + 'malformed', + 'error parsing message', + 10); + } + } + + // Create a new chain entry. + entry = ChainEntry.fromBlock(this, block, prev); + + // The block is on a alternate chain if the + // chainwork is less than or equal to + // our tip's. Add the block but do _not_ + // connect the inputs. + if (entry.chainwork.cmp(this.tip.chainwork) <= 0) { + // Save block to an alternate chain. + yield this.saveAlternate(entry, block, prev, flags); + } else { + // Attempt to add block to the chain index. + yield this.setBestChain(entry, block, prev, flags); + } + + // Keep track of stats. + this.logStatus(start, block, entry); // Check sync state. this.maybeSync(); - return result; + return entry; +}); + +/** + * Handle orphans. + * @method + * @private + * @param {ChainEntry} entry + * @returns {Promise} + */ + +Chain.prototype.handleOrphans = co(function* handleOrphans(entry) { + var orphan = this.resolveOrphan(entry.hash); + var block, flags, id; + + while (orphan) { + block = orphan.block; + flags = orphan.flags; + id = orphan.id; + + try { + entry = yield this.connect(entry, block, flags); + } catch (err) { + if (err.type === 'VerifyError') { + this.logger.warning( + 'Could not resolve orphan block %s: %s.', + block.rhash(), err.message); + + this.emit('bad orphan', err, id); + + break; + } + throw err; + } + + this.logger.debug( + 'Orphan block was resolved: %s (%d).', + block.rhash(), entry.height); + + this.emit('resolved', block, entry); + + orphan = this.resolveOrphan(entry.hash); + } }); /** @@ -1344,43 +1388,37 @@ Chain.prototype.isSlow = function isSlow() { if (this.options.spv) return false; - if (this.total === 1 || this.total % 20 === 0) + if (this.synced) return true; - return this.synced || this.height >= this.network.block.slowHeight; -}; + if (this.height === 1 || this.height % 20 === 0) + return true; -/** - * Mark the start time for block processing. - * @private - */ + if (this.height >= this.network.block.slowHeight) + return true; -Chain.prototype.mark = function mark() { - this.startTime = util.hrtime(); + return false; }; /** * Calculate the time difference from * start time and log block. * @private + * @param {Array} start * @param {Block} block * @param {ChainEntry} entry */ -Chain.prototype.finish = function finish(block, entry) { +Chain.prototype.logStatus = function logStatus(start, block, entry) { var elapsed; - // Keep track of total blocks handled. - this.total += 1; - if (!this.isSlow()) return; // Report memory for debugging. - util.gc(); this.logger.memory(); - elapsed = util.hrtime(this.startTime); + elapsed = util.hrtime(start); this.logger.info( 'Block %s (%d) added to chain (size=%d txs=%d time=%d).', @@ -1443,29 +1481,32 @@ Chain.prototype.verifyCheckpoint = function verifyCheckpoint(prev, hash) { * Store an orphan. * @private * @param {Block} block + * @param {Number?} flags + * @param {Number?} id */ -Chain.prototype.storeOrphan = function storeOrphan(block) { +Chain.prototype.storeOrphan = function storeOrphan(block, flags, id) { var hash = block.hash('hex'); var height = block.getCoinbaseHeight(); var orphan = this.orphanPrev[block.prevBlock]; // The orphan chain forked. if (orphan) { - assert(orphan.hash('hex') !== hash); - assert(orphan.prevBlock === block.prevBlock); + assert(orphan.block.hash('hex') !== hash); + assert(orphan.block.prevBlock === block.prevBlock); this.logger.warning( 'Removing forked orphan block: %s (%d).', - orphan.rhash(), height); + orphan.block.rhash(), height); - this.resolveOrphan(block.prevBlock); + this.removeOrphan(orphan.block.hash('hex')); } - this.orphanCount++; - this.orphanSize += block.getSize(); - this.orphanPrev[block.prevBlock] = block; - this.orphanMap[hash] = block; + this.limitOrphans(); + + orphan = new Orphan(block, flags, id); + + this.addOrphan(orphan); this.logger.debug( 'Storing orphan block: %s (%d).', @@ -1474,26 +1515,75 @@ Chain.prototype.storeOrphan = function storeOrphan(block) { this.emit('orphan', block); }; +/** + * Add an orphan. + * @private + * @param {Orphan} orphan + * @returns {Orphan} + */ + +Chain.prototype.addOrphan = function addOrphan(orphan) { + var block = orphan.block; + var hash = block.hash('hex'); + + assert(!this.orphanMap[hash]); + assert(!this.orphanPrev[block.prevBlock]); + assert(this.orphanCount >= 0); + + this.orphanMap[hash] = orphan; + this.orphanPrev[block.prevBlock] = orphan; + this.orphanCount += 1; + + return orphan; +}; + +/** + * Remove an orphan. + * @private + * @param {Hash} hash + * @returns {Orphan} + */ + +Chain.prototype.removeOrphan = function removeOrphan(hash) { + var orphan = this.orphanMap[hash]; + var block = orphan.block; + + assert(this.orphanMap[hash]); + assert(this.orphanPrev[block.prevBlock]); + assert(this.orphanCount > 0); + + delete this.orphanMap[hash]; + delete this.orphanPrev[block.prevBlock]; + this.orphanCount -= 1; + + return orphan; +}; + +/** + * Test whether a hash would resolve the next orphan. + * @private + * @param {Hash} hash - Previous block hash. + * @returns {Boolean} + */ + +Chain.prototype.hasNextOrphan = function hasNextOrphan(hash) { + return this.orphanPrev[hash] != null; +}; + /** * Resolve an orphan. * @private * @param {Hash} hash - Previous block hash. - * @returns {Block} + * @returns {Orphan} */ Chain.prototype.resolveOrphan = function resolveOrphan(hash) { - var block = this.orphanPrev[hash]; + var orphan = this.orphanPrev[hash]; - if (!block) + if (!orphan) return; - delete this.orphanMap[block.hash('hex')]; - delete this.orphanPrev[hash]; - - this.orphanCount--; - this.orphanSize -= block.getSize(); - - return block; + return this.removeOrphan(orphan.block.hash('hex')); }; /** @@ -1502,18 +1592,15 @@ Chain.prototype.resolveOrphan = function resolveOrphan(hash) { Chain.prototype.purgeOrphans = function purgeOrphans() { var count = this.orphanCount; - var size = this.orphanSize; if (count === 0) return; - this.orphanPrev = {}; this.orphanMap = {}; + this.orphanPrev = {}; this.orphanCount = 0; - this.orphanSize = 0; - this.logger.debug('Purged %d orphans (%dmb).', - count, util.mb(size)); + this.logger.debug('Purged %d orphans.', count); }; /** @@ -1521,59 +1608,38 @@ Chain.prototype.purgeOrphans = function purgeOrphans() { * coinbase height (likely to be the peer's tip). */ -Chain.prototype.pruneOrphans = function pruneOrphans() { - var i, hashes, hash, orphan, height; - var bestOrphan, bestHeight, lastOrphan; - - if (this.orphanSize <= this.options.orphanLimit) - return false; - - this.logger.warning('Pruning %d (%dmb) orphans!', - this.orphanCount, util.mb(this.orphanSize)); - - hashes = Object.keys(this.orphanPrev); - - if (hashes.length === 0) - return false; - - for (i = 0; i < hashes.length; i++) { - hash = hashes[i]; - orphan = this.orphanPrev[hash]; - height = orphan.getCoinbaseHeight(); - - delete this.orphanPrev[hash]; - - if (!bestOrphan || height > bestHeight) { - bestOrphan = orphan; - bestHeight = height; - } - - lastOrphan = orphan; - } - - // Save the best for last... or the - // last for best in this case. - if (bestHeight === -1) - bestOrphan = lastOrphan; - - hashes = Object.keys(this.orphanMap); +Chain.prototype.limitOrphans = function limitOrphans() { + var now = util.now(); + var hashes = Object.keys(this.orphanMap); + var total = 0; + var i, hash, orphan, oldest; for (i = 0; i < hashes.length; i++) { hash = hashes[i]; orphan = this.orphanMap[hash]; - delete this.orphanMap[hash]; + if (now < orphan.ts + 60 * 60) { + if (!oldest || orphan.ts < oldest.ts) + oldest = orphan; + continue; + } - if (orphan !== bestOrphan) - this.emit('unresolved', orphan); + this.removeOrphan(hash); + + total++; } - this.orphanPrev[bestOrphan.prevBlock] = bestOrphan; - this.orphanMap[bestOrphan.hash('hex')] = bestOrphan; - this.orphanCount = 1; - this.orphanSize = bestOrphan.getSize(); + if (this.orphanCount >= this.options.maxOrphans) { + if (total === 0 && oldest) { + hash = oldest.block.hash('hex'); + this.removeOrphan(hash); + } + } - return true; + if (total > 0) + this.logger.warning('Pruned %d orphans!', total); + + return total; }; /** @@ -1850,13 +1916,18 @@ Chain.prototype._getLocator = co(function* getLocator(start) { */ Chain.prototype.getOrphanRoot = function getOrphanRoot(hash) { - var root; + var root, orphan; assert(hash); - while (this.orphanMap[hash]) { + for (;;) { + orphan = this.orphanMap[hash]; + + if (!orphan) + break; + root = hash; - hash = this.orphanMap[hash].prevBlock; + hash = orphan.block.prevBlock; } return root; @@ -2330,7 +2401,7 @@ function ChainOptions(options) { this.coinCache = 0; this.entryCache = 5000; - this.orphanLimit = 20 << 20; + this.maxOrphans = 20; this.checkpoints = true; if (options) @@ -2428,9 +2499,9 @@ ChainOptions.prototype.fromOptions = function fromOptions(options) { this.entryCache = options.entryCache; } - if (options.orphanLimit != null) { - assert(util.isNumber(options.orphanLimit)); - this.orphanLimit = options.orphanLimit; + if (options.maxOrphans != null) { + assert(util.isNumber(options.maxOrphans)); + this.maxOrphans = options.maxOrphans; } if (options.checkpoints != null) { @@ -2555,6 +2626,19 @@ function ContextResult(view, state) { this.state = state; } +/** + * Orphan + * @constructor + * @ignore + */ + +function Orphan(block, flags, id) { + this.block = block; + this.flags = flags; + this.id = id; + this.ts = util.now(); +} + /* * Expose */ diff --git a/lib/mempool/mempool.js b/lib/mempool/mempool.js index 5efbc139..e61cdbd2 100644 --- a/lib/mempool/mempool.js +++ b/lib/mempool/mempool.js @@ -292,7 +292,7 @@ Mempool.prototype._removeBlock = co(function* removeBlock(block, txs) { continue; try { - yield this.insertTX(tx); + yield this.insertTX(tx, -1); total++; } catch (e) { this.emit('error', e); @@ -448,25 +448,6 @@ Mempool.prototype.limitSize = function limitSize(added) { return !this.hasEntry(added); }; -/** - * Purge orphan transactions from the mempool. - */ - -Mempool.prototype.limitOrphans = function limitOrphans() { - var orphans = Object.keys(this.orphans); - var i, hash; - - while (this.totalOrphans > this.options.maxOrphans) { - i = crypto.randomRange(0, orphans.length); - hash = orphans[i]; - orphans.splice(i, 1); - - this.logger.spam('Removing orphan %s from mempool.', util.revHex(hash)); - - this.removeOrphan(hash); - } -}; - /** * Retrieve a transaction from the mempool. * @param {Hash} hash @@ -721,14 +702,15 @@ Mempool.prototype.hasReject = function hasReject(hash) { * fully processed. * @method * @param {TX} tx + * @param {Number?} id * @returns {Promise} */ -Mempool.prototype.addTX = co(function* addTX(tx) { +Mempool.prototype.addTX = co(function* addTX(tx, id) { var hash = tx.hash('hex'); var unlock = yield this.locker.lock(hash); try { - return yield this._addTX(tx); + return yield this._addTX(tx, id); } finally { unlock(); } @@ -739,14 +721,18 @@ Mempool.prototype.addTX = co(function* addTX(tx) { * @method * @private * @param {TX} tx + * @param {Number?} id * @returns {Promise} */ -Mempool.prototype._addTX = co(function* _addTX(tx) { +Mempool.prototype._addTX = co(function* _addTX(tx, id) { var missing; + if (id == null) + id = -1; + try { - missing = yield this.insertTX(tx); + missing = yield this.insertTX(tx, id); } catch (err) { if (err.type === 'VerifyError') { if (!tx.hasWitness() && !err.malleated) @@ -768,11 +754,13 @@ Mempool.prototype._addTX = co(function* _addTX(tx) { * @method * @private * @param {TX} tx + * @param {Number?} id * @returns {Promise} */ -Mempool.prototype.insertTX = co(function* insertTX(tx) { +Mempool.prototype.insertTX = co(function* insertTX(tx, id) { var lockFlags = common.lockFlags.STANDARD_LOCKTIME_FLAGS; + var height = this.chain.height; var hash = tx.hash('hex'); var ret = new VerifyResult(); var entry, view, missing; @@ -883,11 +871,11 @@ Mempool.prototype.insertTX = co(function* insertTX(tx) { // Maybe store as an orphan. if (missing) - return this.storeOrphan(tx, missing); + return this.storeOrphan(tx, missing, id); // Create a new mempool entry // at current chain height. - entry = MempoolEntry.fromTX(tx, view, this.chain.height); + entry = MempoolEntry.fromTX(tx, view, height); // Contextual verification. yield this.verify(entry, view); @@ -902,6 +890,8 @@ Mempool.prototype.insertTX = co(function* insertTX(tx) { 'mempool full', 0); } + + return null; }); /** @@ -1498,9 +1488,11 @@ Mempool.prototype.hasOrphan = function hasOrphan(hash) { /** * Store an orphaned transaction. * @param {TX} tx + * @param {Hash[]} missing + * @param {Number} id */ -Mempool.prototype.storeOrphan = function storeOrphan(tx, missing) { +Mempool.prototype.storeOrphan = function storeOrphan(tx, missing, id) { var hash = tx.hash('hex'); var i, prev; @@ -1520,60 +1512,73 @@ Mempool.prototype.storeOrphan = function storeOrphan(tx, missing) { } } + if (this.options.maxOrphans === 0) + return []; + + this.limitOrphans(); + for (i = 0; i < missing.length; i++) { prev = missing[i]; if (!this.waiting[prev]) - this.waiting[prev] = []; + this.waiting[prev] = new Map(); - this.waiting[prev].push(hash); + this.waiting[prev].insert(hash); } - this.orphans[hash] = new Orphan(tx, missing.length); + this.orphans[hash] = new Orphan(tx, missing.length, id); this.totalOrphans++; this.logger.debug('Added orphan %s to mempool.', tx.txid()); this.emit('add orphan', tx); - this.limitOrphans(); - return missing; }; /** * Resolve orphans and attempt to add to mempool. * @method - * @param {TX} tx + * @param {TX} parent * @returns {Promise} - Returns {@link TX}[]. */ -Mempool.prototype.handleOrphans = co(function* handleOrphans(tx) { - var resolved = this.resolveOrphans(tx); - var i, orphan; +Mempool.prototype.handleOrphans = co(function* handleOrphans(parent) { + var resolved = this.resolveOrphans(parent); + var i, orphan, tx, missing; for (i = 0; i < resolved.length; i++) { orphan = resolved[i]; try { - yield this.insertTX(orphan); + tx = orphan.toTX(); + } catch (e) { + this.logger.warning('%s %s', + 'Warning: possible memory corruption.', + 'Orphan failed deserialization.'); + } + + try { + missing = yield this.insertTX(tx, -1); } catch (err) { if (err.type === 'VerifyError') { this.logger.debug( 'Could not resolve orphan %s: %s.', - orphan.txid(), err.message); + tx.txid(), err.message); - if (!orphan.hasWitness() && !err.malleated) - this.rejects.add(orphan.hash()); + if (!tx.hasWitness() && !err.malleated) + this.rejects.add(tx.hash()); + + this.emit('bad orphan', err, orphan.id); continue; } throw err; } - this.logger.debug( - 'Resolved orphan %s in mempool (txs=%d).', - orphan.txid(), this.totalTX); + assert(!missing); + + this.logger.debug('Resolved orphan %s in mempool.', tx.txid()); } return resolved; @@ -1583,37 +1588,33 @@ Mempool.prototype.handleOrphans = co(function* handleOrphans(tx) { * Potentially resolve any transactions * that redeem the passed-in transaction. * Deletes all orphan entries and - * returns orphan hashes. - * @param {TX} tx - * @returns {TX[]} Resolved + * returns orphan objects. + * @param {TX} parent + * @returns {Orphan[]} */ -Mempool.prototype.resolveOrphans = function resolveOrphans(tx) { - var hash = tx.hash('hex'); +Mempool.prototype.resolveOrphans = function resolveOrphans(parent) { + var hash = parent.hash('hex'); + var map = this.waiting[hash]; var resolved = []; - var hashes = this.waiting[hash]; - var i, orphanHash, orphan; + var i, hashes, orphanHash, orphan; - if (!hashes) + if (!map) return resolved; + hashes = map.keys(); + assert(hashes.length > 0); + for (i = 0; i < hashes.length; i++) { orphanHash = hashes[i]; orphan = this.getOrphan(orphanHash); - if (!orphan) - continue; + assert(orphan); if (--orphan.missing === 0) { delete this.orphans[orphanHash]; this.totalOrphans--; - try { - resolved.push(orphan.toTX()); - } catch (e) { - this.logger.warning('%s %s', - 'Warning: possible memory corruption.', - 'Orphan failed deserialization.'); - } + resolved.push(orphan); } } @@ -1625,14 +1626,15 @@ Mempool.prototype.resolveOrphans = function resolveOrphans(tx) { /** * Remove a transaction from the mempool. * @param {Hash} tx + * @returns {Boolean} */ Mempool.prototype.removeOrphan = function removeOrphan(hash) { var orphan = this.getOrphan(hash); - var i, j, tx, hashes, prevout, prev; + var i, tx, map, prevout, prev; if (!orphan) - return; + return false; try { tx = orphan.toTX(); @@ -1649,19 +1651,16 @@ Mempool.prototype.removeOrphan = function removeOrphan(hash) { for (i = 0; i < prevout.length; i++) { prev = prevout[i]; - hashes = this.waiting[prev]; + map = this.waiting[prev]; - if (!hashes) + if (!map) continue; - j = hashes.indexOf(hash); + assert(map.has(hash)); - if (j === -1) - continue; + map.remove(hash); - hashes.splice(j, 1); - - if (hashes.length === 0) + if (map.size === 0) delete this.waiting[prev]; } @@ -1669,6 +1668,30 @@ Mempool.prototype.removeOrphan = function removeOrphan(hash) { this.totalOrphans--; this.emit('remove orphan', tx); + + return true; +}; + +/** + * Remove a random orphan transaction from the mempool. + * @returns {Boolean} + */ + +Mempool.prototype.limitOrphans = function limitOrphans() { + var hashes = Object.keys(this.orphans); + var index, hash; + + if (this.totalOrphans < this.options.maxOrphans) + return false; + + index = crypto.randomRange(0, hashes.length); + hash = hashes[index]; + + this.logger.debug('Removing orphan %s from mempool.', util.revHex(hash)); + + this.removeOrphan(hash); + + return true; }; /** @@ -2059,7 +2082,7 @@ MempoolOptions.prototype.fromOptions = function fromOptions(options) { } if (options.limitFreeRelay != null) { - assert(util.isNumber(options.limitFreeRelay)); + assert(util.isUInt32(options.limitFreeRelay)); this.limitFreeRelay = options.limitFreeRelay; } @@ -2094,27 +2117,27 @@ MempoolOptions.prototype.fromOptions = function fromOptions(options) { } if (options.maxSize != null) { - assert(util.isNumber(options.maxSize)); + assert(util.isUInt53(options.maxSize)); this.maxSize = options.maxSize; } if (options.maxOrphans != null) { - assert(util.isNumber(options.maxOrphans)); + assert(util.isUInt32(options.maxOrphans)); this.maxOrphans = options.maxOrphans; } if (options.maxAncestors != null) { - assert(util.isNumber(options.maxAncestors)); + assert(util.isUInt32(options.maxAncestors)); this.maxAncestors = options.maxAncestors; } if (options.expiryTime != null) { - assert(util.isNumber(options.expiryTime)); + assert(util.isUInt32(options.expiryTime)); this.expiryTime = options.expiryTime; } if (options.minRelay != null) { - assert(util.isNumber(options.minRelay)); + assert(util.isUint53(options.minRelay)); this.minRelay = options.minRelay; } @@ -2135,12 +2158,12 @@ MempoolOptions.prototype.fromOptions = function fromOptions(options) { } if (options.maxFiles != null) { - assert(util.isNumber(options.maxFiles)); + assert(util.isUInt32(options.maxFiles)); this.maxFiles = options.maxFiles; } if (options.cacheSize != null) { - assert(util.isNumber(options.cacheSize)); + assert(util.isUInt53(options.cacheSize)); this.cacheSize = options.cacheSize; } @@ -2382,11 +2405,13 @@ IndexedCoin.prototype.toCoin = function toCoin() { * @ignore * @param {TX} tx * @param {Hash[]} missing + * @param {Number} id */ -function Orphan(tx, missing) { +function Orphan(tx, missing, id) { this.raw = tx.toRaw(); this.missing = missing; + this.id = id; } Orphan.prototype.toTX = function toTX() { diff --git a/lib/net/packets.js b/lib/net/packets.js index 709809ed..a3ad3f54 100644 --- a/lib/net/packets.js +++ b/lib/net/packets.js @@ -1674,10 +1674,11 @@ RejectPacket.fromRaw = function fromRaw(data, enc) { * @private * @param {Number} code * @param {String} reason - * @param {(TX|Block)?} msg + * @param {String?} msg + * @param {Hash?} hash */ -RejectPacket.prototype.fromReason = function fromReason(code, reason, msg) { +RejectPacket.prototype.fromReason = function fromReason(code, reason, msg, hash) { if (typeof code === 'string') code = RejectPacket.codes[code.toUpperCase()]; @@ -1692,8 +1693,9 @@ RejectPacket.prototype.fromReason = function fromReason(code, reason, msg) { this.reason = reason; if (msg) { - this.message = (msg instanceof TX) ? 'tx' : 'block'; - this.hash = msg.hash('hex'); + assert(hash); + this.message = msg; + this.hash = hash; } return this; @@ -1703,12 +1705,13 @@ RejectPacket.prototype.fromReason = function fromReason(code, reason, msg) { * Instantiate reject packet from reason message. * @param {Number} code * @param {String} reason - * @param {(TX|Block)?} obj + * @param {String?} msg + * @param {Hash?} hash * @returns {RejectPacket} */ -RejectPacket.fromReason = function fromReason(code, reason, obj) { - return new RejectPacket().fromReason(code, reason, obj); +RejectPacket.fromReason = function fromReason(code, reason, msg, hash) { + return new RejectPacket().fromReason(code, reason, msg, hash); }; /** diff --git a/lib/net/peer.js b/lib/net/peer.js index bfe64fe3..2c1018fa 100644 --- a/lib/net/peer.js +++ b/lib/net/peer.js @@ -79,7 +79,7 @@ function Peer(options) { this.parser = new Parser(this.network); this.framer = new Framer(this.network); - this.id = Peer.uid++; + this.id = -1; this.socket = null; this.opened = false; this.outbound = false; @@ -149,13 +149,6 @@ function Peer(options) { util.inherits(Peer, EventEmitter); -/** - * Peer ID counter. - * @type {Number} - */ - -Peer.uid = 0; - /** * Max output bytes buffered before * invoking stall behavior for peer. @@ -2151,12 +2144,12 @@ Peer.prototype.sendMempool = function sendMempool() { * @param {TX|Block} msg */ -Peer.prototype.sendReject = function sendReject(code, reason, msg) { - var reject = packets.RejectPacket.fromReason(code, reason, msg); +Peer.prototype.sendReject = function sendReject(code, reason, msg, hash) { + var reject = packets.RejectPacket.fromReason(code, reason, msg, hash); if (msg) { this.logger.debug('Rejecting %s %s (%s): code=%s reason=%s.', - reject.message, msg.rhash(), this.hostname(), code, reason); + msg, util.revHex(hash), this.hostname(), code, reason); } else { this.logger.debug('Rejecting packet from %s: code=%s reason=%s.', this.hostname(), code, reason); @@ -2223,16 +2216,14 @@ Peer.prototype.ban = function ban() { /** * Send a `reject` packet to peer. * @see Framer.reject - * @param {(TX|Block)?} msg - * @param {String} code - * @param {String} reason - * @param {Number} score + * @param {String msg + * @param {VerifyError} err * @returns {Boolean} */ -Peer.prototype.reject = function reject(msg, code, reason, score) { - this.sendReject(code, reason, msg); - return this.increaseBan(score); +Peer.prototype.reject = function reject(msg, err) { + this.sendReject(err.code, err.reason, msg, err.hash); + return this.increaseBan(err.score); }; /** diff --git a/lib/net/pool.js b/lib/net/pool.js index 46267295..83a8a314 100644 --- a/lib/net/pool.js +++ b/lib/net/pool.js @@ -109,6 +109,7 @@ function Pool(options) { this.peers = new PeerList(); this.authdb = new BIP150.AuthDB(this.options); this.hosts = new HostList(this.options); + this.id = 0; if (this.options.spv) this.spvFilter = Bloom.fromRate(20000, 0.001, Bloom.flags.ALL); @@ -190,6 +191,10 @@ Pool.prototype._init = function _init() { this.mempool.on('tx', function(tx) { self.announceTX(tx); }); + + this.mempool.on('bad orphan', function(err, id) { + self.handleBadOrphan('tx', err, id); + }); } // Normally we would also broadcast @@ -203,6 +208,10 @@ Pool.prototype._init = function _init() { return; self.announceBlock(block); }); + + this.chain.on('bad orphan', function(err, id) { + self.handleBadOrphan('block', err, id); + }); } }; @@ -1162,6 +1171,29 @@ Pool.prototype.createInbound = function createInbound(socket) { return peer; }; +/** + * Allocate new peer id. + * @returns {Number} + */ + +Pool.prototype.uid = function uid() { + var MAX = util.MAX_SAFE_INTEGER; + + if (this.id >= MAX - this.peers.size() - 1) + this.id = 0; + + // Once we overflow, there's a chance + // of collisions. Unlikely to happen + // unless we have tried to connect 9 + // quadrillion times, but still + // account for it. + do { + this.id += 1; + } while (this.peers.find(this.id)); + + return this.id; +}; + /** * Bind to peer events. * @private @@ -1171,6 +1203,8 @@ Pool.prototype.createInbound = function createInbound(socket) { Pool.prototype.bindPeer = function bindPeer(peer) { var self = this; + peer.id = this.uid(); + peer.onPacket = function onPacket(packet) { return self.handlePacket(peer, packet); }; @@ -2265,10 +2299,10 @@ Pool.prototype._addBlock = co(function* addBlock(peer, block, flags) { peer.blockTime = util.ms(); try { - entry = yield this.chain.add(block, flags); + entry = yield this.chain.add(block, flags, peer.id); } catch (err) { if (err.type === 'VerifyError') { - peer.reject(block, err.code, err.reason, err.score); + peer.reject('block', err); this.logger.warning(err); return; } @@ -2395,6 +2429,33 @@ Pool.prototype.switchSync = co(function* switchSync(peer, hash) { yield this.getBlocks(peer, hash); }); +/** + * Handle bad orphan. + * @method + * @private + * @param {String} msg + * @param {VerifyError} err + * @param {Number} id + */ + +Pool.prototype.handleBadOrphan = function handleBadOrphan(msg, err, id) { + var peer = this.peers.find(id); + + if (!peer) { + this.logger.warning( + 'Could not find offending peer for orphan: %s (%d).', + util.revHex(err.hash), id); + return; + } + + this.logger.debug( + 'Punishing peer for sending a bad orphan (%s).', + peer.hostname()); + + // Punish the original peer who sent this. + peer.reject(msg, err); +}; + /** * Log sync status. * @private @@ -2402,23 +2463,21 @@ Pool.prototype.switchSync = co(function* switchSync(peer, hash) { */ Pool.prototype.logStatus = function logStatus(block) { - if (this.chain.total % 20 === 0) { + if (this.chain.height % 20 === 0) { this.logger.debug('Status:' + ' ts=%s height=%d progress=%s' - + ' blocks=%d orphans=%d active=%d' - + ' target=%s peers=%d jobs=%d', + + ' orphans=%d active=%d' + + ' target=%s peers=%d', util.date(block.ts), this.chain.height, (this.chain.getProgress() * 100).toFixed(2) + '%', - this.chain.total, this.chain.orphanCount, this.blockMap.size, block.bits, - this.peers.size(), - this.locker.jobs.length); + this.peers.size()); } - if (this.chain.total % 2000 === 0) { + if (this.chain.height % 2000 === 0) { this.logger.info( 'Received 2000 more blocks (height=%d, hash=%s).', this.chain.height, @@ -2504,10 +2563,10 @@ Pool.prototype._handleTX = co(function* handleTX(peer, packet) { } try { - missing = yield this.mempool.addTX(tx); + missing = yield this.mempool.addTX(tx, peer.id); } catch (err) { if (err.type === 'VerifyError') { - peer.reject(tx, err.code, err.reason, err.score); + peer.reject('tx', err); this.logger.info(err); return; } @@ -4164,6 +4223,7 @@ PoolOptions.prototype._resolve = function resolve(name) { function PeerList() { this.map = {}; + this.ids = {}; this.list = new List(); this.load = null; this.inbound = 0; @@ -4208,6 +4268,9 @@ PeerList.prototype.add = function add(peer) { assert(!this.map[peer.hostname()]); this.map[peer.hostname()] = peer; + assert(!this.ids[peer.id]); + this.ids[peer.id] = peer; + if (peer.outbound) this.outbound++; else @@ -4222,6 +4285,9 @@ PeerList.prototype.add = function add(peer) { PeerList.prototype.remove = function remove(peer) { assert(this.list.remove(peer)); + assert(this.ids[peer.id]); + delete this.ids[peer.id]; + assert(this.map[peer.hostname()]); delete this.map[peer.hostname()]; @@ -4257,6 +4323,16 @@ PeerList.prototype.has = function has(hostname) { return this.map[hostname] != null; }; +/** + * Get peer by ID. + * @param {Number} id + * @returns {Peer} + */ + +PeerList.prototype.find = function find(id) { + return this.ids[id]; +}; + /** * Destroy peer list (kills peers). */ diff --git a/lib/protocol/errors.js b/lib/protocol/errors.js index fb9dd987..8a7ba35f 100644 --- a/lib/protocol/errors.js +++ b/lib/protocol/errors.js @@ -50,6 +50,7 @@ function VerifyError(msg, code, reason, score, malleated) { this.code = code; this.reason = reason; this.score = score; + this.hash = msg.hash('hex'); this.malleated = malleated || false; this.message = 'Verification failure: ' + reason diff --git a/lib/utils/util.js b/lib/utils/util.js index 18fa796f..61f8e2a6 100644 --- a/lib/utils/util.js +++ b/lib/utils/util.js @@ -7,8 +7,6 @@ 'use strict'; -/* global gc */ - var assert = require('assert'); var nodeUtil = require('util'); var os = require('os'); @@ -83,15 +81,6 @@ if (os.homedir) { util.nop = function() {}; -/** - * Garbage collector for `--expose-gc`. - * @type function - * @static - * @method - */ - -util.gc = !util.isBrowser && typeof gc === 'function' ? gc : util.nop; - /** * Clone a buffer. * @param {Buffer} data