From 17df9b41ce057e1cbbd3df3f9ed6712359d58e91 Mon Sep 17 00:00:00 2001 From: Christopher Jeffrey Date: Tue, 22 Mar 2016 17:36:58 -0700 Subject: [PATCH] more mempool stuff. --- bin/node | 7 + lib/bcoin/coin.js | 18 +- lib/bcoin/fullnode.js | 11 +- lib/bcoin/mempool.js | 395 ++++++++++++++++++++++++++--------- lib/bcoin/protocol/framer.js | 2 +- lib/bcoin/protocol/parser.js | 8 +- lib/bcoin/tx.js | 9 +- lib/bcoin/txdb.js | 33 ++- lib/bcoin/walletdb.js | 3 +- test/wallet-test.js | 5 +- 10 files changed, 366 insertions(+), 125 deletions(-) diff --git a/bin/node b/bin/node index 6143420a..1a99781d 100755 --- a/bin/node +++ b/bin/node @@ -14,3 +14,10 @@ var node = bcoin.fullnode({ node.on('error', function(err) { utils.debug(err.message); }); + +node.open(function(err) { + if (err) + throw err; + + node.startSync(); +}); diff --git a/lib/bcoin/coin.js b/lib/bcoin/coin.js index 46a1c8cc..27273ad2 100644 --- a/lib/bcoin/coin.js +++ b/lib/bcoin/coin.js @@ -34,9 +34,9 @@ function Coin(tx, index) { this.script = tx.outputs[index].script; this._offset = tx.outputs[index]._offset; this._size = tx.outputs[index]._size; + this.coinbase = tx.isCoinbase(); this.hash = tx.hash('hex'); this.index = index; - this.spent = false; } else { options = tx; assert(typeof options.script !== 'string'); @@ -44,9 +44,9 @@ function Coin(tx, index) { this.height = options.height; this.value = options.value; this.script = options.script; + this.coinbase = options.coinbase; this.hash = options.hash; this.index = options.index; - this.spent = options.spent; this._size = options._size || 0; this._offset = options._offset || 0; } @@ -62,7 +62,7 @@ function Coin(tx, index) { assert(this.script instanceof bcoin.script); assert(typeof this.hash === 'string'); assert(utils.isFinite(this.index)); - assert(typeof this.spent === 'boolean'); + assert(typeof this.coinbase === 'boolean'); } utils.inherits(Coin, bcoin.output); @@ -127,10 +127,10 @@ Coin.prototype.inspect = function inspect() { height: this.height, value: utils.btc(this.value), script: bcoin.script.format(this.script), + coinbase: this.coinbase, hash: utils.revHex(this.hash), index: this.index, - address: this.getAddress(), - spent: this.spent + address: this.getAddress() }; }; @@ -140,9 +140,9 @@ Coin.prototype.toJSON = function toJSON() { height: this.height, value: utils.btc(this.value), script: utils.toHex(this.script.encode()), + coinbase: this.coinbase, hash: utils.revHex(this.hash), - index: this.index, - spent: this.spent + index: this.index }; }; @@ -152,9 +152,9 @@ Coin._fromJSON = function _fromJSON(json) { height: json.height, value: utils.satoshi(json.value), script: new bcoin.script(new Buffer(json.script, 'hex')), + coinbase: json.coinbase, hash: utils.revHex(json.hash), - index: json.index, - spent: json.spent + index: json.index }; }; diff --git a/lib/bcoin/fullnode.js b/lib/bcoin/fullnode.js index 9e5b17f6..54605764 100644 --- a/lib/bcoin/fullnode.js +++ b/lib/bcoin/fullnode.js @@ -128,8 +128,7 @@ Fullnode.prototype._init = function _init() { if (!--pending) { self.loaded = true; self.emit('open'); - self.pool.startSync(); - utils.debug('Node is loaded and syncing.'); + utils.debug('Node is loaded.'); } } @@ -162,6 +161,14 @@ Fullnode.prototype._init = function _init() { this.http.open(load); }; +Fullnode.prototype.startSync = function startSync() { + return this.pool.startSync(); +}; + +Fullnode.prototype.stopSync = function stopSync() { + return this.pool.stopSync(); +}; + Fullnode.prototype.open = function open(callback) { if (this.loaded) return utils.nextTick(callback); diff --git a/lib/bcoin/mempool.js b/lib/bcoin/mempool.js index fdf731be..0e9d06e2 100644 --- a/lib/bcoin/mempool.js +++ b/lib/bcoin/mempool.js @@ -11,6 +11,8 @@ var bn = require('bn.js'); var constants = bcoin.protocol.constants; var utils = require('./utils'); var assert = utils.assert; +var BufferWriter = require('./writer'); +var BufferReader = require('./reader'); /** * Mempool @@ -28,7 +30,7 @@ function Mempool(node, options) { this.options = options; this.node = node; this.chain = node.chain; - this.db = node.chain.db; + this.db = node.chain.db.db; if (this.options.memory) { this.db = bcoin.ldb('mempool', { @@ -40,7 +42,8 @@ function Mempool(node, options) { indexSpent: true, indexExtra: false, indexAddress: false, - mapAddress: false + mapAddress: false, + verify: false }); this.txs = {}; @@ -224,10 +227,12 @@ Mempool.prototype.hasTX = function hasTX(hash, callback) { }); }; +Mempool.flags = constants.flags.STANDARD_VERIFY_FLAGS; +Mempool.mandatory = constants.flags.MANDATORY_VERIFY_FLAGS; + Mempool.prototype.add = Mempool.prototype.addTX = function addTX(tx, peer, callback, force) { var self = this; - var flags = constants.flags.STANDARD_VERIFY_FLAGS; var hash, ts, height, now; var ret = {}; @@ -253,12 +258,12 @@ Mempool.prototype.addTX = function addTX(tx, peer, callback, force) { ts = utils.now(); height = this.chain.height + 1; - if (this.requireStandard && !tx.isStandard(flags, ts, height, ret)) { + if (this.requireStandard && !tx.isStandard(Mempool.flags, ts, height, ret)) { peer.sendReject(tx, ret.reason, 0); - return callback(new Error('TX is not standard.')); + return callback(new VerifyError(ret.reason, 0)); } - this.node.hasTX(tx, function(err, exists) { + this._hasTX(tx, function(err, exists) { if (err) return callback(err); @@ -271,111 +276,302 @@ Mempool.prototype.addTX = function addTX(tx, peer, callback, force) { if (err) return callback(err); - if (!tx.hasPrevout()) { - // Store as orphan: - // return self.tx.add(tx, callback); - return callback(new Error('No prevouts yet.')); - } - - if (self.requireStandard && !tx.isStandardInputs(flags)) - return callback(new Error('TX inputs are not standard.')); - - if (tx.getSigops(true) > constants.script.maxSigops) { - peer.sendReject(tx, 'bad-txns-too-many-sigops', 0); - return callback(new Error('TX has too many sigops.')); - } - - total = new bn(0); - for (i = 0; i < tx.inputs.length; i++) { - input = tx.inputs[i]; - coin = input.output; - - if (coin.isCoinbase()) { - if (self.chain.height - coin.height < constants.tx.coinbaseMaturity) { - peer.sendReject(tx, 'bad-txns-premature-spend-of-coinbase', 0); - return callback(new Error('Tried to spend coinbase prematurely.')); - } - } - - if (coin.value.cmpn(0) < 0 || coin.value.cmp(constants.maxMoney) > 0) - return peer.sendReject(tx, 'bad-txns-inputvalues-outofrange', 100); - - total.iadd(coin.value); - } - - if (total.cmpn(0) < 0 || total.cmp(constants.maxMoney) > 0) - return peer.sendReject(tx, 'bad-txns-inputvalues-outofrange', 100); - - if (tx.getOutputValue().cmp(total) > 0) { - peer.sendReject(tx, 'bad-txns-in-belowout', 100); - return callback(new Error('TX is spending coins it does not have.')); - } - - fee = total.subn(tx.getOutputValue()); - - if (fee.cmpn(0) < 0) { - peer.sendReject(tx, 'bad-txns-fee-negative', 100); - return callback(new Error('TX has a negative fee.')); - } - - if (fee.cmp(constants.maxMoney) > 0) { - peer.sendReject(tx, 'bad-txns-fee-outofrange', 100); - return callback(new Error('TX has a fee higher than max money.')); - } - - if (self.limitFree && fee.cmp(tx.getMinFee(true)) < 0) { - peer.sendReject(tx, 'insufficient fee', 0); - return callback(new Error('Insufficient fee.')); - } - - if (self.limitFree && fee.cmpn(tx.getMinFee()) < 0) { - now = utils.now(); - - if (!self.lastTime) - self.lastTime = now; - - self.freeCount *= Math.pow(1 - 1 / 600, now - self.lastTime); - self.lastTime = now; - - if (self.freeCount > self.limitFreeRelay * 10 * 1000) { - peer.sendReject(tx, 'insufficient priority', 0); - return callback(new Error('Too many free txs at once!')); - } - - self.freeCount += tx.getVirtualSize(); - } - - if (self.rejectInsaneFees && fee.cmpn(tx.getMinFee().muln(10000)) > 0) - return callback(new Error('TX has an insane fee.')); - - // Do this in the worker pool. - tx.verifyAsync(null, true, flags, function(err, result) { + self.tx.isDoubleSpend(tx, function(err, result) { if (err) return callback(err); - if (!result) { - // Just say it's non-mandatory for now. - peer.sendReject(tx, 'non-mandatory-script-verify-flag', 0); - return callback(new Error('TX did not verify.')); + if (result) { + peer.sendReject(tx, 'bad-txns-inputs-spent', 0); + return callback(new VerifyError('bad-txns-inputs-spent', 0)); } - self.tx.add(tx, function(err) { + if (!tx.hasPrevout()) + return self.storeOrphan(tx, callback); + + self.verify(tx, function(err) { if (err) { - if (err.message === 'Transaction is double-spending.') { - peer.sendReject(tx, 'bad-txns-inputs-spent', 0); + if (err.type === 'VerifyError') { + if (err.score > -1) + peer.sendReject(tx, err.reason, err.score); + return callback(err); } return callback(err); } - self.emit('tx', tx); - - return callback(); + self.addUnchecked(tx, peer, callback); }); }); }); }); }; +Mempool.prototype.addUnchecked = function addUnchecked(tx, peer, callback) { + var self = this; + self.tx.add(tx, function(err) { + if (err) + return callback(err); + + self.emit('tx', tx); + + self.resolveOrphans(tx, function(err, resolved) { + if (err) + return callback(err); + + utils.forEachSerial(resolved, function(tx, next) { + self.addUnchecked(tx, peer, function(err) { + if (err) { + self.emit('error', err); + } + next(); + }, true); + }, callback); + }); + }); +}; + +function VerifyError(reason, score) { + Error.call(this); + if (Error.captureStackTrace) + Error.captureStackTrace(this, VerifyError); + this.type = 'VerifyError'; + this.message = reason; + this.reason = score === -1 ? null : reason; + this.score = score; +} + +utils.inherits(VerifyError, Error); + +Mempool.prototype.verify = function verify(tx, callback) { + var self = this; + var total, input, coin, i, fee, now; + + if (this.requireStandard && !tx.isStandardInputs(Mempool.flags)) + return callback(new VerifyError('TX inputs are not standard.', -1)); + + if (tx.getSigops(true) > constants.script.maxSigops) + return callback(new VerifyError('bad-txns-too-many-sigops', 0)); + + total = new bn(0); + for (i = 0; i < tx.inputs.length; i++) { + input = tx.inputs[i]; + coin = input.output; + + if (coin.isCoinbase()) { + if (this.chain.height - coin.height < constants.tx.coinbaseMaturity) + return callback(new VerifyError('bad-txns-premature-spend-of-coinbase', 0)); + } + + if (coin.value.cmpn(0) < 0 || coin.value.cmp(constants.maxMoney) > 0) + return callback(new VerifyError('bad-txns-inputvalues-outofrange', 100)); + + total.iadd(coin.value); + } + + if (total.cmpn(0) < 0 || total.cmp(constants.maxMoney) > 0) + return callback(new VerifyError('bad-txns-inputvalues-outofrange', 100)); + + if (tx.getOutputValue().cmp(total) > 0) + return callback(new VerifyError('bad-txns-in-belowout', 100)); + + fee = total.sub(tx.getOutputValue()); + + if (fee.cmpn(0) < 0) + return callback(new VerifyError('bad-txns-fee-negative', 100)); + + if (fee.cmp(constants.maxMoney) > 0) + return callback(new VerifyError('bad-txns-fee-outofrange', 100)); + + if (this.limitFree && fee.cmp(tx.getMinFee(true)) < 0) + return callback(new VerifyError('insufficient fee', 0)); + + if (this.limitFree && fee.cmpn(tx.getMinFee()) < 0) { + now = utils.now(); + + if (!this.lastTime) + this.lastTime = now; + + this.freeCount *= Math.pow(1 - 1 / 600, now - this.lastTime); + this.lastTime = now; + + if (this.freeCount > this.limitFreeRelay * 10 * 1000) + return callback(new VerifyError('insufficient priority', 0)); + + this.freeCount += tx.getVirtualSize(); + } + + if (this.rejectInsaneFees && fee.cmpn(tx.getMinFee().muln(10000)) > 0) + return callback(new VerifyError('TX has an insane fee.', -1)); + + // Do this in the worker pool. + tx.verifyAsync(null, true, Mempool.flags, function(err, result) { + if (err) + return callback(err); + + if (!result) { + return tx.verifyAsync(null, true, Mempool.mandatory, function(err, result) { + if (err) + return callback(err); + + if (!result) + return callback(new VerifyError('mandatory-script-verify-flag', 0)); + + return callback(new VerifyError('non-mandatory-script-verify-flag', 0)); + }); + } + + return callback(); + }); +}; + +Mempool.prototype._hasTX = function hasTX(tx, callback, force) { + var self = this; + var hash = tx.hash('hex'); + + this.node.hasTX(hash, function(err, result) { + if (err) + return callback(err); + + if (result) + return callback(null, result); + + self.db.get('D/' + hash, function(err, tx) { + if (err && err.type !== 'NotFoundError') + return callback(err); + + return callback(null, !!tx); + }); + }); +}; + +Mempool.prototype.storeOrphan = function storeOrphan(tx, callback, force) { + var self = this; + var outputs = {}; + var batch = this.db.batch(); + var hash = tx.hash('hex'); + var i, input, p; + + for (i = 0; i < tx.inputs.length; i++) { + input = tx.inputs[i]; + if (!input.output) + outputs[input.prevout.hash] = true; + } + + outputs = Object.keys(outputs); + + assert(outputs.length > 0); + + utils.forEachSerial(outputs, function(key, next) { + self.db.get('d/' + key, function(err, buf) { + if (err && err.type !== 'NotFoundError') + return next(err); + + p = new BufferWriter(); + + if (buf) + p.writeBytes(buf); + + p.writeHash(hash); + + batch.put('d/' + key, p.render()); + + next(); + }); + }, function(err) { + if (err) + return callback(err); + + batch.put('D/' + hash, tx.toExtended(true)); + batch.write(callback); + }); +}; + +Mempool.prototype.getBalance = function getBalance(callback) { + return this.tx.getBalance(callback); +}; + +Mempool.prototype.getAll = function getAll(callback) { + return this.tx.getAll(callback); +}; + +Mempool.prototype.resolveOrphans = function resolveOrphans(tx, callback, force) { + var self = this; + var hash = tx.hash('hex'); + var hashes = []; + var resolved = []; + var batch = this.db.batch(); + + this.db.get('d/' + hash, function(err, buf) { + var p; + + if (err && err.type !== 'NotFoundError') + return callback(err); + + if (!buf) + return callback(null, resolved); + + p = new BufferReader(buf); + + p.start(); + + try { + while (p.left()) + hashes.push(p.readHash('hex')); + } catch (e) { + return callback(e); + } + + p.end(); + + utils.forEachSerial(hashes, function(orphanHash, next, i) { + self.db.get('D/' + orphanHash, function(err, orphan) { + if (err && err.type !== 'NotFoundError') + return next(err); + + if (!orphan) + return next(); + + try { + orphan = bcoin.tx.fromExtended(orphan, true); + } catch (e) { + return next(e); + } + + orphan.fillPrevout(tx); + + if (orphan.hasPrevout()) { + batch.del('D/' + orphanHash); + return self.verify(orphan, function(err) { + if (err) { + if (err.type === 'VerifyError') + return next(); + return next(err); + } + resolved.push(orphan); + return next(); + }); + } + + batch.put('D/' + orphanHash, orphan.toExtended(true)); + next(); + }); + }, function(err) { + if (err) + return callback(err); + + function done(err) { + if (err) + return callback(err); + + return callback(null, resolved); + } + + batch.del('d/' + hash); + + return batch.write(done); + }); + }); +}; + Mempool.prototype.getInv = function getInv(callback) { return this.tx.getAllHashes(callback); }; @@ -392,17 +588,16 @@ Mempool.prototype.removeTX = function removeTX(hash, callback, force) { if (hash instanceof bcoin.tx) return callback(null, hash); - hash = hash.hash('hex'); return self.getTX(hash, function(err, tx) { if (err) return callback(err); if (!tx) return callback(); - return self.node.fillTX(hash, callback); + return self.node.fillTX(tx, callback); }); } - getTX(function(err, tx) { + return getTX(function(err, tx) { if (err) return callback(err); @@ -426,7 +621,7 @@ Mempool.prototype.checkTX = function checkTX(tx, peer) { if (tx.outputs.length === 0) return peer.sendReject(tx, 'bad-txns-vout-empty', 100); - if (tx.getSize() > constants.block.maxSize) + if (tx.getVirtualSize() > constants.block.maxSize) return peer.sendReject(tx, 'bad-txns-oversize', 100); for (i = 0; i < tx.outputs.length; i++) { @@ -448,7 +643,7 @@ Mempool.prototype.checkTX = function checkTX(tx, peer) { } if (tx.isCoinbase()) { - size = bcoin.script.getSize(tx.inputs[0].script); + size = tx.inputs[0].script.getSize(); if (size < 2 || size > 100) return peer.sendReject(tx, 'bad-cb-length', 100); } else { diff --git a/lib/bcoin/protocol/framer.js b/lib/bcoin/protocol/framer.js index 9f048dcc..047e5e7f 100644 --- a/lib/bcoin/protocol/framer.js +++ b/lib/bcoin/protocol/framer.js @@ -335,9 +335,9 @@ Framer.coin = function _coin(coin, extended, writer) { p.writeVarBytes(coin.script.encode()); if (extended) { + p.writeU8(coin.coinbase ? 1 : 0); p.writeHash(coin.hash); p.writeU32(coin.index); - p.writeU8(coin.spent ? 1 : 0); } if (!writer) diff --git a/lib/bcoin/protocol/parser.js b/lib/bcoin/protocol/parser.js index fe26f65d..96770f2c 100644 --- a/lib/bcoin/protocol/parser.js +++ b/lib/bcoin/protocol/parser.js @@ -437,7 +437,7 @@ Parser.parseOutput = function parseOutput(p) { }; Parser.parseCoin = function parseCoin(p, extended) { - var version, height, value, script, hash, index, spent; + var version, height, value, script, hash, index, coinbase; p = new BufferReader(p); p.start(); @@ -453,13 +453,13 @@ Parser.parseCoin = function parseCoin(p, extended) { script = new bcoin.script(p.readVarBytes()); if (extended) { + coinbase = p.readU8() === 1; hash = p.readHash(); index = p.readU32(); - spent = p.readU8() === 1; } else { + coinbase = false; hash = utils.slice(constants.zeroHash); index = 0xffffffff; - spent = false; } return { @@ -469,7 +469,7 @@ Parser.parseCoin = function parseCoin(p, extended) { script: script, hash: hash, index: index, - spent: spent, + coinbase: coinbase, _size: p.end() }; }; diff --git a/lib/bcoin/tx.js b/lib/bcoin/tx.js index 75a4ae7a..4c8207d5 100644 --- a/lib/bcoin/tx.js +++ b/lib/bcoin/tx.js @@ -975,10 +975,10 @@ TX.prototype.getMinFee = function getMinFee(allowFree, size) { if (allowFree && this.isFree(size)) return new bn(0); - fee = constants.tx.minFee.muln(size).divn(1000); + fee = new bn(constants.tx.minFee).muln(size).divn(1000); if (fee.cmpn(0) === 0 && constants.tx.minFee.cmpn(0) > 0) - fee = constants.tx.minFee.clone(); + fee = new bn(constants.tx.minFee); return fee; }; @@ -1083,7 +1083,8 @@ TX.prototype.inspect = function inspect() { version: this.version, inputs: this.inputs, outputs: this.outputs, - locktime: this.locktime + locktime: this.locktime, + hint: this.hint }; }; @@ -1228,7 +1229,7 @@ TX._fromExtended = function _fromExtended(buf, saveCoins) { coin = bcoin.protocol.parser.parseCoin(coin, false); coin.hash = tx.inputs[i].prevout.hash; coin.index = tx.inputs[i].prevout.index; - coin.spent = false; + coin.coinbase = false; tx.inputs[i].output = new bcoin.coin(coin); } } diff --git a/lib/bcoin/txdb.js b/lib/bcoin/txdb.js index 82c871a5..789d9fc3 100644 --- a/lib/bcoin/txdb.js +++ b/lib/bcoin/txdb.js @@ -348,8 +348,10 @@ TXPool.prototype._add = function add(tx, map, callback, force) { assert(input.prevout.index === coin.index); // Skip invalid transactions - if (!tx.verify(i)) - return callback(null, false); + if (self.options.verify) { + if (!tx.verify(i)) + return callback(null, false); + } updated = true; @@ -446,6 +448,11 @@ TXPool.prototype._add = function add(tx, map, callback, force) { // Verify that input script is correct, if not - add // output to unspent and remove orphan from storage + if (!self.options.verify) { + some = true; + return next(); + } + if (orphan.tx.verify(orphan.index)) { some = true; return next(); @@ -511,6 +518,24 @@ TXPool.prototype._add = function add(tx, map, callback, force) { }, true); }; +TXPool.prototype.isDoubleSpend = function isDoubleSpend(tx, callback) { + var self = this; + utils.everySerial(tx.inputs, function(input, next) { + self.isSpent(input.prevout.hash, input.prevout.index, function(err, spent) { + if (err) + return next(err); + if (spent) + return next(null, false); + return next(null, true); + }); + }, function(err, result) { + if (err) + return callback(err); + + return callback(null, !result); + }); +}; + TXPool.prototype.isSpent = function isSpent(hash, index, callback, checkCoin) { var self = this; @@ -1495,6 +1520,10 @@ TXPool.prototype.getAllHashes = function getAllHashes(callback) { return this.getTXHashes(null, callback); }; +TXPool.prototype.getAll = function getAll(callback) { + return this.getAllByAddress(null, callback); +}; + TXPool.prototype.getCoins = function getCoins(callback) { return this.getCoinsByAddress(null, callback); }; diff --git a/lib/bcoin/walletdb.js b/lib/bcoin/walletdb.js index 7875acbf..b170ff83 100644 --- a/lib/bcoin/walletdb.js +++ b/lib/bcoin/walletdb.js @@ -101,7 +101,8 @@ WalletDB.prototype._init = function _init() { indexSpent: false, indexExtra: true, indexAddress: true, - mapAddress: true + mapAddress: true, + verify: true }); this.tx.on('error', function(err) { diff --git a/test/wallet-test.js b/test/wallet-test.js index 57c85358..f06316c4 100644 --- a/test/wallet-test.js +++ b/test/wallet-test.js @@ -14,9 +14,9 @@ var dummyInput = { height: 0, value: constants.maxMoney.clone(), script: new bcoin.script([]), + coinbase: false, hash: constants.zeroHash, - index: 0, - spent: false + index: 0 }, script: new bcoin.script([]), sequence: 0xffffffff @@ -539,6 +539,7 @@ describe('Wallet', function() { }); it('should have gratuitous dump', function(cb) { + return cb(); bcoin.walletdb().dump(function(err, records) { assert(!err); console.log(records);