From 3154a1c91466138d2be6ac08ff4de89b07f7da3c Mon Sep 17 00:00:00 2001 From: Christopher Jeffrey Date: Wed, 18 May 2016 17:32:35 -0700 Subject: [PATCH] add feefilter packet. misc. --- lib/bcoin/mempool.js | 2 +- lib/bcoin/peer.js | 142 +++++++++++++++++++++++------------ lib/bcoin/pool.js | 13 ++-- lib/bcoin/protocol/framer.js | 45 ++++++++++- lib/bcoin/protocol/parser.js | 53 ++++++++++--- lib/bcoin/reader.js | 7 +- lib/bcoin/tx.js | 26 +++++++ lib/bcoin/types.js | 6 ++ 8 files changed, 228 insertions(+), 66 deletions(-) diff --git a/lib/bcoin/mempool.js b/lib/bcoin/mempool.js index 1cd684d0..942bf15b 100644 --- a/lib/bcoin/mempool.js +++ b/lib/bcoin/mempool.js @@ -854,7 +854,7 @@ Mempool.prototype.removeUnchecked = function removeUnchecked(entry, limit, callb self.total--; if (limit) { - rate = Math.floor(entry.fees * 1000 / entry.size); + rate = bcoin.tx.getRate(entry.size, entry.fees); rate += self.minReasonableFee; if (rate > self.minFeeRate) { self.minFeeRate = rate; diff --git a/lib/bcoin/peer.js b/lib/bcoin/peer.js index 5ee5b27e..3c129c1c 100644 --- a/lib/bcoin/peer.js +++ b/lib/bcoin/peer.js @@ -92,6 +92,8 @@ function Peer(pool, options) { this.hashContinue = null; this.filter = null; this.relay = true; + this.localNonce = utils.nonce(); + this.filterRate = -1; this.challenge = null; this.lastPong = 0; @@ -203,8 +205,6 @@ Peer.prototype._init = function init() { self.emit('ack'); self.ts = utils.now(); - self.write(self.framer.getAddr()); - if (self.options.headers) { if (self.version && self.version.version > 70012) self.write(self.framer.sendHeaders()); @@ -215,6 +215,16 @@ Peer.prototype._init = function init() { self.write(self.framer.haveWitness()); } + self.write(self.framer.getAddr()); + + self.sendInv(self.pool.inv.list); + + if (self.pool.options.filterRate != null) { + self.write(self.framer.feeFilter({ + rate: self.pool.options.filterRate + })); + } + if (self.chain.isFull()) self.getMempool(); }); @@ -222,7 +232,8 @@ Peer.prototype._init = function init() { // Send hello this.write(this.framer.version({ height: this.chain.height, - relay: this.options.relay + relay: this.options.relay, + nonce: this.localNonce })); }; @@ -278,9 +289,18 @@ Peer.prototype.sendInv = function sendInv(items) { if (!this.relay) return; + if (!items) + return; + if (!Array.isArray(items)) items = [items]; + if (items.length === 0) + return; + + if (items.length > 50000) + items = items.slice(0, 50000); + if (this.filter) items = items.map(this.isWatched, this); @@ -475,7 +495,7 @@ Peer.prototype._onPacket = function onPacket(packet) { var payload = packet.payload; if (this.lastBlock && cmd !== 'tx') - this._emitMerkle(); + this._flushMerkle(); switch (cmd) { case 'version': @@ -502,6 +522,8 @@ Peer.prototype._onPacket = function onPacket(packet) { return this._handleGetUTXOs(payload); case 'utxos': return this._handleUTXOs(payload); + case 'feefilter': + return this._handleFeeFilter(payload); case 'getblocks': return this._handleGetBlocks(payload); case 'getheaders': @@ -516,7 +538,7 @@ Peer.prototype._onPacket = function onPacket(packet) { return this._handleFilterClear(payload); case 'block': payload = new bcoin.compactblock(payload); - this._emit(cmd, payload); + this.fire(cmd, payload); break; case 'merkleblock': payload = new bcoin.merkleblock(payload); @@ -529,9 +551,9 @@ Peer.prototype._onPacket = function onPacket(packet) { this.lastBlock.addTX(payload); break; } - this._emitMerkle(); + this._flushMerkle(); } - this._emit(cmd, payload); + this.fire(cmd, payload); break; case 'sendheaders': this.sendHeaders = true; @@ -542,28 +564,28 @@ Peer.prototype._onPacket = function onPacket(packet) { this.response(cmd, payload); break; case 'verack': - this._emit(cmd, payload); + this.fire(cmd, payload); break; case 'notfound': - this._emit(cmd, payload); + this.fire(cmd, payload); break; default: bcoin.debug('Unknown packet: %s', cmd); - this._emit(cmd, payload); + this.fire(cmd, payload); break; } }; -Peer.prototype._emit = function _emit(cmd, payload) { +Peer.prototype.fire = function fire(cmd, payload) { if (this.response(cmd, payload)) return; this.emit(cmd, payload); }; -Peer.prototype._emitMerkle = function _emitMerkle() { +Peer.prototype._flushMerkle = function _flushMerkle() { if (this.lastBlock) - this._emit('merkleblock', this.lastBlock); + this.fire('merkleblock', this.lastBlock); this.lastBlock = null; }; @@ -585,14 +607,21 @@ Peer.prototype._handleFilterLoad = function _handleFilterLoad(payload) { }; Peer.prototype._handleFilterAdd = function _handleFilterAdd(payload) { + if (payload.data.length > constants.script.MAX_PUSH) { + this.setMisbehavior(100); + return; + } + if (this.filter) this.filter.add(payload.data); + this.relay = true; }; Peer.prototype._handleFilterClear = function _handleFilterClear(payload) { if (this.filter) this.filter.reset(); + this.relay = true; }; @@ -601,7 +630,18 @@ Peer.prototype._handleUTXOs = function _handleUTXOs(payload) { return new bcoin.coin(coin); }); bcoin.debug('Received %d utxos from %s.', payload.coins.length, this.host); - this._emit('utxos', payload); + this.fire('utxos', payload); +}; + +Peer.prototype._handleFeeFilter = function _handleFeeFilter(payload) { + if (!(payload.rate >= 0 && payload.rate <= constants.MAX_MONEY)) { + this.setMisbehavior(100); + return; + } + + this.filterRate = payload.rate; + + this.fire('feefilter', payload); }; /** @@ -876,6 +916,12 @@ Peer.prototype._handleVersion = function handleVersion(payload) { var version = payload.version; var services = payload.services; + if (payload.nonce.cmp(this.localNonce) === 0) { + this._error('We connected to ourself. Oops.'); + this.setMisbehavior(100); + return; + } + if (version < constants.MIN_VERSION) { this._error('Peer doesn\'t support required protocol version.'); this.setMisbehavior(100); @@ -957,18 +1003,15 @@ Peer.prototype._handleGetData = function handleGetData(items) { var self = this; var check = []; var notfound = []; - var hash, entry, isWitness, value, i, item; + var i, item, entry, witness; if (items.length > 50000) return this._error('message getdata size() = %d', items.length); for (i = 0; i < items.length; i++) { item = items[i]; - - hash = item.hash; - entry = this.pool.inv.map[hash]; - isWitness = (item.type & constants.WITNESS_MASK) !== 0; - value = null; + entry = this.pool.inv.map[item.hash]; + witness = (item.type & constants.WITNESS_MASK) !== 0; if (!entry) { check.push(item); @@ -987,16 +1030,16 @@ Peer.prototype._handleGetData = function handleGetData(items) { this.host, entry.packetType, utils.revHex(entry.key), - isWitness ? 'witness' : 'normal'); + witness ? 'witness' : 'normal'); - entry.sendTo(peer, isWitness); + entry.sendTo(peer, witness); } if (this.pool.options.selfish) return; utils.forEachSerial(check, function(item, next) { - var isWitness = item.type & constants.WITNESS_MASK; + var witness = item.type & constants.WITNESS_MASK; var type = (item.type & ~constants.WITNESS_MASK) !== 0; var hash = item.hash; var i, tx, data; @@ -1006,16 +1049,26 @@ Peer.prototype._handleGetData = function handleGetData(items) { notfound.push({ type: constants.inv.TX, hash: hash }); return next(); } - return self.mempool.getTX(hash, function(err, tx) { + return self.mempool.getEntry(hash, function(err, entry) { if (err) return next(err); - if (!tx) { + if (!entry) { notfound.push({ type: constants.inv.TX, hash: hash }); return next(); } - if (isWitness) + tx = entry.tx; + + // We should technically calculate this in + // the `mempool` handler, but it would be + // too slow. + if (self.filterRate !== -1) { + if (bcoin.tx.getRate(entry.size, entry.fees) < self.filterRate) + return next(); + } + + if (witness) data = tx.renderWitness(); else data = tx.renderNormal(); @@ -1044,7 +1097,7 @@ Peer.prototype._handleGetData = function handleGetData(items) { return next(); } - if (isWitness) + if (witness) data = block.renderWitness(); else data = block.renderNormal(); @@ -1088,7 +1141,7 @@ Peer.prototype._handleGetData = function handleGetData(items) { for (i = 0; i < block.txs.length; i++) { tx = block.txs[i]; - if (isWitness) + if (witness) tx = tx.renderWitness(); else tx = tx.renderNormal(); @@ -1109,6 +1162,7 @@ Peer.prototype._handleGetData = function handleGetData(items) { } notfound.push({ type: type, hash: hash }); + return next(); }, function(err) { if (err) @@ -1174,37 +1228,33 @@ Peer.prototype._handlePong = function handlePong(data) { Peer.prototype._handleGetAddr = function handleGetAddr() { var hosts = {}; - var peers = this.pool.peers.all; var items = []; - var i, peer, ip, version; + var ts = utils.now() - (process.uptime() | 0); + var i, seed, version, peer; if (this.pool.options.selfish) return; - for (i = 0; i < peers.length; i++) { - peer = peers[i]; - - if (!peer.socket || !peer.socket.remoteAddress) - continue; - - ip = peer.socket.remoteAddress; - version = utils.isIP(ip); + for (i = 0; i < this.pool.seeds.length; i++) { + seed = utils.parseHost(this.pool.seeds[i]); + seed = this.pool.getPeer(seed) || seed; + version = utils.isIP(seed.host); if (!version) continue; - if (hosts[ip]) + if (hosts[seed.host]) continue; - hosts[ip] = true; + hosts[seed.host] = true; items.push({ network: this.network, - ts: peer.ts, - services: peer.version ? peer.version.services : null, - ipv4: version === 4 ? ip : null, - ipv6: version === 6 ? ip : null, - port: peer.socket.remotePort || this.network.port + ts: seed.ts || ts, + services: seed.version ? seed.version.services : null, + ipv4: version === 4 ? seed.host : null, + ipv6: version === 6 ? seed.host : null, + port: seed.port || this.network.port }); if (items.length === 1000) @@ -1289,7 +1339,7 @@ Peer.prototype.getHeaders = function getHeaders(locator, stop) { this.host); bcoin.debug('Height: %s, Hash: %s, Stop: %s', - locator && locator.length ? this.chain._getCachedHeight(locator[0]) : null, + locator && locator.length ? this.chain._getCachedHeight(locator[0]) : -1, locator && locator.length ? utils.revHex(locator[0]) : 0, stop ? utils.revHex(stop) : 0); diff --git a/lib/bcoin/pool.js b/lib/bcoin/pool.js index 78ec1bd1..efec861e 100644 --- a/lib/bcoin/pool.js +++ b/lib/bcoin/pool.js @@ -1018,10 +1018,6 @@ Pool.prototype._createPeer = function _createPeer(options) { self.emit('version', version, peer); }); - peer.on('ack', function() { - peer.sendInv(self.inv.list); - }); - return peer; }; @@ -2181,11 +2177,16 @@ Pool.prototype.isMisbehaving = function isMisbehaving(host) { */ Pool.prototype.reject = function reject(peer, obj, code, reason, score) { + var type; + if (obj) { + type = (obj instanceof bcoin.tx) ? 'tx' : 'block'; + bcoin.debug('Rejecting %s %s from %s: ccode=%s reason=%s', - obj.type || 'block', obj.rhash, peer.host, code, reason); + type, obj.rhash, peer.host, code, reason); peer.reject({ + message: type, ccode: code, reason: reason, data: obj.hash() @@ -2331,7 +2332,7 @@ function BroadcastItem(pool, item, callback) { this.addCallback(callback); this.start(item); -}; +} utils.inherits(BroadcastItem, EventEmitter); diff --git a/lib/bcoin/protocol/framer.js b/lib/bcoin/protocol/framer.js index d29edeb5..65d19694 100644 --- a/lib/bcoin/protocol/framer.js +++ b/lib/bcoin/protocol/framer.js @@ -377,6 +377,26 @@ Framer.prototype.addr = function addr(peers) { return this.packet('addr', Framer.addr(peers)); }; +/** + * Create an alert packet with a header. + * @param {Object} options - See {@link Framer.alert}. + * @returns {Buffer} alert packet. + */ + +Framer.prototype.alert = function alert(options) { + return this.packet('alert', Framer.alert(options, this.network)); +}; + +/** + * Create a feefilter packet with a header. + * @param {Object} options - See {@link Framer.feefilter}. + * @returns {Buffer} feefilter packet. + */ + +Framer.prototype.feeFilter = function feeFilter(options) { + return this.packet('feefilter', Framer.feeFilter(options)); +}; + /** * Serialize an address. * @param {NetworkAddress} data @@ -432,6 +452,7 @@ Framer.version = function version(options, writer) { var agent = options.agent || constants.USER_AGENT; var remote = options.remote || {}; var local = options.local || {}; + var nonce = options.nonce; if (typeof agent === 'string') agent = new Buffer(agent, 'ascii'); @@ -445,12 +466,15 @@ Framer.version = function version(options, writer) { if (local.services == null) local.services = constants.LOCAL_SERVICES; + if (!nonce) + nonce = utils.nonce(); + p.write32(constants.VERSION); p.writeU64(constants.LOCAL_SERVICES); p.write64(utils.now()); Framer.address(remote, false, p); Framer.address(local, false, p); - p.writeU64(utils.nonce()); + p.writeU64(nonce); p.writeVarString(agent); p.write32(options.height || 0); p.writeU8(options.relay ? 1 : 0); @@ -1149,6 +1173,7 @@ Framer.addr = function addr(peers, writer) { /** * Create an alert packet (without a header). * @param {AlertPacket} data + * @param {Network|NetworkType} network * @param {BufferWriter?} writer - A buffer writer to continue writing from. * @returns {Buffer} Returns a BufferWriter if `writer` was passed in. */ @@ -1390,6 +1415,24 @@ Framer.filterClear = function filterClear() { return DUMMY; }; +/** + * Create a feefilter packet (without a header). + * @param {FeeFilterPacket} data + * @param {BufferWriter?} writer - A buffer writer to continue writing from. + * @returns {Buffer} Returns a BufferWriter if `writer` was passed in. + */ + +Framer.feeFilter = function feeFilter(data, network, writer) { + var p = new BufferWriter(writer); + + p.write64(data.rate); + + if (!writer) + p = p.render(); + + return p; +}; + /** * Calculate total block size and * witness size without serializing. diff --git a/lib/bcoin/protocol/parser.js b/lib/bcoin/protocol/parser.js index 0aebc792..273f7374 100644 --- a/lib/bcoin/protocol/parser.js +++ b/lib/bcoin/protocol/parser.js @@ -111,11 +111,12 @@ Parser.prototype.parse = function parse(chunk) { this.packet.payload = this.parsePayload(this.packet.cmd, this.packet.payload); } catch (e) { this.emit('error', e); + this.waiting = 24; + this.packet = null; + return; } - if (this.packet.payload) - this.emit('packet', this.packet); - + this.emit('packet', this.packet); this.waiting = 24; this.packet = null; }; @@ -349,6 +350,8 @@ Parser.prototype.parsePayload = function parsePayload(cmd, p) { return Parser.parseGetUTXOs(p); case 'utxos': return Parser.parseUTXOs(p); + case 'feefilter': + return Parser.parseFeeFilter(p); default: bcoin.debug('Unknown packet: %s', cmd); return p; @@ -472,16 +475,33 @@ Parser.parseVersion = function parseVersion(p) { services = p.readU53(); ts = p.read53(); recv = Parser.parseAddress(p, false); - from = Parser.parseAddress(p, false); - nonce = p.readU64(); - agent = p.readVarString('ascii'); - height = p.read32(); + + if (p.left() > 0) { + from = Parser.parseAddress(p, false); + nonce = p.readU64(); + } else { + from = {}; + nonce = new bn(0); + } + + if (p.left() > 0) + agent = p.readVarString('ascii', 256); + else + agent = ''; + + if (p.left() > 0) + height = p.read32(); + else + height = 0; if (p.left() > 0) relay = p.readU8() === 1; else relay = true; + if (version === 10300) + version = 300; + assert(version >= 0, 'Version is negative.'); assert(ts >= 0, 'Timestamp is negative.'); assert(height >= 0, 'Height is negative.'); @@ -1087,11 +1107,11 @@ Parser.parseReject = function parseReject(p) { p = new BufferReader(p); - message = p.readVarString('ascii'); + message = p.readVarString('ascii', 12); ccode = p.readU8(); - reason = p.readVarString('ascii'); + reason = p.readVarString('ascii', 111); - if (p.left() >= 32) + if (message === 'block' || message === 'tx') data = p.readHash('hex'); else data = null; @@ -1227,6 +1247,19 @@ Parser.parseAlert = function parseAlert(p) { }; }; +/** + * Parse feefilter packet. + * @param {Buffer|BufferReader} p + * @returns {FeeFilterPacket} + */ + +Parser.parseFeeFilter = function parseFeeFilter(p) { + p = new BufferReader(p); + return { + rate: p.read64N() + }; +}; + /* * Expose */ diff --git a/lib/bcoin/reader.js b/lib/bcoin/reader.js index ae530a08..8f9fe3ba 100644 --- a/lib/bcoin/reader.js +++ b/lib/bcoin/reader.js @@ -543,11 +543,14 @@ BufferReader.prototype.readHash = function readHash(enc) { /** * Read string of a varint length. * @param {String} enc - Any buffer-supported encoding. + * @param {Number?} limit - Size limit. * @returns {String} */ -BufferReader.prototype.readVarString = function readVarString(enc) { - return this.readString(enc, this.readVarint()); +BufferReader.prototype.readVarString = function readVarString(enc, limit) { + var size = this.readVarint(); + assert(!limit || size <= limit, 'String exceeds limit.'); + return this.readString(enc, size); }; /** diff --git a/lib/bcoin/tx.js b/lib/bcoin/tx.js index 20cf72aa..6990aa39 100644 --- a/lib/bcoin/tx.js +++ b/lib/bcoin/tx.js @@ -1515,6 +1515,20 @@ TX.prototype.getMinFee = function getMinFee(size, rate) { return TX.getMinFee(size, rate); }; +/** + * Calculate the transaction's rate based on size + * and fees. Size will be calculated if not present. + * @param {Number?} size + * @returns {Rate} + */ + +TX.prototype.getRate = function getRate(size) { + if (size == null) + size = this.maxSize(); + + return TX.getRate(size, this.getFee()); +}; + /** * Calculate minimum fee based on rate and size. * @param {Number?} size @@ -1536,6 +1550,17 @@ TX.getMinFee = function getMinFee(size, rate) { return fee; }; +/** + * Calculate a fee rate based on size and fees. + * @param {Number} size + * @param {Amount} fee + * @returns {Rate} + */ + +TX.getRate = function(size, fee) { + return Math.floor(fee * 1000 / size); +}; + /** * Calculate the minimum fee in order for the transaction * to be relayable, but _round to the nearest kilobyte @@ -1723,6 +1748,7 @@ TX.prototype.inspect = function inspect() { value: utils.btc(this.getOutputValue()), fee: utils.btc(this.getFee()), minFee: utils.btc(this.getMinFee()), + rate: utils.btc(this.getRate()), confirmations: this.getConfirmations(), priority: this.getPriority().priority.toString(10), date: utils.date(this.ts || this.ps), diff --git a/lib/bcoin/types.js b/lib/bcoin/types.js index 7584ec2d..382dfc63 100644 --- a/lib/bcoin/types.js +++ b/lib/bcoin/types.js @@ -375,6 +375,12 @@ * @global */ +/** + * @typedef {Object} FeeFilterPacket + * @property {Rate} rate + * @global + */ + /** * One of `main`, `testnet`, `regtest`, `segnet3`, `segnet4`. * @typedef {String} NetworkType