From 5537eb19957868d230ecef14dd38e650ab74c14c Mon Sep 17 00:00:00 2001 From: Christopher Jeffrey Date: Sun, 22 May 2016 00:52:31 -0700 Subject: [PATCH] optimize parsing and serialization. strict parsing. --- lib/bcoin/abstractblock.js | 1 + lib/bcoin/block.js | 6 ++- lib/bcoin/chain.js | 1 + lib/bcoin/compactblock.js | 2 - lib/bcoin/peer.js | 27 ++++++------ lib/bcoin/pool.js | 4 +- lib/bcoin/protocol/framer.js | 77 +++++++++++++++++++++++++++++------ lib/bcoin/protocol/network.js | 1 + lib/bcoin/protocol/parser.js | 55 +++++++++++++++++++++---- lib/bcoin/reader.js | 35 ++++++++-------- lib/bcoin/tx.js | 8 ++-- lib/bcoin/utils.js | 9 +++- 12 files changed, 162 insertions(+), 64 deletions(-) diff --git a/lib/bcoin/abstractblock.js b/lib/bcoin/abstractblock.js index c1823547..e41883db 100644 --- a/lib/bcoin/abstractblock.js +++ b/lib/bcoin/abstractblock.js @@ -6,6 +6,7 @@ */ var bcoin = require('./env'); +var constants = bcoin.protocol.constants; var utils = bcoin.utils; var assert = utils.assert; diff --git a/lib/bcoin/block.js b/lib/bcoin/block.js index 5efd0fcc..75853f6d 100644 --- a/lib/bcoin/block.js +++ b/lib/bcoin/block.js @@ -47,9 +47,13 @@ function Block(data) { bcoin.abstractblock.call(this, data); this.txs = []; + this._cbHeight = null; this._commitmentHash = null; - this._raw = null; + + this._raw = data._raw || null; + this._size = data._size || null; + this._witnessSize = data._witnessSize != null ? data._witnessSize : null; if (data.txs) { for (i = 0; i < data.txs.length; i++) diff --git a/lib/bcoin/chain.js b/lib/bcoin/chain.js index 6e70343b..baa33736 100644 --- a/lib/bcoin/chain.js +++ b/lib/bcoin/chain.js @@ -1523,6 +1523,7 @@ Chain.prototype.add = function add(block, callback, force) { try { block = block.toBlock(); } catch (e) { + bcoin.error(e); return done(new VerifyError(block, 'malformed', 'error parsing message', diff --git a/lib/bcoin/compactblock.js b/lib/bcoin/compactblock.js index 3289951f..a45da193 100644 --- a/lib/bcoin/compactblock.js +++ b/lib/bcoin/compactblock.js @@ -101,8 +101,6 @@ CompactBlock.prototype.getCoinbaseHeight = function getCoinbaseHeight() { CompactBlock.prototype.toBlock = function toBlock() { var data = bcoin.protocol.parser.parseBlock(this.raw); delete this.raw; - assert(!data.raw); - assert(!data._raw); return new bcoin.block(data); }; diff --git a/lib/bcoin/peer.js b/lib/bcoin/peer.js index 7b6a8a06..23c724ab 100644 --- a/lib/bcoin/peer.js +++ b/lib/bcoin/peer.js @@ -1127,12 +1127,11 @@ Peer.prototype._handleGetData = function handleGetData(items) { return next(); } - if (witness) - data = tx.renderWitness(); - else - data = tx.renderNormal(); + data = witness + ? self.framer.witnessTX(tx) + : self.framer.tx(tx); - self.write(self.framer.packet('tx', data)); + self.write(data); next(); }); @@ -1156,12 +1155,11 @@ Peer.prototype._handleGetData = function handleGetData(items) { return next(); } - if (witness) - data = block.renderWitness(); - else - data = block.renderNormal(); + data = witness + ? self.framer.witnessBlock(block) + : self.framer.block(block); - self.write(self.framer.packet('block', data)); + self.write(data); if (hash === self.hashContinue) { self.sendInv({ @@ -1200,12 +1198,11 @@ Peer.prototype._handleGetData = function handleGetData(items) { for (i = 0; i < block.txs.length; i++) { tx = block.txs[i]; - if (witness) - tx = tx.renderWitness(); - else - tx = tx.renderNormal(); + tx = witness + ? self.framer.witnessTX(tx) + : self.framer.tx(tx); - self.write(self.framer.packet('tx', tx)); + self.write(tx); } if (hash === self.hashContinue) { diff --git a/lib/bcoin/pool.js b/lib/bcoin/pool.js index ddaad66e..75ba5f80 100644 --- a/lib/bcoin/pool.js +++ b/lib/bcoin/pool.js @@ -1570,9 +1570,7 @@ Pool.prototype.has = function has(type, hash, force, callback) { // If we recently rejected this item. Ignore. if (this.rejects.test(hash, 'hex')) { callback = utils.asyncify(callback); - bcoin.debug( - 'Peer sent a known reject: %s.', - hash); + bcoin.debug('Peer sent a known reject: %s.', hash); return callback(null, true); } } diff --git a/lib/bcoin/protocol/framer.js b/lib/bcoin/protocol/framer.js index b4f593ec..d6539b11 100644 --- a/lib/bcoin/protocol/framer.js +++ b/lib/bcoin/protocol/framer.js @@ -41,7 +41,7 @@ function Framer(options) { * @returns {Buffer} Header. */ -Framer.prototype.header = function header(cmd, payload) { +Framer.prototype.header = function header(cmd, payload, checksum) { var h = new Buffer(24); var len, i; @@ -61,8 +61,11 @@ Framer.prototype.header = function header(cmd, payload) { // Payload length h.writeUInt32LE(payload.length, 16, true); + if (!checksum) + checksum = utils.dsha256(payload); + // Checksum - utils.checksum(payload).copy(h, 20); + checksum.copy(h, 20, 0, 4); return h; }; @@ -74,12 +77,12 @@ Framer.prototype.header = function header(cmd, payload) { * @returns {Buffer} Payload with header prepended. */ -Framer.prototype.packet = function packet(cmd, payload) { +Framer.prototype.packet = function packet(cmd, payload, checksum) { var header; assert(payload, 'No payload.'); - header = this.header(cmd, payload); + header = this.header(cmd, payload, checksum); return Buffer.concat([header, payload]); }; @@ -304,7 +307,14 @@ Framer.prototype.getBlocks = function getBlocks(data) { */ Framer.prototype.tx = function tx(tx) { - return this.packet('tx', Framer.renderTX(tx, false)); + var checksum; + + // Save some time by using the + // cached hash as our checksum. + if (tx.hash) + checksum = tx.hash(); + + return this.packet('tx', Framer.renderTX(tx, false), checksum); }; /** @@ -314,7 +324,22 @@ Framer.prototype.tx = function tx(tx) { */ Framer.prototype.witnessTX = function witnessTX(tx) { - return this.packet('tx', Framer.renderTX(tx, true)); + var checksum; + + // Save some time by using the + // cached hash as our checksum. + if (tx.witnessHash) { + if (tx.hasWitness()) { + // We can't use the coinbase + // hash since it is all zeroes. + if (!tx.isCoinbase()) + checksum = tx.witnessHash(); + } else { + checksum = tx.hash(); + } + } + + return this.packet('tx', Framer.renderTX(tx, true), checksum); }; /** @@ -912,19 +937,45 @@ Framer.renderTX = function renderTX(tx, useWitness, writer) { var p = new BufferWriter(writer); var witnessSize; + // Cache the serialization if we can. + if (tx.render && !tx.mutable && !tx._raw) + tx.render(); + + // Try the cached raw data first. if (tx._raw) { - if (!useWitness && bcoin.protocol.parser.isWitnessTX(tx._raw)) { - Framer.tx(tx, p); - witnessSize = p._witnessSize; - } else { + if (useWitness) { + // If we're serializing the witness, + // we can use whatever data getRaw() + // gave us. p.writeBytes(tx._raw); witnessSize = tx._witnessSize; + } else { + // We have to use the standard format + // here. Try to grab it from cache. + if (bcoin.protocol.parser.isWitnessTX(tx._raw)) { + Framer.tx(tx, p); + witnessSize = p._witnessSize; + } else { + p.writeBytes(tx._raw); + witnessSize = tx._witnessSize; + } } } else { - if (useWitness && bcoin.tx.prototype.hasWitness.call(tx)) - Framer.witnessTX(tx, p); - else + if (useWitness) { + if (bcoin.tx.prototype.hasWitness.call(tx)) { + Framer.witnessTX(tx, p); + } else { + // Only use the witness serialization if + // we have a witness. This clause isn't + // necessary above since we already + // determined this in getRaw(). + Framer.tx(tx, p); + } + } else { + // Any other case, we use + // the standard serialization. Framer.tx(tx, p); + } witnessSize = p._witnessSize; } diff --git a/lib/bcoin/protocol/network.js b/lib/bcoin/protocol/network.js index b961e55e..103d020c 100644 --- a/lib/bcoin/protocol/network.js +++ b/lib/bcoin/protocol/network.js @@ -871,6 +871,7 @@ segnet4.type = 'segnet4'; segnet4.height = -1; segnet4.seeds = [ + '104.243.38.34', '37.34.48.17' ]; diff --git a/lib/bcoin/protocol/parser.js b/lib/bcoin/protocol/parser.js index 24f2c02b..18df2923 100644 --- a/lib/bcoin/protocol/parser.js +++ b/lib/bcoin/protocol/parser.js @@ -784,9 +784,12 @@ Parser.parseBlock = function parseBlock(p) { var txs = []; var version, prevBlock, merkleRoot, ts, bits, nonce; var i, totalTX, tx; + var raw, size, witnessSize; p = new BufferReader(p); + p.start(); + version = p.readU32(); // Technically signed prevBlock = p.readHash('hex'); merkleRoot = p.readHash('hex'); @@ -795,11 +798,17 @@ Parser.parseBlock = function parseBlock(p) { nonce = p.readU32(); totalTX = p.readVarint(); + witnessSize = 0; + for (i = 0; i < totalTX; i++) { tx = Parser.parseTX(p); + witnessSize += tx._witnessSize; txs.push(tx); } + raw = p.endData(); + size = raw.length; + return { version: version, prevBlock: prevBlock, @@ -807,7 +816,10 @@ Parser.parseBlock = function parseBlock(p) { ts: ts, bits: bits, nonce: nonce, - txs: txs + txs: txs, + _raw: raw, + _size: size, + _witnessSize: witnessSize }; }; @@ -961,16 +973,20 @@ Parser.parseTX = function parseTX(p) { var inCount, inputs; var outCount, outputs; var version, locktime, i; + var raw, size, witnessSize; if (Parser.isWitnessTX(p)) return Parser.parseWitnessTX(p); p = new BufferReader(p); - version = p.readU32(); // Technically signed - inCount = p.readVarint(); + p.start(); + version = p.readU32(); // Technically signed + + inCount = p.readVarint(); inputs = new Array(inCount); + for (i = 0; i < inCount; i++) { inputs[i] = Parser.parseInput(p); inputs[i].witness = { items: [] }; @@ -978,17 +994,25 @@ Parser.parseTX = function parseTX(p) { outCount = p.readVarint(); outputs = new Array(outCount); + for (i = 0; i < outCount; i++) outputs[i] = Parser.parseOutput(p); locktime = p.readU32(); + raw = p.endData(); + size = raw.length; + witnessSize = 0; + return { version: version, flag: 1, inputs: inputs, outputs: outputs, - locktime: locktime + locktime: locktime, + _raw: raw, + _size: size, + _witnessSize: witnessSize }; }; @@ -1023,9 +1047,12 @@ Parser.parseWitnessTX = function parseWitnessTX(p) { var outCount, outputs; var marker, flag; var version, locktime, i; + var raw, size, witnessSize, hasWitness; p = new BufferReader(p); + p.start(); + version = p.readU32(); // Technically signed marker = p.readU8(); @@ -1038,30 +1065,44 @@ Parser.parseWitnessTX = function parseWitnessTX(p) { throw new Error('Invalid witness tx (flag == 0)'); inCount = p.readVarint(); - inputs = new Array(inCount); + for (i = 0; i < inCount; i++) inputs[i] = Parser.parseInput(p); outCount = p.readVarint(); - outputs = new Array(outCount); + for (i = 0; i < outCount; i++) outputs[i] = Parser.parseOutput(p); + p.start(); + for (i = 0; i < inCount; i++) { witness = Parser.parseWitness(p); inputs[i].witness = witness; + if (witness.items.length > 0) + hasWitness = true; } + assert(hasWitness, 'Serialized wtx has an empty witness.'); + + witnessSize = p.end() + 2; + locktime = p.readU32(); + raw = p.endData(); + size = raw.length; + return { version: version, flag: flag, inputs: inputs, outputs: outputs, - locktime: locktime + locktime: locktime, + _raw: raw, + _size: size, + _witnessSize: witnessSize }; }; diff --git a/lib/bcoin/reader.js b/lib/bcoin/reader.js index 8f9fe3ba..d0c1cc07 100644 --- a/lib/bcoin/reader.js +++ b/lib/bcoin/reader.js @@ -72,9 +72,7 @@ BufferReader.prototype.start = function start() { /** * Stop reading. Pop the start position off the stack - * and calculate the size of the data read. This will - * destroy the BufferReader if no positions are left - * on the stack. + * and calculate the size of the data read. * @returns {Number} Size. * @throws on empty stack. */ @@ -87,22 +85,20 @@ BufferReader.prototype.end = function end() { start = this.stack.pop(); end = this.offset; - if (this.stack.length === 0) - this.destroy(); - return end - start; }; /** * Stop reading. Pop the start position off the stack - * and return the data read. This will - * destroy the BufferReader if no positions are left - * on the stack. + * and return the data read. + * @param {Bolean?} zeroCopy - Do a fast buffer + * slice instead of allocating a new buffer (warning: + * may cause memory leaks if not used with care). * @returns {Buffer} Data read. * @throws on empty stack. */ -BufferReader.prototype.endData = function endData() { +BufferReader.prototype.endData = function endData(zeroCopy) { var ret, start, end, size, data; assert(this.stack.length > 0); @@ -112,13 +108,10 @@ BufferReader.prototype.endData = function endData() { size = end - start; data = this.data; - if (this.stack.length === 0) - this.destroy(); - if (size === data.length) return data; - if (this.zeroCopy) + if (this.zeroCopy || zeroCopy) return data.slice(start, end); ret = new Buffer(size); @@ -482,16 +475,19 @@ BufferReader.prototype.readVarint = function readVarint(big) { /** * Read N bytes (will do a fast slice if zero copy). * @param {Number} size + * @param {Bolean?} zeroCopy - Do a fast buffer + * slice instead of allocating a new buffer (warning: + * may cause memory leaks if not used with care). * @returns {Buffer} */ -BufferReader.prototype.readBytes = function readBytes(size) { +BufferReader.prototype.readBytes = function readBytes(size, zeroCopy) { var ret; assert(size >= 0); assert(this.offset + size <= this.data.length); - if (this.zeroCopy) { + if (this.zeroCopy || zeroCopy) { ret = this.data.slice(this.offset, this.offset + size); } else { ret = new Buffer(size); @@ -505,11 +501,14 @@ BufferReader.prototype.readBytes = function readBytes(size) { /** * Read a varint number of bytes (will do a fast slice if zero copy). + * @param {Bolean?} zeroCopy - Do a fast buffer + * slice instead of allocating a new buffer (warning: + * may cause memory leaks if not used with care). * @returns {Buffer} */ -BufferReader.prototype.readVarBytes = function readVarBytes() { - return this.readBytes(this.readVarint()); +BufferReader.prototype.readVarBytes = function readVarBytes(zeroCopy) { + return this.readBytes(this.readVarint(), zeroCopy); }; /** diff --git a/lib/bcoin/tx.js b/lib/bcoin/tx.js index 09ce95ec..e8a9a3be 100644 --- a/lib/bcoin/tx.js +++ b/lib/bcoin/tx.js @@ -75,9 +75,11 @@ function TX(data) { this._hash = null; this._whash = null; - this._raw = null; - this._size = null; - this._witnessSize = null; + + this._raw = data._raw || null; + this._size = data._size || null; + this._witnessSize = data._witnessSize != null ? data._witnessSize : null; + this._outputValue = null; this._inputValue = null; this._hashPrevouts = null; diff --git a/lib/bcoin/utils.js b/lib/bcoin/utils.js index 315ebbbe..c1173dd7 100644 --- a/lib/bcoin/utils.js +++ b/lib/bcoin/utils.js @@ -1713,21 +1713,26 @@ utils.readVarint = function readVarint(data, off, big) { size = 3; assert(off + size <= data.length); value = data[off + 1] | (data[off + 2] << 8); + assert(value >= 0xfd); if (big) value = new bn(value); } else if (data[off] === 0xfe) { size = 5; assert(off + size <= data.length); value = data.readUInt32LE(off + 1, true); + assert(value > 0xffff); if (big) value = new bn(value); } else if (data[off] === 0xff) { size = 9; assert(off + size <= data.length); - if (big) + if (big) { value = utils.readU64(data, off + 1); - else + assert(value.bitLength() > 32); + } else { value = utils.readU64N(data, off + 1); + assert(value > 0xffffffff); + } } else { assert(false, 'Malformed varint.'); }