From c1a9655943ed083de119a62c521713d04b3a85be Mon Sep 17 00:00:00 2001 From: Christopher Jeffrey Date: Tue, 15 Mar 2016 01:44:16 -0700 Subject: [PATCH] framer using bufferwriter. --- lib/bcoin/protocol/framer.js | 587 ++++++++++++----------------------- 1 file changed, 192 insertions(+), 395 deletions(-) diff --git a/lib/bcoin/protocol/framer.js b/lib/bcoin/protocol/framer.js index 2c135b2e..4a3a0720 100644 --- a/lib/bcoin/protocol/framer.js +++ b/lib/bcoin/protocol/framer.js @@ -147,9 +147,8 @@ Framer.prototype.mempool = function mempool() { return this.packet('mempool', Framer.mempool()); }; -Framer.address = function addr(data, full) { - var p = new Buffer(26 + (full ? 4 : 0)); - var off = 0; +Framer.address = function address(data, full, writer) { + var p = new BufferWriter(writer); if (!data) data = {}; @@ -166,44 +165,38 @@ Framer.address = function addr(data, full) { if (!data.port) data.port = network.port; - // timestamp if (full) - off += utils.writeU32(p, data.ts, off); + p.writeU32(data.ts); - // NODE_NETWORK services - off += utils.writeU64(p, data.services, off); + p.writeU64(data.services); - // ipv6 if (data.ipv6) { data.ipv6 = utils.ip2array(data.ipv6, 6); - off += utils.writeU64BE(p, utils.readU64BE(data.ipv6, 0), off); - off += utils.writeU64BE(p, utils.readU64BE(data.ipv6, 8), off); + p.writeBytes(data.ipv6); } else { data.ipv4 = utils.ip2array(data.ipv4, 4); // We don't have an ipv6, convert ipv4 to ipv4-mapped ipv6 address - off += utils.writeU32BE(p, 0x00000000, off); - off += utils.writeU32BE(p, 0x00000000, off); - off += utils.writeU32BE(p, 0x0000ffff, off); - off += utils.writeU32BE(p, utils.readU32BE(data.ipv4, 0), off); + p.writeU32BE(0x00000000); + p.writeU32BE(0x00000000); + p.writeU32BE(0x0000ffff); + p.writeBytes(data.ipv4); } - // port - off += utils.writeU16BE(p, data.port, off); + p.writeU16BE(data.port); + + if (!writer) + p = p.render(); return p; }; -Framer.version = function version(options) { - var off = 0; - var p, i, remote, local; +Framer.version = function version(options, writer) { + var p = new BufferWriter(writer); + var i, remote, local; if (!options.agent) options.agent = new Buffer(constants.userAgent, 'ascii'); - p = new Buffer(85 - + utils.sizeIntv(options.agent.length) - + options.agent.length); - if (!options) options = {}; @@ -216,39 +209,18 @@ Framer.version = function version(options) { if (options.local.services == null) options.local.services = constants.bcoinServices; - // Version - off += utils.writeU32(p, constants.version, off); + p.writeU32(constants.version); + p.writeU64(constants.bcoinServices); + p.write64(utils.now()); + Framer.address(options.remote, false, p); + Framer.address(options.local, false, p); + p.writeU64(utils.nonce()); + p.writeVarString(options.agent); + p.writeU32(options.height || 0); + p.writeU8(options.relay ? 1 : 0); - // Services - off += utils.writeU64(p, constants.bcoinServices, off); - - // Timestamp - off += utils.write64(p, utils.now(), off); - - // Their address (recv) - remote = Framer.address(options.remote); - off += utils.copy(remote, p, off); - - // Our address (from) - local = Framer.address(options.local); - off += utils.copy(local, p, off); - - // Nonce, very dramatic - off += utils.writeU64(p, utils.nonce(), off); - - assert.equal(off, 80); - - // User-agent - off += utils.writeIntv(p, options.agent.length, off); - off += utils.copy(options.agent, p, off); - - // Start height - off += utils.writeU32(p, options.height || 0, off); - - // Relay - p[off++] = options.relay ? 1 : 0; - - assert(off === p.length); + if (!writer) + p = p.render(); return p; }; @@ -257,41 +229,36 @@ Framer.verack = function verack() { return new Buffer([]); }; -Framer._inv = function _inv(items) { - var p, i, hash; +Framer._inv = function _inv(items, writer) { + var p = new BufferWriter(writer); + var i, hash; var off = 0; assert(items.length <= 50000); - p = new Buffer(utils.sizeIntv(items.length) + items.length * 36); - - off += utils.writeIntv(p, items.length, off); + p.writeIntv(items.length); for (i = 0; i < items.length; i++) { - // Type - off += utils.writeU32(p, constants.inv[items[i].type], off); - - // Hash - hash = items[i].hash; - if (typeof hash === 'string') - hash = new Buffer(hash, 'hex'); - assert.equal(hash.length, 32); - off += utils.copy(hash, p, off); + p.writeU32(constants.inv[items[i].type]); + p.writeHash(items[i].hash); } + if (!writer) + p = p.render(); + return p; }; -Framer.inv = function inv(items) { - return Framer._inv(items); +Framer.inv = function inv(items, writer) { + return Framer._inv(items, writer); }; -Framer.getData = function getData(items) { - return Framer._inv(items); +Framer.getData = function getData(items, writer) { + return Framer._inv(items, writer); }; -Framer.notFound = function notFound(items) { - return Framer._inv(items); +Framer.notFound = function notFound(items, writer) { + return Framer._inv(items, writer); }; Framer.ping = function ping(data) { @@ -306,22 +273,16 @@ Framer.pong = function pong(data) { return p; }; -Framer.filterLoad = function filterLoad(bloom, update) { - var filter = bloom.toBuffer(); - var p = new Buffer(utils.sizeIntv(filter.length) + filter.length + 9); - var off = 0; +Framer.filterLoad = function filterLoad(bloom, update, writer) { + var p = new BufferWriter(writer); - off += utils.writeIntv(p, filter.length, off); - off += utils.copy(filter, p, off); + p.writeVarBytes(bloom.toBuffer()); + p.writeU32(bloom.n); + p.writeU32(bloom.tweak); + p.writeU8(constants.filterFlags[update]); - // Number of hash functions - off += utils.writeU32(p, bloom.n, off); - - // nTweak - off += utils.writeU32(p, bloom.tweak, off); - - // nFlags - p[off++] = constants.filterFlags[update]; + if (!writer) + p = p.render(); return p; }; @@ -330,56 +291,37 @@ Framer.filterClear = function filterClear() { return new Buffer([]); }; -Framer.getHeaders = function getHeaders(hashes, stop) { +Framer.getHeaders = function getHeaders(hashes, stop, writer) { // NOTE: getheaders can have a null hash if (!hashes) hashes = []; - return Framer._getBlocks(hashes, stop); + return Framer._getBlocks(hashes, stop, writer); }; -Framer.getBlocks = function getBlocks(hashes, stop) { - return Framer._getBlocks(hashes, stop); +Framer.getBlocks = function getBlocks(hashes, stop, writer) { + return Framer._getBlocks(hashes, stop, writer); }; -Framer._getBlocks = function _getBlocks(hashes, stop) { - var p, i, hash, len; - var off = 0; +Framer._getBlocks = function _getBlocks(hashes, stop, writer) { + var p = new BufferWriter(writer); + var i; - p = new Buffer(4 + utils.sizeIntv(hashes.length) + 32 * (hashes.length + 1)); + p.writeU32(constants.version); + p.writeIntv(hashes.length); - off += utils.writeU32(p, constants.version, off); - off += utils.writeIntv(p, hashes.length, off); + for (i = 0; i < hashes.length; i++) + p.writeHash(hashes[i]); - for (i = 0; i < hashes.length; i++) { - hash = hashes[i]; + p.writeHash(stop || constants.zeroHash); - if (typeof hash === 'string') - hash = new Buffer(hash, 'hex'); - - len = utils.copy(hash, p, off); - assert(len === 32); - - off += len; - } - - if (stop) { - if (typeof stop === 'string') - stop = new Buffer(stop, 'hex'); - len = utils.copy(stop, p, off); - assert(len === 32); - } else { - len = 0; - for (; len < 32; len++) - p[off + len] = 0; - } - - assert.equal(off + len, p.length); + if (!writer) + p = p.render(); return p; }; -Framer.input = function _input(input) { +Framer.binput = function _input(input) { var off = 0; var script, p; @@ -397,7 +339,7 @@ Framer.input = function _input(input) { return p; }; -Framer.output = function _output(output) { +Framer.boutput = function _output(output) { var off = 0; var script, p; @@ -414,35 +356,33 @@ Framer.output = function _output(output) { }; Framer.utxo = -Framer.coin = function _coin(coin, extended) { - var script = coin.script.encode(); - var intSize = utils.sizeIntv(script.length); +Framer.coin = function _coin(coin, extended, writer) { + var p = new BufferWriter(writer); var height = coin.height; - var off = 0; - var data; - - data = new Buffer(16 + intSize + script.length + (extended ? 37 : 0)); if (height === -1) height = 0x7fffffff; - off += utils.writeU32(data, coin.version, off); - off += utils.writeU32(data, height, off); - off += utils.write64(data, coin.value, off); assert(coin.value.byteLength() <= 8); - off += utils.writeIntv(data, script.length, off); - off += utils.copy(script, data, off); + + p.writeU32(coin.version); + p.writeU32(height); + p.write64(coin.value); + p.writeVarBytes(coin.script.encode()); if (extended) { - off += utils.copy(new Buffer(coin.hash, 'hex'), data, off); - off += utils.writeU32(data, coin.index, off); - off += utils.writeU8(data, coin.spent ? 1 : 0, off); + p.writeHash(coin.hash); + p.writeU32(coin.index); + p.writeU8(coin.spent ? 1 : 0); } - return data; + if (!writer) + p = p.render(); + + return p; }; -Framer.tx = function _tx(tx) { +Framer.btx = function _tx(tx) { var inputs = []; var outputs = []; var inputSize = 0; @@ -451,13 +391,13 @@ Framer.tx = function _tx(tx) { var p, i, input, output; for (i = 0; i < tx.inputs.length; i++) { - input = Framer.input(tx.inputs[i]); + input = Framer.binput(tx.inputs[i]); inputs.push(input); inputSize += input.length; } for (i = 0; i < tx.outputs.length; i++) { - output = Framer.output(tx.outputs[i]); + output = Framer.boutput(tx.outputs[i]); outputs.push(output); outputSize += output.length; } @@ -488,18 +428,18 @@ Framer.tx = function _tx(tx) { return p; }; -Framer.btx = function btx(tx, writer) { +Framer.tx = function _tx(tx, writer) { var p = new BufferWriter(writer); p.write32(tx.version); p.writeUIntv(tx.inputs.length); for (i = 0; i < tx.inputs.length; i++) - Framer.binput(tx.inputs[i], p); + Framer.input(tx.inputs[i], p); p.writeUIntv(tx.outputs.length); for (i = 0; i < tx.outputs.length; i++) - Framer.boutput(tx.outputs[i], p); + Framer.output(tx.outputs[i], p); p.writeU32(tx.locktime); @@ -511,7 +451,7 @@ Framer.btx = function btx(tx, writer) { return p; }; -Framer.binput = function _input(input, writer) { +Framer.input = function _input(input, writer) { var p = new BufferWriter(writer); p.writeHash(input.prevout.hash); @@ -525,7 +465,7 @@ Framer.binput = function _input(input, writer) { return p; }; -Framer.boutput = function _output(output, writer) { +Framer.output = function _output(output, writer) { var p = new BufferWriter(writer); assert(output.value.byteLength() <= 8); @@ -539,63 +479,34 @@ Framer.boutput = function _output(output, writer) { return p; }; -Framer.witnessTX = function _witnessTX(tx) { - var inputs = []; - var outputs = []; - var witnesses = []; - var inputSize = 0; - var outputSize = 0; +Framer.witnessTX = function _witnessTX(tx, writer) { + var p = new BufferWriter(writer); var witnessSize = 0; - var off = 0; - var p, i, input, output, witness; + + p.write32(tx.version); + p.writeU8(0); + p.writeU8(tx.flag || 1); + + p.writeUIntv(tx.inputs.length); + + for (i = 0; i < tx.inputs.length; i++) + Framer.input(tx.inputs[i], p); + + p.writeUIntv(tx.outputs.length); + + for (i = 0; i < outputs.length; i++) + Framer.output(tx.outputs[i], p); for (i = 0; i < tx.inputs.length; i++) { - input = Framer.input(tx.inputs[i]); - inputs.push(input); - inputSize += input.length; - witness = Framer.witness(tx.inputs[i].witness); - witnesses.push(witness); - witnessSize += witness.length; + var start = p.written; + Framer.witness(tx.inputs[i].witness, p); + witnessSize += p.written - start; } - for (i = 0; i < tx.outputs.length; i++) { - output = Framer.output(tx.outputs[i]); - outputs.push(output); - outputSize += output.length; - } + p.writeU32(tx.locktime); - p = new Buffer(4 - + 2 - + utils.sizeIntv(tx.inputs.length) + inputSize - + utils.sizeIntv(tx.outputs.length) + outputSize - + witnessSize - + 4); - - off += utils.write32(p, tx.version, off); - - // marker - off += utils.writeU8(p, 0, off); - - // flag - off += utils.writeU8(p, tx.flag || 1, off); - - off += utils.writeIntv(p, tx.inputs.length, off); - for (i = 0; i < inputs.length; i++) { - input = inputs[i]; - off += utils.copy(input, p, off); - } - - off += utils.writeIntv(p, tx.outputs.length, off); - for (i = 0; i < outputs.length; i++) { - output = outputs[i]; - off += utils.copy(output, p, off); - } - - // NOTE: No varint item count here. - for (i = 0; i < witnesses.length; i++) - off += utils.copy(witnesses[i], p, off); - - off += utils.writeU32(p, tx.locktime, off); + if (!writer) + p = p.render(); p._witnessSize = witnessSize + 2; @@ -603,266 +514,153 @@ Framer.witnessTX = function _witnessTX(tx) { }; Framer.witnessBlockSize = function witnessBlockSize(block) { - var size = 0; - var i; - - for (i = 0; i < block.txs.length; i++) - size += Framer.witnessTXSize(block.txs[i]); - - return size; + return Framer.witnessBlock(block, new BufferWriter())._witnessSize; }; Framer.witnessTXSize = function witnessTXSize(tx) { - var off = 0; - var size = 0; - var i, j, witness; - - if (!tx.hasWitness()) - return size; - - for (i = 0; i < tx.inputs.length; i++) { - witness = tx.inputs[i].witness; - - if (!witness) - continue; - - size += utils.sizeIntv(witness.items.length); - - for (j = 0; j < witness.items.length; j++) { - chunk = witness.items[j]; - size += utils.sizeIntv(chunk.length) + chunk.length; - } - } - - return size + 2; + return Framer.witnessTXSize(block, new BufferWriter())._witnessSize; }; -Framer.witness = function _witness(witness) { - var off = 0; - var size = 0; - var p, chunk; +Framer.witness = function _witness(witness, writer) { + var p = new BufferWriter(writer); - if (!witness) - return new Buffer([0]); + p.writeUIntv(witness.items.length); - size += utils.sizeIntv(witness.items.length); + for (i = 0; i < witness.items.length; i++) + p.writeVarBytes(witness.items[i]); - for (i = 0; i < witness.items.length; i++) { - chunk = witness.items[i]; - size += utils.sizeIntv(chunk.length) + chunk.length; - } - - p = new Buffer(size); - - off += utils.writeIntv(p, witness.items.length, off); - - for (i = 0; i < witness.items.length; i++) { - chunk = witness.items[i]; - off += utils.writeIntv(p, chunk.length, off); - off += utils.copy(chunk, p, off); - } + if (!writer) + p = p.render(); return p; }; -Framer.block = function _block(block) { - return Framer._block(block, false); +Framer.block = function _block(block, writer) { + return Framer._block(block, false, writer); }; -Framer.witnessBlock = function _witnessBlock(block) { - return Framer._block(block, true); +Framer.witnessBlock = function _witnessBlock(block, writer) { + return Framer._block(block, true, writer); }; -Framer._block = function _block(block, useWitness) { - var off = 0; - var txSize = 0; +Framer._block = function _block(block, useWitness, writer) { + var p = new BufferWriter(writer); var witnessSize = 0; - var txs = []; - var i, tx, p; + var i, tx; + + p.write32(block.version); + p.writeHash(block.prevBlock); + p.writeHash(block.merkleRoot); + p.writeU32(block.ts); + p.writeU32(block.bits); + p.writeU32(block.nonce); + p.writeIntv(block.txs.length); for (i = 0; i < block.txs.length; i++) { tx = block.txs[i]; - if (tx.render) { - p = useWitness - ? tx.render() - : tx.renderNormal(); - witnessSize += tx._witnessSize; - tx = p; - p = null; + if (tx._raw) { + if (!useWitness && bcoin.protocol.parser.isWitnessTX(tx._raw)) { + Framer.tx(tx, p); + witnessSize += p._witnessSize; + } else { + p.writeBytes(tx._raw); + witnessSize += tx._witnessSize; + } } else { - tx = useWitness && bcoin.tx.prototype.hasWitness.call(block.txs[i]) - ? Framer.witnessTX(block.txs[i]) - : Framer.tx(block.txs[i]); - witnessSize += tx._witnessSize; + if (useWitness && bcoin.tx.prototype.hasWitness.call(tx)) + Framer.witnessTX(tx, p); + else + Framer.tx(tx, p); + witnessSize += p._witnessSize; } - - txs.push(tx); - txSize += tx.length; } - p = new Buffer(80 + utils.sizeIntv(block.txs.length) + txSize); - - // version - off += utils.write32(p, block.version, off); - - // prev_block - off += utils.copy(new Buffer(block.prevBlock, 'hex'), p, off); - - // merkle_root - off += utils.copy(new Buffer(block.merkleRoot, 'hex'), p, off); - - // timestamp - off += utils.writeU32(p, block.ts, off); - - // bits - off += utils.writeU32(p, block.bits, off); - - // nonce - off += utils.writeU32(p, block.nonce, off); - - assert.equal(off, 80); - - // txn_count - off += utils.writeIntv(p, block.txs.length, off); - - // txs - for (i = 0; i < txs.length; i++) - off += utils.copy(txs[i], p, off); + if (!writer) + p = p.render(); p._witnessSize = witnessSize; return p; }; -Framer.merkleBlock = function _merkleBlock(block) { - var off = 0; - var p, i; +Framer.merkleBlock = function _merkleBlock(block, writer) { + var p = new BufferWriter(writer); - p = new Buffer(80 + 4 - + utils.sizeIntv(block.hashes.length) + (block.hashes.length * 32) - + utils.sizeIntv(block.flags.length) + block.flags.length); + p.write32(block.version); + p.writeHash(block.prevBlock); + p.writeHash(block.merkleRoot); + p.writeU32(block.ts); + p.writeU32(block.bits); + p.writeU32(block.nonce); + p.writeU32(block.totalTX); - // version - off += utils.write32(p, block.version, off); + p.writeIntv(block.hashes.length); - // prev_block - off += utils.copy(new Buffer(block.prevBlock, 'hex'), p, off); - - // merkle_root - off += utils.copy(new Buffer(block.merkleRoot, 'hex'), p, off); - - // timestamp - off += utils.writeU32(p, block.ts, off); - - // bits - off += utils.writeU32(p, block.bits, off); - - // nonce - off += utils.writeU32(p, block.nonce, off); - - assert.equal(off, 80); - - // txn_count - off += utils.writeU32(p, block.totalTX, off); - - // hash count - off += utils.writeIntv(p, block.hashes.length, off); - - // hashes for (i = 0; i < block.hashes.length; i++) - off += utils.copy(new Buffer(block.hashes[i], 'hex'), p, off); + p.writeHash(block.hashes[i]); - // flag count - off += utils.writeIntv(p, block.flags.length, off); + p.writeVarBytes(block.flags); - // flags - for (i = 0; i < block.flags.length; i++) - p[off++] = block.flags[i]; + if (!writer) + p = p.render(); return p; }; -Framer.headers = function _headers(block) { - var off = 0; - var p, i; +Framer.headers = function _headers(block, writer) { + var p = new BufferWriter(writer); + var i; - p = new Buffer(80 + utils.sizeIntv(data.totalTX)); + p.write32(block.version); + p.writeHash(block.prevBlock); + p.writeHash(block.merkleRoot); + p.writeU32(block.ts); + p.writeU32(block.bits); + p.writeU32(block.nonce); + p.writeIntv(block.totalTX); - // version - off += utils.write32(p, block.version, off); - - // prev_block - off += utils.copy(new Buffer(block.prevBlock, 'hex'), p, off); - - // merkle_root - off += utils.copy(new Buffer(block.merkleRoot, 'hex'), p, off); - - // timestamp - off += utils.writeU32(p, block.ts, off); - - // bits - off += utils.writeU32(p, block.bits, off); - - // nonce - off += utils.writeU32(p, block.nonce, off); - - assert.equal(off, 80); - - // txn_count - off += utils.writeIntv(p, data.totalTX, off); + if (!writer) + p = p.render(); return p; }; -Framer.reject = function reject(details) { - var message = new Buffer(details.message || '', 'ascii'); - var ccode = constants.reject[details.ccode] || constants.reject.malformed; - var reason = new Buffer(details.reason || '', 'ascii'); - var data = details.data || new Buffer([]); - var p = new Buffer( - utils.sizeIntv(message.length) + message.length - + 1 - + utils.sizeIntv(reason.length) + reason.length - + data.length); - var off = 0; +Framer.reject = function reject(details, writer) { + var p = new BufferWriter(writer); - off += utils.writeIntv(p, message.length, off); - off += utils.copy(message, p, off); + p.writeVarString(details.message || '', 'ascii'); + p.writeU8(ccode, off); + p.writeVarString(details.reason || '', 'ascii'); + if (details.data) + p.writeBytes(details.data); - off += utils.writeU8(p, ccode, off); - - off += utils.writeIntv(p, reason.length, off); - off += utils.copy(reason, p, off); - - off += utils.copy(data, p, off); + if (!writer) + p = p.render(); return p; }; -Framer.addr = function addr(peers) { - var p = new Buffer(utils.sizeIntv(peers.length) + peers.length * 30); - var off = 0; - var addrs = []; - var addr; +Framer.addr = function addr(peers, writer) { + var p = new BufferWriter(writer); var i, peer; - off += utils.writeIntv(p, peers.length, off); + p.writeIntv(peers.length); for (i = 0; i < peers.length; i++) { peer = peers[i]; - - addr = Framer.address({ + Framer.address({ ts: peer.ts, services: peer.services, ipv6: peer.ipv6, ipv4: peer.ipv4, port: peer.port - }, true); - - off += addr.copy(p, off, 0, addr.length); + }, true, p); } + if (!writer) + p = p.render(); + return p; }; @@ -879,7 +677,6 @@ function BufferWriter(options) { this.data = []; this.written = 0; - this.offset = 0; } BufferWriter.prototype.render = function render() {