optimize parsing and serialization. strict parsing.

lib/bcoin/abstractblock.js

@@ -6,6 +6,7 @@
  */
 
 var bcoin = require('./env');
+var constants = bcoin.protocol.constants;
 var utils = bcoin.utils;
 var assert = utils.assert;
 

lib/bcoin/block.js

@@ -47,9 +47,13 @@ function Block(data) {
   bcoin.abstractblock.call(this, data);
 
   this.txs = [];
+
   this._cbHeight = null;
   this._commitmentHash = null;
-  this._raw = null;
+
+  this._raw = data._raw || null;
+  this._size = data._size || null;
+  this._witnessSize = data._witnessSize != null ? data._witnessSize : null;
 
   if (data.txs) {
     for (i = 0; i < data.txs.length; i++)

lib/bcoin/chain.js

@@ -1523,6 +1523,7 @@ Chain.prototype.add = function add(block, callback, force) {
           try {
             block = block.toBlock();
           } catch (e) {
+            bcoin.error(e);
             return done(new VerifyError(block,
               'malformed',
               'error parsing message',

lib/bcoin/compactblock.js

@@ -101,8 +101,6 @@ CompactBlock.prototype.getCoinbaseHeight = function getCoinbaseHeight() {
 CompactBlock.prototype.toBlock = function toBlock() {
   var data = bcoin.protocol.parser.parseBlock(this.raw);
   delete this.raw;
-  assert(!data.raw);
-  assert(!data._raw);
   return new bcoin.block(data);
 };
 

lib/bcoin/peer.js

@@ -1127,12 +1127,11 @@ Peer.prototype._handleGetData = function handleGetData(items) {
             return next();
         }
 
-        if (witness)
+        data = witness
-          data = tx.renderWitness();
+          ? self.framer.witnessTX(tx)
-        else
+          : self.framer.tx(tx);
-          data = tx.renderNormal();
 
-        self.write(self.framer.packet('tx', data));
+        self.write(data);
 
         next();
       });
@@ -1156,12 +1155,11 @@ Peer.prototype._handleGetData = function handleGetData(items) {
           return next();
         }
 
-        if (witness)
+        data = witness
-          data = block.renderWitness();
+          ? self.framer.witnessBlock(block)
-        else
+          : self.framer.block(block);
-          data = block.renderNormal();
 
-        self.write(self.framer.packet('block', data));
+        self.write(data);
 
         if (hash === self.hashContinue) {
           self.sendInv({
@@ -1200,12 +1198,11 @@ Peer.prototype._handleGetData = function handleGetData(items) {
         for (i = 0; i < block.txs.length; i++) {
           tx = block.txs[i];
 
-          if (witness)
+          tx = witness
-            tx = tx.renderWitness();
+            ? self.framer.witnessTX(tx)
-          else
+            : self.framer.tx(tx);
-            tx = tx.renderNormal();
 
-          self.write(self.framer.packet('tx', tx));
+          self.write(tx);
         }
 
         if (hash === self.hashContinue) {

lib/bcoin/pool.js

@@ -1570,9 +1570,7 @@ Pool.prototype.has = function has(type, hash, force, callback) {
     // If we recently rejected this item. Ignore.
     if (this.rejects.test(hash, 'hex')) {
       callback = utils.asyncify(callback);
-      bcoin.debug(
+      bcoin.debug('Peer sent a known reject: %s.', hash);
-        'Peer sent a known reject: %s.',
-        hash);
       return callback(null, true);
     }
   }

lib/bcoin/protocol/framer.js

@@ -41,7 +41,7 @@ function Framer(options) {
  * @returns {Buffer} Header.
  */
 
-Framer.prototype.header = function header(cmd, payload) {
+Framer.prototype.header = function header(cmd, payload, checksum) {
   var h = new Buffer(24);
   var len, i;
 
@@ -61,8 +61,11 @@ Framer.prototype.header = function header(cmd, payload) {
   // Payload length
   h.writeUInt32LE(payload.length, 16, true);
 
+  if (!checksum)
+    checksum = utils.dsha256(payload);
+
   // Checksum
-  utils.checksum(payload).copy(h, 20);
+  checksum.copy(h, 20, 0, 4);
 
   return h;
 };
@@ -74,12 +77,12 @@ Framer.prototype.header = function header(cmd, payload) {
  * @returns {Buffer} Payload with header prepended.
  */
 
-Framer.prototype.packet = function packet(cmd, payload) {
+Framer.prototype.packet = function packet(cmd, payload, checksum) {
   var header;
 
   assert(payload, 'No payload.');
 
-  header = this.header(cmd, payload);
+  header = this.header(cmd, payload, checksum);
 
   return Buffer.concat([header, payload]);
 };
@@ -304,7 +307,14 @@ Framer.prototype.getBlocks = function getBlocks(data) {
  */
 
 Framer.prototype.tx = function tx(tx) {
-  return this.packet('tx', Framer.renderTX(tx, false));
+  var checksum;
+
+  // Save some time by using the
+  // cached hash as our checksum.
+  if (tx.hash)
+    checksum = tx.hash();
+
+  return this.packet('tx', Framer.renderTX(tx, false), checksum);
 };
 
 /**
@@ -314,7 +324,22 @@ Framer.prototype.tx = function tx(tx) {
  */
 
 Framer.prototype.witnessTX = function witnessTX(tx) {
-  return this.packet('tx', Framer.renderTX(tx, true));
+  var checksum;
+
+  // Save some time by using the
+  // cached hash as our checksum.
+  if (tx.witnessHash) {
+    if (tx.hasWitness()) {
+      // We can't use the coinbase
+      // hash since it is all zeroes.
+      if (!tx.isCoinbase())
+        checksum = tx.witnessHash();
+    } else {
+      checksum = tx.hash();
+    }
+  }
+
+  return this.packet('tx', Framer.renderTX(tx, true), checksum);
 };
 
 /**
@@ -912,19 +937,45 @@ Framer.renderTX = function renderTX(tx, useWitness, writer) {
   var p = new BufferWriter(writer);
   var witnessSize;
 
+  // Cache the serialization if we can.
+  if (tx.render && !tx.mutable && !tx._raw)
+    tx.render();
+
+  // Try the cached raw data first.
   if (tx._raw) {
-    if (!useWitness && bcoin.protocol.parser.isWitnessTX(tx._raw)) {
+    if (useWitness) {
-      Framer.tx(tx, p);
+      // If we're serializing the witness,
-      witnessSize = p._witnessSize;
+      // we can use whatever data getRaw()
-    } else {
+      // gave us.
       p.writeBytes(tx._raw);
       witnessSize = tx._witnessSize;
+    } else {
+      // We have to use the standard format
+      // here. Try to grab it from cache.
+      if (bcoin.protocol.parser.isWitnessTX(tx._raw)) {
+        Framer.tx(tx, p);
+        witnessSize = p._witnessSize;
+      } else {
+        p.writeBytes(tx._raw);
+        witnessSize = tx._witnessSize;
+      }
     }
   } else {
-    if (useWitness && bcoin.tx.prototype.hasWitness.call(tx))
+    if (useWitness) {
-      Framer.witnessTX(tx, p);
+      if (bcoin.tx.prototype.hasWitness.call(tx)) {
-    else
+        Framer.witnessTX(tx, p);
+      } else {
+        // Only use the witness serialization if
+        // we have a witness. This clause isn't
+        // necessary above since we already
+        // determined this in getRaw().
+        Framer.tx(tx, p);
+      }
+    } else {
+      // Any other case, we use
+      // the standard serialization.
       Framer.tx(tx, p);
+    }
     witnessSize = p._witnessSize;
   }
 

lib/bcoin/protocol/network.js

@@ -871,6 +871,7 @@ segnet4.type = 'segnet4';
 segnet4.height = -1;
 
 segnet4.seeds = [
+  '104.243.38.34',
   '37.34.48.17'
 ];
 

lib/bcoin/protocol/parser.js

@@ -784,9 +784,12 @@ Parser.parseBlock = function parseBlock(p) {
   var txs = [];
   var version, prevBlock, merkleRoot, ts, bits, nonce;
   var i, totalTX, tx;
+  var raw, size, witnessSize;
 
   p = new BufferReader(p);
 
+  p.start();
+
   version = p.readU32(); // Technically signed
   prevBlock = p.readHash('hex');
   merkleRoot = p.readHash('hex');
@@ -795,11 +798,17 @@ Parser.parseBlock = function parseBlock(p) {
   nonce = p.readU32();
   totalTX = p.readVarint();
 
+  witnessSize = 0;
+
   for (i = 0; i < totalTX; i++) {
     tx = Parser.parseTX(p);
+    witnessSize += tx._witnessSize;
     txs.push(tx);
   }
 
+  raw = p.endData();
+  size = raw.length;
+
   return {
     version: version,
     prevBlock: prevBlock,
@@ -807,7 +816,10 @@ Parser.parseBlock = function parseBlock(p) {
     ts: ts,
     bits: bits,
     nonce: nonce,
-    txs: txs
+    txs: txs,
+    _raw: raw,
+    _size: size,
+    _witnessSize: witnessSize
   };
 };
 
@@ -961,16 +973,20 @@ Parser.parseTX = function parseTX(p) {
   var inCount, inputs;
   var outCount, outputs;
   var version, locktime, i;
+  var raw, size, witnessSize;
 
   if (Parser.isWitnessTX(p))
     return Parser.parseWitnessTX(p);
 
   p = new BufferReader(p);
 
-  version = p.readU32(); // Technically signed
+  p.start();
-  inCount = p.readVarint();
 
+  version = p.readU32(); // Technically signed
+
+  inCount = p.readVarint();
   inputs = new Array(inCount);
+
   for (i = 0; i < inCount; i++) {
     inputs[i] = Parser.parseInput(p);
     inputs[i].witness = { items: [] };
@@ -978,17 +994,25 @@ Parser.parseTX = function parseTX(p) {
 
   outCount = p.readVarint();
   outputs = new Array(outCount);
+
   for (i = 0; i < outCount; i++)
     outputs[i] = Parser.parseOutput(p);
 
   locktime = p.readU32();
 
+  raw = p.endData();
+  size = raw.length;
+  witnessSize = 0;
+
   return {
     version: version,
     flag: 1,
     inputs: inputs,
     outputs: outputs,
-    locktime: locktime
+    locktime: locktime,
+    _raw: raw,
+    _size: size,
+    _witnessSize: witnessSize
   };
 };
 
@@ -1023,9 +1047,12 @@ Parser.parseWitnessTX = function parseWitnessTX(p) {
   var outCount, outputs;
   var marker, flag;
   var version, locktime, i;
+  var raw, size, witnessSize, hasWitness;
 
   p = new BufferReader(p);
 
+  p.start();
+
   version = p.readU32(); // Technically signed
   marker = p.readU8();
 
@@ -1038,30 +1065,44 @@ Parser.parseWitnessTX = function parseWitnessTX(p) {
     throw new Error('Invalid witness tx (flag == 0)');
 
   inCount = p.readVarint();
-
   inputs = new Array(inCount);
+
   for (i = 0; i < inCount; i++)
     inputs[i] = Parser.parseInput(p);
 
   outCount = p.readVarint();
-
   outputs = new Array(outCount);
+
   for (i = 0; i < outCount; i++)
     outputs[i] = Parser.parseOutput(p);
 
+  p.start();
+
   for (i = 0; i < inCount; i++) {
     witness = Parser.parseWitness(p);
     inputs[i].witness = witness;
+    if (witness.items.length > 0)
+      hasWitness = true;
   }
 
+  assert(hasWitness, 'Serialized wtx has an empty witness.');
+
+  witnessSize = p.end() + 2;
+
   locktime = p.readU32();
 
+  raw = p.endData();
+  size = raw.length;
+
   return {
     version: version,
     flag: flag,
     inputs: inputs,
     outputs: outputs,
-    locktime: locktime
+    locktime: locktime,
+    _raw: raw,
+    _size: size,
+    _witnessSize: witnessSize
   };
 };
 

lib/bcoin/reader.js

@@ -72,9 +72,7 @@ BufferReader.prototype.start = function start() {
 
 /**
  * Stop reading. Pop the start position off the stack
- * and calculate the size of the data read. This will
+ * and calculate the size of the data read.
- * destroy the BufferReader if no positions are left
- * on the stack.
  * @returns {Number} Size.
  * @throws on empty stack.
  */
@@ -87,22 +85,20 @@ BufferReader.prototype.end = function end() {
   start = this.stack.pop();
   end = this.offset;
 
-  if (this.stack.length === 0)
-    this.destroy();
-
   return end - start;
 };
 
 /**
  * Stop reading. Pop the start position off the stack
- * and return the data read. This will
+ * and return the data read.
- * destroy the BufferReader if no positions are left
+ * @param {Bolean?} zeroCopy - Do a fast buffer
- * on the stack.
+ * slice instead of allocating a new buffer (warning:
+ * may cause memory leaks if not used with care).
  * @returns {Buffer} Data read.
  * @throws on empty stack.
  */
 
-BufferReader.prototype.endData = function endData() {
+BufferReader.prototype.endData = function endData(zeroCopy) {
   var ret, start, end, size, data;
 
   assert(this.stack.length > 0);
@@ -112,13 +108,10 @@ BufferReader.prototype.endData = function endData() {
   size = end - start;
   data = this.data;
 
-  if (this.stack.length === 0)
-    this.destroy();
-
   if (size === data.length)
     return data;
 
-  if (this.zeroCopy)
+  if (this.zeroCopy || zeroCopy)
     return data.slice(start, end);
 
   ret = new Buffer(size);
@@ -482,16 +475,19 @@ BufferReader.prototype.readVarint = function readVarint(big) {
 /**
  * Read N bytes (will do a fast slice if zero copy).
  * @param {Number} size
+ * @param {Bolean?} zeroCopy - Do a fast buffer
+ * slice instead of allocating a new buffer (warning:
+ * may cause memory leaks if not used with care).
  * @returns {Buffer}
  */
 
-BufferReader.prototype.readBytes = function readBytes(size) {
+BufferReader.prototype.readBytes = function readBytes(size, zeroCopy) {
   var ret;
 
   assert(size >= 0);
   assert(this.offset + size <= this.data.length);
 
-  if (this.zeroCopy) {
+  if (this.zeroCopy || zeroCopy) {
     ret = this.data.slice(this.offset, this.offset + size);
   } else {
     ret = new Buffer(size);
@@ -505,11 +501,14 @@ BufferReader.prototype.readBytes = function readBytes(size) {
 
 /**
  * Read a varint number of bytes (will do a fast slice if zero copy).
+ * @param {Bolean?} zeroCopy - Do a fast buffer
+ * slice instead of allocating a new buffer (warning:
+ * may cause memory leaks if not used with care).
  * @returns {Buffer}
  */
 
-BufferReader.prototype.readVarBytes = function readVarBytes() {
+BufferReader.prototype.readVarBytes = function readVarBytes(zeroCopy) {
-  return this.readBytes(this.readVarint());
+  return this.readBytes(this.readVarint(), zeroCopy);
 };
 
 /**

lib/bcoin/tx.js

@@ -75,9 +75,11 @@ function TX(data) {
 
   this._hash = null;
   this._whash = null;
-  this._raw = null;
+
-  this._size = null;
+  this._raw = data._raw || null;
-  this._witnessSize = null;
+  this._size = data._size || null;
+  this._witnessSize = data._witnessSize != null ? data._witnessSize : null;
+
   this._outputValue = null;
   this._inputValue = null;
   this._hashPrevouts = null;

lib/bcoin/utils.js

@@ -1713,21 +1713,26 @@ utils.readVarint = function readVarint(data, off, big) {
     size = 3;
     assert(off + size <= data.length);
     value = data[off + 1] | (data[off + 2] << 8);
+    assert(value >= 0xfd);
     if (big)
       value = new bn(value);
   } else if (data[off] === 0xfe) {
     size = 5;
     assert(off + size <= data.length);
     value = data.readUInt32LE(off + 1, true);
+    assert(value > 0xffff);
     if (big)
       value = new bn(value);
   } else if (data[off] === 0xff) {
     size = 9;
     assert(off + size <= data.length);
-    if (big)
+    if (big) {
       value = utils.readU64(data, off + 1);
-    else
+      assert(value.bitLength() > 32);
+    } else {
       value = utils.readU64N(data, off + 1);
+      assert(value > 0xffffffff);
+    }
   } else {
     assert(false, 'Malformed varint.');
   }