more coin serialization.

lib/bcoin/chain.js

@@ -322,7 +322,7 @@ Chain.prototype.preload = function preload(callback) {
     size = 0;
 
     stream.on('response', function(res) {
-      var height = Math.floor(res.headers['content-length'] / 80);
+      var height = Math.floor(+res.headers['content-length'] / 80);
       if (res.statusCode >= 400) {
         stream.destroy();
         return callback(new Error('Bad response code: ' + res.statusCode));

lib/bcoin/coin.js

@@ -262,6 +262,77 @@ Coin.fromRaw = function fromRaw(data, enc) {
   return new Coin().fromRaw(data);
 };
 
+/**
+ * Serialize the coin to its compressed form.
+ * @param {String?} enc - Encoding, can be `'hex'` or null.
+ * @returns {Buffer|String}
+ */
+
+Coin.prototype.toCompressed = function toCompressed(writer) {
+  var compress = bcoin.coins.compress;
+  var p = bcoin.writer(writer);
+  var height = this.height;
+  var bits;
+
+  if (height === -1)
+    height = 0x7fffffff;
+
+  bits = height << 1;
+
+  if (this.coinbase)
+    bits |= 1;
+
+  if (bits < 0)
+    bits += 0x100000000;
+
+  p.writeVarint(this.version);
+  p.writeU32(bits);
+  p.writeVarint(this.value);
+  compress.script(this.script, p);
+
+  if (!writer)
+    p = p.render();
+
+  return p;
+};
+
+/**
+ * Inject properties from compressed serialized data.
+ * @private
+ * @param {Buffer} data
+ */
+
+Coin.prototype.fromCompressed = function fromCompressed(data) {
+  var decompress = bcoin.coins.decompress;
+  var p = bcoin.reader(data);
+  var bits;
+
+  this.version = p.readVarint();
+  bits = p.readU32();
+  this.height = bits >>> 1;
+  this.coinbase = (bits & 1) !== 0;
+  this.value = p.readVarint();
+  decompress.script(p, this.script);
+
+  if (this.height === 0x7fffffff)
+    this.height = -1;
+
+  return this;
+};
+
+/**
+ * Instantiate an coin from a serialized Buffer.
+ * @param {Buffer} data
+ * @param {String?} enc - Encoding, can be `'hex'` or null.
+ * @returns {Coin}
+ */
+
+Coin.fromCompressed = function fromCompressed(data, enc) {
+  if (typeof data === 'string')
+    data = new Buffer(data, enc);
+  return new Coin().fromCompressed(data);
+};
+
 /**
  * Serialize the coin to an "extended" format,
  * including both the hash and the index.

lib/bcoin/coins.js

@@ -152,6 +152,9 @@ Coins.prototype.count = function count(index) {
 
 Coins.prototype.spend = function spend(index) {
   var coin = this.get(index);
+  if (!coin)
+    return;
+
   this.outputs[index] = null;
   return coin;
 };
@@ -161,30 +164,42 @@ Coins.prototype.spend = function spend(index) {
  * @returns {Number}
  */
 
-Coins.prototype.getLength = function getLength() {
+Coins.prototype.size = function size() {
-  var last = -1;
+  var index = -1;
   var i;
 
-  for (i = 0; i < this.outputs.length; i++) {
+  for (i = this.outputs.length - 1; i >= 0; i--) {
-    if (this.outputs[i])
+    if (this.outputs[i]) {
-      last = i;
+      index = i;
+      break;
+    }
   }
 
-  return last + 1;
+  return index + 1;
 };
 
 /*
  * Coins serialization:
  * version: varint
- * bits: varint ((height << 1) | coinbase-flag)
+ * bits: uint32 (31-bit height | 1-bit coinbase-flag)
  * spent-field: varint size | bitfield (0=unspent, 1=spent)
  * outputs (repeated):
- *   prefix: 0x00 = varint size | raw script
+ *   compressed-script:
- *           0x01 = 20 byte pubkey hash
+ *     prefix: 0x00 = varint size | raw script
- *           0x02 = 20 byte script hash
+ *             0x01 = 20 byte pubkey hash
- *           0x03 = 33 byte compressed key
+ *             0x02 = 20 byte script hash
- *   data: the data mentioned above
+ *             0x03 = 33 byte compressed key
+ *     data: script data, dictated by the prefix
  *   value: varint
+ *
+ * The compression below sacrifices some cpu in exchange
+ * for reduced size, but in some cases the use of varints
+ * actually increases speed (varint versions and values
+ * for example). We do as much compression as possible
+ * without sacrificing too much cpu. Value compression
+ * is intentionally excluded for now as it seems to be
+ * too much of a perf hit. Maybe when v8 optimizes
+ * non-smi arithmetic better we can enable it.
  */
 
 /**
@@ -196,8 +211,8 @@ Coins.prototype.getLength = function getLength() {
 Coins.prototype.toRaw = function toRaw(writer) {
   var p = new BufferWriter();
   var height = this.height;
-  var length = this.getLength();
+  var length = this.size();
-  var i, output, prefix, data, bits, fstart, flen, bit, oct;
+  var i, output, bits, fstart, flen, bit, oct;
 
   // Varint version: hopefully some smartass
   // miner doesn't start mining `-1` versions.
@@ -231,9 +246,9 @@ Coins.prototype.toRaw = function toRaw(writer) {
   // Fill the spent field with zeroes to avoid
   // allocating a buffer. We mark the spents
   // after rendering the final buffer.
-  fstart = p.written;
   flen = Math.ceil(length / 8);
   p.writeVarint(flen);
+  fstart = p.written;
   p.fill(0, flen);
 
   for (i = 0; i < length; i++) {
@@ -251,44 +266,12 @@ Coins.prototype.toRaw = function toRaw(writer) {
       continue;
     }
 
-    // Prefix byte (default=0
+    compressScript(output.script, p);
-    // for uncompressed script).
-    prefix = 0;
-
-    // Attempt to compress the output scripts.
-    // We can _only_ ever compress them if
-    // they are serialized as minimaldata, as
-    // we need to recreate them when we read
-    // them.
-    if (output.script.isPubkeyhash(true)) {
-      prefix = 1;
-      data = output.script.code[2].data;
-    } else if (output.script.isScripthash()) {
-      prefix = 2;
-      data = output.script.code[1].data;
-    } else if (output.script.isPubkey(true)) {
-      prefix = 3;
-      data = output.script.code[0].data;
-
-      // Try to compress the key.
-      data = bcoin.ec.compress(data);
-
-      // If we can't compress it,
-      // just store the script.
-      if (!data)
-        prefix = 0;
-    }
-
-    p.writeU8(prefix);
-
-    if (prefix === 0)
-      p.writeVarBytes(output.script.toRaw());
-    else
-      p.writeBytes(data);
-
     p.writeVarint(output.value);
   }
 
+  // Render the buffer with all
+  // zeroes in the spent field.
   p = p.render();
 
   // Mark the spents in the spent field.
@@ -319,7 +302,7 @@ Coins.prototype.toRaw = function toRaw(writer) {
 Coins.prototype.fromRaw = function fromRaw(data, hash, index) {
   var p = new BufferReader(data);
   var i = 0;
-  var bits, coin, prefix, offset, size, fstart, bit, oct, spent;
+  var bits, coin, offset, size, fstart, flen, bit, oct, spent;
 
   this.version = p.readVarint();
 
@@ -334,8 +317,9 @@ Coins.prototype.fromRaw = function fromRaw(data, hash, index) {
 
   // Mark the start of the spent field and
   // seek past it to avoid reading a buffer.
+  flen = p.readVarint();
   fstart = p.offset;
-  p.seek(p.readVarint());
+  p.seek(flen);
 
   while (p.left()) {
     // Read a single bit out of the spent field.
@@ -360,10 +344,9 @@ Coins.prototype.fromRaw = function fromRaw(data, hash, index) {
     }
 
     offset = p.offset;
-    prefix = p.readU8();
 
     // Skip past the compressed scripts.
-    switch (prefix) {
+    switch (p.readU8()) {
       case 0:
         p.seek(p.readVarint());
         break;
@@ -375,7 +358,7 @@ Coins.prototype.fromRaw = function fromRaw(data, hash, index) {
         p.seek(33);
         break;
       default:
-        assert(false, 'Bad prefix.');
+        throw new Error('Bad prefix.');
     }
 
     // Skip past the value.
@@ -468,197 +451,6 @@ Coins.fromTX = function fromTX(tx) {
   return new Coins().fromTX(tx);
 };
 
-Coins.compressScript = function compressScript(script, p) {
-  // Prefix byte (default=0
-  // for uncompressed script).
-  var prefix = 0;
-  var data;
-
-  // Attempt to compress the output scripts.
-  // We can _only_ ever compress them if
-  // they are serialized as minimaldata, as
-  // we need to recreate them when we read
-  // them.
-  if (script.isPubkeyhash(true)) {
-    prefix = 1;
-    data = script.code[2].data;
-  } else if (script.isScripthash()) {
-    prefix = 2;
-    data = script.code[1].data;
-  } else if (script.isPubkey(true)) {
-    prefix = 3;
-    data = script.code[0].data;
-
-    // Try to compress the key.
-    data = Coins.compressKey(data);
-
-    // If we can't compress it,
-    // just store the script.
-    if (!data)
-      prefix = 0;
-  }
-
-  p.writeU8(prefix);
-
-  if (prefix === 0)
-    p.writeVarBytes(script.toRaw());
-  else
-    p.writeBytes(data);
-};
-
-Coins.decompressScript = function decompressScript(p, script) {
-  var prefix = p.readU8();
-  var key;
-
-  // Decompress the script.
-  switch (prefix) {
-    case 0:
-      script.fromRaw(p.readVarBytes());
-      break;
-    case 1:
-      script.fromPubkeyhash(p.readBytes(20));
-      break;
-    case 2:
-      script.fromScripthash(p.readBytes(20));
-      break;
-    case 3:
-      // Decompress the key. If this fails,
-      // we have database corruption!
-      key = Coins.decompressKey(p.readBytes(33));
-      script.fromPubkey(key);
-      break;
-    default:
-      assert(false, 'Bad prefix.');
-  }
-};
-
-// See:
-// https://github.com/btcsuite/btcd/blob/master/blockchain/compress.go
-
-Coins.compressValue = function compressValue(value) {
-  var exp, last;
-
-  if (value === 0)
-    return 0;
-
-  exp = 0;
-  while (value % 10 === 0 && exp < 9) {
-    value /= 10;
-    exp++;
-  }
-
-  if (exp < 9) {
-    last = value % 10;
-    value = (value - last) / 10;
-    return 1 + 10 * (9 * value + last - 1) + exp;
-  }
-
-  return 10 + 10 * (value - 1);
-};
-
-Coins.decompressValue = function decompressValue(value) {
-  var exp, n, last;
-
-  if (value === 0)
-    return 0;
-
-  value--;
-
-  exp = value % 10;
-  value = (value - exp) / 10;
-
-  n = 0;
-  if (exp < 9) {
-    last = value % 9;
-    value = (value - last) / 9;
-    n = value * 10 + last + 1;
-  } else {
-    n = value + 1;
-  }
-
-  while (exp > 0) {
-    n *= 10;
-    exp--;
-  }
-
-  return n;
-};
-
-/**
- * Compress a public key to coins compression format.
- * @param {Buffer} key
- * @returns {Buffer}
- */
-
-Coins.compressKey = function compressKey(key) {
-  var out;
-
-  // We can't compress it if it's not valid.
-  if (!bcoin.ec.publicKeyVerify(key))
-    return;
-
-  switch (key[0]) {
-    case 0x02:
-    case 0x03:
-      // Key is already compressed.
-      out = key;
-      break;
-    case 0x04:
-    case 0x06:
-    case 0x07:
-      // Compress the key normally.
-      out = bcoin.ec.publicKeyConvert(key, true);
-      // Store the original format (which
-      // may be a hybrid byte) in the hi
-      // 3 bits so we can restore it later.
-      // The hi bits being set also lets us
-      // know that this key was originally
-      // decompressed.
-      out[0] |= key[0] << 2;
-      break;
-    default:
-      throw new Error('Bad point format.');
-  }
-
-  assert(out.length === 33);
-
-  return out;
-};
-
-/**
- * Decompress a public key from the coins compression format.
- * @param {Buffer} key
- * @returns {Buffer}
- */
-
-Coins.decompressKey = function decompressKey(key) {
-  var format = key[0] >>> 2;
-  var out;
-
-  assert(key.length === 33);
-
-  // Hi bits are not set. This key
-  // is not meant to be decompressed.
-  if (format === 0)
-    return key;
-
-  // Decompress the key, and off the
-  // low bits so publicKeyConvert
-  // actually understands it.
-  key[0] &= 0x03;
-  out = bcoin.ec.publicKeyConvert(key, false);
-
-  // Reset the hi bits so as not to
-  // mutate the original buffer.
-  key[0] |= format << 2;
-
-  // Set the original format, which
-  // may have been a hybrid prefix byte.
-  out[0] = format;
-
-  return out;
-};
-
 /**
  * A compressed coin is an object which defers
  * parsing of a coin. Say there is a transaction
@@ -697,7 +489,6 @@ function CompressedCoin(offset, size, raw) {
 CompressedCoin.prototype.toCoin = function toCoin(coins, index) {
   var p = new BufferReader(this.raw);
   var coin = new bcoin.coin();
-  var prefix, key;
 
   // Load in all necessary properties
   // from the parent Coins object.
@@ -710,28 +501,7 @@ CompressedCoin.prototype.toCoin = function toCoin(coins, index) {
   // Seek to the coin's offset.
   p.seek(this.offset);
 
-  prefix = p.readU8();
+  decompressScript(p, coin.script);
-
-  // Decompress the script.
-  switch (prefix) {
-    case 0:
-      coin.script.fromRaw(p.readVarBytes());
-      break;
-    case 1:
-      coin.script.fromPubkeyhash(p.readBytes(20));
-      break;
-    case 2:
-      coin.script.fromScripthash(p.readBytes(20));
-      break;
-    case 3:
-      // Decompress the key. If this fails,
-      // we have database corruption!
-      key = bcoin.ec.decompress(p.readBytes(33));
-      coin.script.fromPubkey(key);
-      break;
-    default:
-      assert(false, 'Bad prefix.');
-  }
 
   coin.value = p.readVarint();
 
@@ -749,11 +519,235 @@ CompressedCoin.prototype.toRaw = function toRaw() {
 };
 
 /*
- * Helpers
+ * Compression
  */
 
+/**
+ * Compress a script, write directly to the buffer.
+ * @param {Script} script
+ * @param {BufferWriter} p
+ */
+
+function compressScript(script, p) {
+  var prefix = 0;
+  var data;
+
+  // Attempt to compress the output scripts.
+  // We can _only_ ever compress them if
+  // they are serialized as minimaldata, as
+  // we need to recreate them when we read
+  // them.
+  if (script.isPubkeyhash(true)) {
+    prefix = 1;
+    data = script.code[2].data;
+  } else if (script.isScripthash()) {
+    prefix = 2;
+    data = script.code[1].data;
+  } else if (script.isPubkey(true)) {
+    prefix = 3;
+    data = script.code[0].data;
+
+    // Try to compress the key.
+    data = compressKey(data);
+
+    // If we can't compress it,
+    // just store the script.
+    if (!data)
+      prefix = 0;
+  }
+
+  p.writeU8(prefix);
+
+  if (prefix === 0)
+    p.writeVarBytes(script.toRaw());
+  else
+    p.writeBytes(data);
+}
+
+/**
+ * Decompress a script from buffer reader.
+ * @param {BufferReader} p
+ * @param {Script} script
+ */
+
+function decompressScript(p, script) {
+  var key;
+
+  // Decompress the script.
+  switch (p.readU8()) {
+    case 0:
+      script.fromRaw(p.readVarBytes());
+      break;
+    case 1:
+      script.fromPubkeyhash(p.readBytes(20));
+      break;
+    case 2:
+      script.fromScripthash(p.readBytes(20));
+      break;
+    case 3:
+      // Decompress the key. If this fails,
+      // we have database corruption!
+      key = decompressKey(p.readBytes(33));
+      script.fromPubkey(key);
+      break;
+    default:
+      throw new Error('Bad prefix.');
+  }
+}
+
+/**
+ * Compress value using an exponent. Takes advantage of
+ * the fact that many bitcoin values are divisible by 10.
+ * @see https://github.com/btcsuite/btcd/blob/master/blockchain/compress.go
+ * @param {Amount} value
+ * @returns {Number}
+ */
+
+function compressValue(value) {
+  var exp, last;
+
+  if (value === 0)
+    return 0;
+
+  exp = 0;
+  while (value % 10 === 0 && exp < 9) {
+    value /= 10;
+    exp++;
+  }
+
+  if (exp < 9) {
+    last = value % 10;
+    value = (value - last) / 10;
+    return 1 + 10 * (9 * value + last - 1) + exp;
+  }
+
+  return 10 + 10 * (value - 1);
+}
+
+/**
+ * Decompress value.
+ * @param {Number} value - Compressed value.
+ * @returns {Amount} value
+ */
+
+function decompressValue(value) {
+  var exp, n, last;
+
+  if (value === 0)
+    return 0;
+
+  value--;
+
+  exp = value % 10;
+  value = (value - exp) / 10;
+
+  if (exp < 9) {
+    last = value % 9;
+    value = (value - last) / 9;
+    n = value * 10 + last + 1;
+  } else {
+    n = value + 1;
+  }
+
+  while (exp > 0) {
+    n *= 10;
+    exp--;
+  }
+
+  return n;
+}
+
+/**
+ * Compress a public key to coins compression format.
+ * @param {Buffer} key
+ * @returns {Buffer}
+ */
+
+function compressKey(key) {
+  var out;
+
+  // We can't compress it if it's not valid.
+  if (!bcoin.ec.publicKeyVerify(key))
+    return;
+
+  switch (key[0]) {
+    case 0x02:
+    case 0x03:
+      // Key is already compressed.
+      out = key;
+      break;
+    case 0x04:
+    case 0x06:
+    case 0x07:
+      // Compress the key normally.
+      out = bcoin.ec.publicKeyConvert(key, true);
+      // Store the original format (which
+      // may be a hybrid byte) in the hi
+      // 3 bits so we can restore it later.
+      // The hi bits being set also lets us
+      // know that this key was originally
+      // decompressed.
+      out[0] |= key[0] << 2;
+      break;
+    default:
+      throw new Error('Bad point format.');
+  }
+
+  assert(out.length === 33);
+
+  return out;
+}
+
+/**
+ * Decompress a public key from the coins compression format.
+ * @param {Buffer} key
+ * @returns {Buffer}
+ */
+
+function decompressKey(key) {
+  var format = key[0] >>> 2;
+  var out;
+
+  assert(key.length === 33);
+
+  // Hi bits are not set. This key
+  // is not meant to be decompressed.
+  if (format === 0)
+    return key;
+
+  // Decompress the key, and off the
+  // low bits so publicKeyConvert
+  // actually understands it.
+  key[0] &= 0x03;
+  out = bcoin.ec.publicKeyConvert(key, false);
+
+  // Reset the hi bits so as not to
+  // mutate the original buffer.
+  key[0] |= format << 2;
+
+  // Set the original format, which
+  // may have been a hybrid prefix byte.
+  out[0] = format;
+
+  return out;
+}
+
 /*
  * Expose
  */
 
+exports = Coins;
+
+exports.compress = {
+  script: compressScript,
+  value: compressValue,
+  key: compressKey
+};
+
+exports.decompress = {
+  script: decompressScript,
+  value: decompressValue,
+  key: decompressKey
+};
+
 module.exports = Coins;

lib/bcoin/ec.js

@@ -134,81 +134,6 @@ ec.publicKeyConvert = function publicKeyConvert(key, compressed) {
   return new Buffer(point.encode('array', compressed));
 };
 
-/**
- * Compress a public key to coins compression format.
- * @param {Buffer} key
- * @returns {Buffer}
- */
-
-ec.compress = function compress(key) {
-  var out;
-
-  // We can't compress it if it's not valid.
-  if (!ec.publicKeyVerify(key))
-    return;
-
-  switch (key[0]) {
-    case 0x02:
-    case 0x03:
-      // Key is already compressed.
-      out = key;
-      break;
-    case 0x04:
-    case 0x06:
-    case 0x07:
-      // Compress the key normally.
-      out = ec.publicKeyConvert(key, true);
-      // Store the original format (which
-      // may be a hybrid byte) in the hi
-      // 3 bits so we can restore it later.
-      // The hi bits being set also lets us
-      // know that this key was originally
-      // decompressed.
-      out[0] |= key[0] << 2;
-      break;
-    default:
-      throw new Error('Bad point format.');
-  }
-
-  assert(out.length === 33);
-
-  return out;
-};
-
-/**
- * Decompress a public key from the coins compression format.
- * @param {Buffer} key
- * @returns {Buffer}
- */
-
-ec.decompress = function decompress(key) {
-  var format = key[0] >>> 2;
-  var out;
-
-  assert(key.length === 33);
-
-  // Hi bits are not set. This key
-  // is not meant to be decompressed.
-  if (format === 0)
-    return key;
-
-  // Decompress the key, and off the
-  // low bits so publicKeyConvert
-  // actually understands it.
-  key[0] &= 0x03;
-  out = ec.publicKeyConvert(key, false);
-
-  // Reset the hi bits so as not to
-  // mutate the original buffer.
-  key[0] |= format << 2;
-
-  // Set the original format, which
-  // may have been a hybrid prefix byte.
-  out[0] = format;
-
-  return out;
-};
-
 /**
  * Create an ecdh.
  * @param {Buffer} pub

lib/bcoin/protocol/network.js

@@ -368,7 +368,7 @@ main.keyPrefix = {
 
 /**
  * {@link Address} prefixes.
- * @enum {Object}
+ * @enum {Number}
  */
 
 main.addressPrefix = {