better coin compression.

lib/bcoin/coins.js

@@ -121,7 +121,7 @@ Coins.prototype.get = function get(index) {
   if (!coin)
     return;
 
-  if (coin instanceof DeferredCoin)
+  if (coin instanceof CompressedCoin)
     coin = coin.toCoin(this, index);
 
   return coin;
@@ -156,6 +156,37 @@ Coins.prototype.spend = function spend(index) {
   return coin;
 };
 
+/**
+ * Count up to the last available index.
+ * @returns {Number}
+ */
+
+Coins.prototype.getLength = function getLength() {
+  var last = -1;
+  var i;
+
+  for (i = 0; i < this.outputs.length; i++) {
+    if (this.outputs[i])
+      last = i;
+  }
+
+  return last + 1;
+};
+
+/*
+ * Coins serialization:
+ * version: varint
+ * bits: varint ((height << 1) | coinbase-flag)
+ * outputs (repeated):
+ *   prefix: 0xff = spent
+ *           0x00 = varint size | raw script
+ *           0x01 = 20 byte pubkey hash
+ *           0x02 = 20 byte script hash
+ *           0x03 = 33 byte compressed key
+ *   data: the data mentioned above
+ *   value: varint
+ */
+
 /**
  * Serialize the coins object.
  * @param {TX|Coins} tx
@@ -165,19 +196,26 @@ Coins.prototype.spend = function spend(index) {
 Coins.prototype.toRaw = function toRaw(writer) {
   var p = new BufferWriter(writer);
   var height = this.height;
-  var i, output, prefix, hash, coinbase, mask;
+  var length = this.getLength();
+  var i, output, prefix, data, bits;
 
+  // Unfortunately, we don't have a compact
+  // way to store unconfirmed height.
   if (height === -1)
     height = 0x7fffffff;
 
-  coinbase = this.coinbase;
+  bits = height << 1;
 
-  mask = (height << 1) | (coinbase ? 1 : 0);
+  if (this.coinbase)
+    bits |= 1;
+
+  if (bits < 0)
+    bits += 0x100000000;
 
   p.writeVarint(this.version);
-  p.writeU32(mask >>> 0);
+  p.writeVarint(bits);
 
-  for (i = 0; i < this.outputs.length; i++) {
+  for (i = 0; i < length; i++) {
     output = this.outputs[i];
 
     if (!output) {
@@ -185,28 +223,43 @@ Coins.prototype.toRaw = function toRaw(writer) {
       continue;
     }
 
-    if (output instanceof DeferredCoin) {
+    if (output instanceof CompressedCoin) {
       p.writeBytes(output.toRaw());
       continue;
     }
 
     prefix = 0;
 
-    // Saves up to 7 bytes.
+    // Attempt to compress the output scripts.
+    // We can _only_ ever compress them if
+    // they are serialized as minimaldata, as
+    // we need to recreate them when we read
+    // them.
     if (output.script.isPubkeyhash(true)) {
       prefix = 1;
-      hash = output.script.code[2].data;
+      data = output.script.code[2].data;
     } else if (output.script.isScripthash()) {
       prefix = 2;
-      hash = output.script.code[1].data;
+      data = output.script.code[1].data;
+    } else if (output.script.isPubkey(true)) {
+      prefix = 3;
+      data = output.script.code[0].data;
+
+      // Try to compress the key.
+      data = bcoin.ec.compress(data);
+
+      // If we can't compress it,
+      // just store the script.
+      if (!data)
+        prefix = 0;
     }
 
     p.writeU8(prefix);
 
-    if (prefix)
+    if (prefix === 0)
-      p.writeBytes(hash);
-    else
       p.writeVarBytes(output.script.toRaw());
+    else
+      p.writeBytes(data);
 
     p.writeVarint(output.value);
   }
@@ -227,65 +280,84 @@ Coins.prototype.toRaw = function toRaw(writer) {
 Coins.prototype.fromRaw = function fromRaw(data, hash, index) {
   var p = new BufferReader(data);
   var i = 0;
-  var version, height, coin, mask, prefix, offset, size;
+  var bits, coin, prefix, offset, size;
 
-  version = p.readVarint();
+  this.version = p.readVarint();
-  height = p.readU32();
 
-  this.version = version;
+  bits = p.readVarint();
-  this.height = height >>> 1;
+
+  this.height = bits >>> 1;
   this.hash = hash;
-  this.coinbase = (height & 1) !== 0;
+  this.coinbase = (bits & 1) !== 0;
 
   if (this.height === 0x7fffffff)
     this.height = -1;
 
   while (p.left()) {
     offset = p.start();
-    mask = p.readU8();
+    prefix = p.readU8();
 
-    if (mask === 0xff) {
+    // Already spent.
+    if (prefix === 0xff) {
       p.end();
+
+      // Don't bother pushing outputs on if
+      // we're seeking to a specific index.
       if (index != null) {
         if (i === index)
           return;
         i++;
         continue;
       }
+
       this.outputs.push(null);
       i++;
       continue;
     }
 
-    prefix = mask & 3;
+    // Skip past the compressed scripts.
-
+    switch (prefix & 3) {
-    if (prefix === 0)
+      case 0:
-      p.seek(p.readVarint());
+        p.seek(p.readVarint());
-    else if (prefix <= 2)
+        break;
-      p.seek(20);
+      case 1:
-    else
+      case 2:
-      assert(false, 'Bad prefix.');
+        p.seek(20);
+        break;
+      case 3:
+        p.seek(33);
+        break;
+      default:
+        assert(false, 'Bad prefix.');
+    }
 
+    // Skip past the value.
     p.readVarint();
 
     size = p.end();
 
+    // Keep going if we're seeking
+    // to a specific index.
     if (index != null && i !== index) {
       i++;
       continue;
     }
 
-    coin = new DeferredCoin(offset, size, data);
+    // Store the offset and size
+    // in the compressed coin object.
+    coin = new CompressedCoin(offset, size, data);
 
+    // We found our coin.
     if (index != null)
       return coin.toCoin(this, i);
 
     this.outputs.push(coin);
-
     i++;
   }
 
-  assert(index == null, 'Bad coin index.');
+  // We couldn't find our coin.
+  if (index != null)
+    return;
 
   return this;
 };
@@ -350,17 +422,17 @@ Coins.fromTX = function fromTX(tx) {
 };
 
 /**
- * A "deferred" coin is an object which defers
+ * A compressed coin is an object which defers
- * parsing of a compressed coin. Say there is
+ * parsing of a coin. Say there is a transaction
- * a transaction with 100 outputs. When a block
+ * with 100 outputs. When a block comes in,
- * comes in, there may only be _one_ input in
+ * there may only be _one_ input in that entire
- * that entire block which redeems an output
+ * block which redeems an output from that
- * from that transaction. When parsing the
+ * transaction. When parsing the Coins, there
- * Coins, there is no sense to get _all_ of
+ * is no sense to get _all_ of them into their
- * them into their abstract form. A deferred
+ * abstract form. A compressed coin is just a
- * coin is just a pointer to that coin in the
+ * pointer to that coin in the Coins buffer, as
- * Coins buffer, as well as a size. Parsing
+ * well as a size. Parsing is done only if that
- * is done only if that coin is being redeemed.
+ * coin is being redeemed.
  * @constructor
  * @private
  * @param {Number} offset
@@ -368,9 +440,9 @@ Coins.fromTX = function fromTX(tx) {
  * @param {Buffer} raw
  */
 
-function DeferredCoin(offset, size, raw) {
+function CompressedCoin(offset, size, raw) {
-  if (!(this instanceof DeferredCoin))
+  if (!(this instanceof CompressedCoin))
-    return new DeferredCoin(offset, size, raw);
+    return new CompressedCoin(offset, size, raw);
 
   this.offset = offset;
   this.size = size;
@@ -384,22 +456,26 @@ function DeferredCoin(offset, size, raw) {
  * @returns {Coin}
  */
 
-DeferredCoin.prototype.toCoin = function toCoin(coins, index) {
+CompressedCoin.prototype.toCoin = function toCoin(coins, index) {
   var p = new BufferReader(this.raw);
   var coin = new bcoin.coin();
-  var prefix;
+  var prefix, key;
 
+  // Load in all necessary properties
+  // from the parent Coins object.
   coin.version = coins.version;
   coin.coinbase = coins.coinbase;
   coin.height = coins.height;
   coin.hash = coins.hash;
   coin.index = index;
 
+  // Seek to the coin's offset.
   p.seek(this.offset);
 
-  prefix = p.readU8() & 3;
+  prefix = p.readU8();
 
-  switch (prefix) {
+  // Decompress the script.
+  switch (prefix & 3) {
     case 0:
       coin.script.fromRaw(p.readVarBytes());
       break;
@@ -409,6 +485,12 @@ DeferredCoin.prototype.toCoin = function toCoin(coins, index) {
     case 2:
       coin.script.fromScripthash(p.readBytes(20));
       break;
+    case 3:
+      // Decompress the key. If this fails,
+      // we have database corruption!
+      key = bcoin.ec.decompress(p.readBytes(33));
+      coin.script.fromPubkey(key);
+      break;
     default:
       assert(false, 'Bad prefix.');
   }
@@ -424,7 +506,7 @@ DeferredCoin.prototype.toCoin = function toCoin(coins, index) {
  * @returns {Buffer}
  */
 
-DeferredCoin.prototype.toRaw = function toRaw() {
+CompressedCoin.prototype.toRaw = function toRaw() {
   return this.raw.slice(this.offset, this.offset + this.size);
 };
 

lib/bcoin/ec.js

@@ -114,7 +114,7 @@ ec.decodePoint = function decodePoint(key) {
  * @returns {Buffer}
  */
 
-ec.publicKeyConvert = function(key, compressed) {
+ec.publicKeyConvert = function publicKeyConvert(key, compressed) {
   var point;
 
   if (secp256k1)