optimize coins serialization.

lib/bcoin/coins.js

@@ -177,7 +177,7 @@ Coins.prototype.getLength = function getLength() {
  * Coins serialization:
  * version: varint
  * bits: varint ((height << 1) | coinbase-flag)
- * spent-field: varint size | bitmap (0=unspent, 1=spent)
+ * spent-field: varint size | bitfield (0=unspent, 1=spent)
  * outputs (repeated):
  *   prefix: 0x00 = varint size | raw script
  *           0x01 = 20 byte pubkey hash
@@ -194,16 +194,22 @@ Coins.prototype.getLength = function getLength() {
  */
 
 Coins.prototype.toRaw = function toRaw(writer) {
-  var p = new BufferWriter(writer);
+  var p = new BufferWriter();
   var height = this.height;
   var length = this.getLength();
-  var i, output, prefix, data, bits, field, pos, bit, oct;
+  var i, output, prefix, data, bits, fstart, flen, bit, oct;
+
+  // Varint version: hopefully some smartass
+  // miner doesn't start mining `-1` versions.
+  p.writeVarint(this.version);
 
   // Unfortunately, we don't have a compact
   // way to store unconfirmed height.
   if (height === -1)
     height = 0x7fffffff;
 
+  // Create the `bits` value:
+  // (height | coinbase-flag).
   bits = height << 1;
 
   if (this.coinbase)
@@ -212,22 +218,23 @@ Coins.prototype.toRaw = function toRaw(writer) {
   if (bits < 0)
     bits += 0x100000000;
 
-  p.writeVarint(this.version);
+  // Making this a varint would actually
+  // make 99% of coins bigger. Varints
+  // are really only useful up until
+  // 0x10000, but since we're also
+  // storing the coinbase flag on the
+  // lo bit, varints are useless (and
+  // actually harmful) after height
+  // 32767 (0x7fff).
   p.writeU32(bits);
 
-  field = new Buffer(Math.ceil(length / 8));
+  // Fill the spent field with zeroes to avoid
-  field.fill(0);
+  // allocating a buffer. We mark the spents
-  pos = 0;
+  // after rendering the final buffer.
-
+  fstart = p.written;
-  for (i = 0; i < length; i++) {
+  flen = Math.ceil(length / 8);
-    output = this.outputs[i] ? 0 : 1;
+  p.writeVarint(flen);
-    bit = pos % 8;
+  p.fill(0, flen);
-    oct = (pos - bit) / 8;
-    field[oct] |= output << (7 - bit);
-    pos++;
-  }
-
-  p.writeVarBytes(field);
 
   for (i = 0; i < length; i++) {
     output = this.outputs[i];
@@ -235,11 +242,17 @@ Coins.prototype.toRaw = function toRaw(writer) {
     if (!output)
       continue;
 
+    // If we read this coin from the db and
+    // didn't use it, it's still in its
+    // compressed form. Just write it back
+    // as a buffer for speed.
     if (output instanceof CompressedCoin) {
       p.writeBytes(output.toRaw());
       continue;
     }
 
+    // Prefix byte (default=0
+    // for uncompressed script).
     prefix = 0;
 
     // Attempt to compress the output scripts.
@@ -276,8 +289,22 @@ Coins.prototype.toRaw = function toRaw(writer) {
     p.writeVarint(output.value);
   }
 
-  if (!writer)
+  p = p.render();
-    p = p.render();
+
+  // Mark the spents in the spent field.
+  // This is essentially a NOP for new coins.
+  for (i = 0; i < length; i++) {
+    if (this.outputs[i])
+      continue;
+    bit = i % 8;
+    oct = (i - bit) / 8;
+    p[fstart + oct] |= 1 << (7 - bit);
+  }
+
+  if (writer) {
+    writer.writeBytes(p);
+    return writer;
+  }
 
   return p;
 };
@@ -292,7 +319,7 @@ Coins.prototype.toRaw = function toRaw(writer) {
 Coins.prototype.fromRaw = function fromRaw(data, hash, index) {
   var p = new BufferReader(data);
   var i = 0;
-  var bits, coin, prefix, offset, size, field, bit, oct, spent;
+  var bits, coin, prefix, offset, size, fstart, bit, oct, spent;
 
   this.version = p.readVarint();
 
@@ -305,12 +332,16 @@ Coins.prototype.fromRaw = function fromRaw(data, hash, index) {
   if (this.height === 0x7fffffff)
     this.height = -1;
 
-  field = p.readVarBytes(true);
+  // Mark the start of the spent field and
+  // seek past it to avoid reading a buffer.
+  fstart = p.offset;
+  p.seek(p.readVarint());
 
   while (p.left()) {
+    // Read a single bit out of the spent field.
     bit = i % 8;
     oct = (i - bit) / 8;
-    spent = (field[oct] >>> (7 - bit)) & 1;
+    spent = (p.data[fstart + oct] >>> (7 - bit)) & 1;
 
     // Already spent.
     if (spent) {
@@ -328,7 +359,7 @@ Coins.prototype.fromRaw = function fromRaw(data, hash, index) {
       continue;
     }
 
-    offset = p.start();
+    offset = p.offset;
     prefix = p.readU8();
 
     // Skip past the compressed scripts.
@@ -350,7 +381,7 @@ Coins.prototype.fromRaw = function fromRaw(data, hash, index) {
     // Skip past the value.
     p.readVarint();
 
-    size = p.end();
+    size = p.offset - offset;
 
     // Keep going if we're seeking
     // to a specific index.
@@ -437,6 +468,197 @@ Coins.fromTX = function fromTX(tx) {
   return new Coins().fromTX(tx);
 };
 
+Coins.compressScript = function compressScript(script, p) {
+  // Prefix byte (default=0
+  // for uncompressed script).
+  var prefix = 0;
+  var data;
+
+  // Attempt to compress the output scripts.
+  // We can _only_ ever compress them if
+  // they are serialized as minimaldata, as
+  // we need to recreate them when we read
+  // them.
+  if (script.isPubkeyhash(true)) {
+    prefix = 1;
+    data = script.code[2].data;
+  } else if (script.isScripthash()) {
+    prefix = 2;
+    data = script.code[1].data;
+  } else if (script.isPubkey(true)) {
+    prefix = 3;
+    data = script.code[0].data;
+
+    // Try to compress the key.
+    data = Coins.compressKey(data);
+
+    // If we can't compress it,
+    // just store the script.
+    if (!data)
+      prefix = 0;
+  }
+
+  p.writeU8(prefix);
+
+  if (prefix === 0)
+    p.writeVarBytes(script.toRaw());
+  else
+    p.writeBytes(data);
+};
+
+Coins.decompressScript = function decompressScript(p, script) {
+  var prefix = p.readU8();
+  var key;
+
+  // Decompress the script.
+  switch (prefix) {
+    case 0:
+      script.fromRaw(p.readVarBytes());
+      break;
+    case 1:
+      script.fromPubkeyhash(p.readBytes(20));
+      break;
+    case 2:
+      script.fromScripthash(p.readBytes(20));
+      break;
+    case 3:
+      // Decompress the key. If this fails,
+      // we have database corruption!
+      key = Coins.decompressKey(p.readBytes(33));
+      script.fromPubkey(key);
+      break;
+    default:
+      assert(false, 'Bad prefix.');
+  }
+};
+
+// See:
+// https://github.com/btcsuite/btcd/blob/master/blockchain/compress.go
+
+Coins.compressValue = function compressValue(value) {
+  var exp, last;
+
+  if (value === 0)
+    return 0;
+
+  exp = 0;
+  while (value % 10 === 0 && exp < 9) {
+    value /= 10;
+    exp++;
+  }
+
+  if (exp < 9) {
+    last = value % 10;
+    value = (value - last) / 10;
+    return 1 + 10 * (9 * value + last - 1) + exp;
+  }
+
+  return 10 + 10 * (value - 1);
+};
+
+Coins.decompressValue = function decompressValue(value) {
+  var exp, n, last;
+
+  if (value === 0)
+    return 0;
+
+  value--;
+
+  exp = value % 10;
+  value = (value - exp) / 10;
+
+  n = 0;
+  if (exp < 9) {
+    last = value % 9;
+    value = (value - last) / 9;
+    n = value * 10 + last + 1;
+  } else {
+    n = value + 1;
+  }
+
+  while (exp > 0) {
+    n *= 10;
+    exp--;
+  }
+
+  return n;
+};
+
+/**
+ * Compress a public key to coins compression format.
+ * @param {Buffer} key
+ * @returns {Buffer}
+ */
+
+Coins.compressKey = function compressKey(key) {
+  var out;
+
+  // We can't compress it if it's not valid.
+  if (!bcoin.ec.publicKeyVerify(key))
+    return;
+
+  switch (key[0]) {
+    case 0x02:
+    case 0x03:
+      // Key is already compressed.
+      out = key;
+      break;
+    case 0x04:
+    case 0x06:
+    case 0x07:
+      // Compress the key normally.
+      out = bcoin.ec.publicKeyConvert(key, true);
+      // Store the original format (which
+      // may be a hybrid byte) in the hi
+      // 3 bits so we can restore it later.
+      // The hi bits being set also lets us
+      // know that this key was originally
+      // decompressed.
+      out[0] |= key[0] << 2;
+      break;
+    default:
+      throw new Error('Bad point format.');
+  }
+
+  assert(out.length === 33);
+
+  return out;
+};
+
+/**
+ * Decompress a public key from the coins compression format.
+ * @param {Buffer} key
+ * @returns {Buffer}
+ */
+
+Coins.decompressKey = function decompressKey(key) {
+  var format = key[0] >>> 2;
+  var out;
+
+  assert(key.length === 33);
+
+  // Hi bits are not set. This key
+  // is not meant to be decompressed.
+  if (format === 0)
+    return key;
+
+  // Decompress the key, and off the
+  // low bits so publicKeyConvert
+  // actually understands it.
+  key[0] &= 0x03;
+  out = bcoin.ec.publicKeyConvert(key, false);
+
+  // Reset the hi bits so as not to
+  // mutate the original buffer.
+  key[0] |= format << 2;
+
+  // Set the original format, which
+  // may have been a hybrid prefix byte.
+  out[0] = format;
+
+  return out;
+};
+
 /**
  * A compressed coin is an object which defers
  * parsing of a coin. Say there is a transaction
@@ -526,6 +748,10 @@ CompressedCoin.prototype.toRaw = function toRaw() {
   return this.raw.slice(this.offset, this.offset + this.size);
 };
 
+/*
+ * Helpers
+ */
+
 /*
  * Expose
  */