migrate: add pertxout migration.

migrate/chaindb2to3.js

@@ -0,0 +1,523 @@
+'use strict';
+
+const assert = require('assert');
+const encoding = require('../lib/utils/encoding');
+const co = require('../lib/utils/co');
+const digest = require('../lib/crypto/digest');
+const BN = require('../lib/crypto/bn');
+const StaticWriter = require('../lib/utils/staticwriter');
+const BufferReader = require('../lib/utils/reader');
+const OldCoins = require('./coins/coins');
+const OldUndoCoins = require('./coins/undocoins');
+const CoinEntry = require('../lib/coins/coinentry');
+const UndoCoins = require('../lib/coins/undocoins');
+const Block = require('../lib/primitives/block');
+const LDB = require('../lib/db/ldb');
+
+const MIGRATION_ID = 0;
+
+let file = process.argv[2];
+
+assert(typeof file === 'string', 'Please pass in a database path.');
+
+file = file.replace(/\.ldb\/?$/, '');
+
+const db = LDB({
+  location: file,
+  db: 'leveldb',
+  compression: true,
+  cacheSize: 32 << 20,
+  createIfMissing: false,
+  bufferKeys: true
+});
+
+// \0\0migrate
+const JOURNAL_KEY = Buffer.from('00006d696772617465', 'hex');
+const STATE_VERSION = -1;
+const STATE_UNDO = 0;
+const STATE_CLEANUP = 1;
+const STATE_COINS = 2;
+const STATE_ENTRY = 3;
+const STATE_FINAL = 4;
+const STATE_DONE = 5;
+
+const heightCache = new Map();
+
+function writeJournal(batch, state, hash) {
+  let data = Buffer.allocUnsafe(34);
+
+  if (!hash)
+    hash = encoding.NULL_HASH;
+
+  data[0] = MIGRATION_ID;
+  data[1] = state;
+  data.write(hash, 2, 'hex');
+
+  batch.put(JOURNAL_KEY, data);
+}
+
+async function readJournal() {
+  let data = await db.get(JOURNAL_KEY);
+  let state, hash;
+
+  if (!data)
+    return [STATE_VERSION, encoding.NULL_HASH];
+
+  if (data[0] !== MIGRATION_ID)
+    throw new Error('Bad migration id.');
+
+  if (data.length !== 34)
+    throw new Error('Bad migration length.');
+
+  state = data.readUInt8(1, true);
+  hash = data.toString('hex', 2, 34);
+
+  return [state, hash];
+}
+
+async function updateVersion() {
+  let batch = db.batch();
+  let data, version;
+
+  console.log('Checking version.');
+
+  data = await db.get('V');
+
+  if (!data)
+    throw new Error('No DB version found!');
+
+  version = data.readUInt32LE(0, true);
+
+  if (version !== 2)
+    throw Error(`DB is version ${version}.`);
+
+  data = Buffer.allocUnsafe(4);
+
+  // Set to 255 temporarily.
+  data.writeUInt32LE(255, 0, true);
+  batch.put('V', data);
+
+  writeJournal(batch, STATE_UNDO);
+
+  await batch.write();
+
+  return [STATE_UNDO, encoding.NULL_HASH];
+}
+
+async function reserializeUndo(hash) {
+  let batch = db.batch();
+  let tip = await getTip();
+  let total = 0;
+
+  if (hash !== encoding.NULL_HASH)
+    tip = await getEntry(hash);
+
+  while (tip.height !== 0) {
+    let undoData = await db.get(pair('u', tip.hash));
+    let blockData = await db.get(pair('b', tip.hash));
+    let block, undo, newUndo;
+
+    assert(undoData);
+
+    if (!blockData) {
+      if (!(await isPruned()))
+        throw new Error(`Block not found: ${tip.hash}.`);
+      break;
+    }
+
+    block = Block.fromRaw(blockData);
+
+    newUndo = new UndoCoins();
+    undo = OldUndoCoins.fromRaw(undoData);
+
+    for (let i = block.txs.length - 1; i >= 1; i--) {
+      let tx = block.txs[i];
+      for (let j = tx.inputs.length - 1; j >= 0; j--) {
+        let {prevout} = tx.inputs[j];
+        let coin = undo.items.pop();
+        let output = coin.toOutput();
+        let version, height, write, item;
+
+        assert(coin);
+
+        [version, height, write] = await getProps(coin, prevout);
+
+        item = new CoinEntry();
+        item.version = version;
+        item.height = height;
+        item.coinbase = coin.coinbase;
+        item.output.script = output.script;
+        item.output.value = output.value;
+        item.spent = false;
+        item.raw = null;
+
+        // Store an index of heights and versions for later.
+        if (write) {
+          let data = Buffer.allocUnsafe(8);
+          data.writeUInt32LE(version, 0, true);
+          data.writeUInt32LE(height, 4, true);
+          batch.put(pair(0x01, prevout.hash), data);
+          heightCache.set(prevout.hash, [version, height]);
+        }
+
+        newUndo.items.push(item);
+      }
+    }
+
+    batch.put(pair('u', tip.hash), newUndo.toRaw());
+
+    if (++total % 10000 === 0) {
+      console.log('Reserialized %d undo coins.', total);
+      writeJournal(batch, STATE_UNDO, tip.prevBlock);
+      await batch.write();
+      heightCache.clear();
+      batch = db.batch();
+    }
+
+    tip = await getEntry(tip.prevBlock);
+    assert(tip);
+  }
+
+  writeJournal(batch, STATE_CLEANUP);
+  await batch.write();
+
+  heightCache.clear();
+
+  console.log('Reserialized %d undo coins.', total);
+
+  return [STATE_CLEANUP, encoding.NULL_HASH];
+}
+
+async function cleanupIndex() {
+  let batch = db.batch();
+  let total = 0;
+
+  let iter = db.iterator({
+    gte: pair(0x01, encoding.ZERO_HASH),
+    lte: pair(0x01, encoding.MAX_HASH),
+    keys: true
+  });
+
+  for (;;) {
+    let item = await iter.next();
+
+    if (!item)
+      break;
+
+    batch.del(item.key);
+
+    if (++total % 100000 === 0) {
+      console.log('Cleaned up %d undo records.', total);
+      writeJournal(batch, STATE_CLEANUP);
+      await batch.write();
+      batch = db.batch();
+    }
+  }
+
+  writeJournal(batch, STATE_COINS);
+  await batch.write();
+
+  console.log('Cleaned up %d undo records.', total);
+
+  return [STATE_COINS, encoding.NULL_HASH];
+}
+
+async function reserializeCoins(hash) {
+  let batch = db.batch();
+  let start = false;
+  let total = 0;
+
+  let iter = db.iterator({
+    gte: pair('c', hash),
+    lte: pair('c', encoding.MAX_HASH),
+    keys: true,
+    values: true
+  });
+
+  if (hash !== encoding.NULL_HASH) {
+    let item = await iter.next();
+    if (!item)
+      start = false;
+  }
+
+  while (start) {
+    let item = await iter.next();
+    let update = false;
+    let hash, old;
+
+    if (!item)
+      break;
+
+    if (item.key.length !== 33)
+      continue;
+
+    hash = item.key.toString('hex', 1, 33);
+    old = OldCoins.fromRaw(item.value, hash);
+
+    for (let i = 0; i < old.outputs.length; i++) {
+      let coin = old.getCoin(i);
+      let item;
+
+      if (!coin)
+        continue;
+
+      item = new CoinEntry();
+      item.version = coin.version;
+      item.height = coin.height;
+      item.coinbase = coin.coinbase;
+      item.output.script = coin.script;
+      item.output.value = coin.value;
+      item.spent = false;
+      item.raw = null;
+
+      batch.put(bpair('c', hash, i), item.toRaw());
+
+      if (++total % 100000 === 0)
+        update = true;
+    }
+
+    batch.del(item.key);
+
+    if (update) {
+      console.log('Reserialized %d coins.', total);
+      writeJournal(batch, STATE_COINS);
+      await batch.write();
+      batch = db.batch();
+    }
+  }
+
+  writeJournal(batch, STATE_ENTRY);
+  await batch.write();
+
+  console.log('Reserialized %d coins.', total);
+
+  return [STATE_ENTRY, encoding.NULL_HASH];
+}
+
+async function reserializeEntries(hash) {
+  let tip = await getTipHash();
+  let batch = db.batch();
+  let start = true;
+  let total = 0;
+
+  let iter = db.iterator({
+    gte: pair('e', hash),
+    lte: pair('e', encoding.MAX_HASH),
+    values: true
+  });
+
+  if (hash !== encoding.NULL_HASH) {
+    let item = await iter.next();
+    if (!item)
+      start = false;
+    else
+      assert(item.key.equals(pair('e', hash)));
+  }
+
+  while (start) {
+    let item = await iter.next();
+    let entry, main;
+
+    if (!item)
+      break;
+
+    entry = entryFromRaw(item.value);
+    main = await isMainChain(entry, tip);
+
+    batch.put(item.key, entryToRaw(entry, main));
+
+    if (++total % 100000 === 0) {
+      console.log('Reserialized %d entries.', total);
+      writeJournal(batch, STATE_ENTRY, entry.hash);
+      await batch.write();
+      batch = db.batch();
+    }
+  }
+
+  writeJournal(batch, STATE_FINAL);
+  await batch.write();
+
+  console.log('Reserialized %d entries.', total);
+
+  return [STATE_FINAL, encoding.NULL_HASH];
+}
+
+async function finalize() {
+  let batch = db.batch();
+  let data = Buffer.allocUnsafe(4);
+  data.writeUInt32LE(3, 0, true);
+
+  batch.del(JOURNAL_KEY);
+  batch.put('V', data);
+
+  console.log('Finalizing...');
+
+  await batch.write();
+
+  return [STATE_DONE, encoding.NULL_HASH];
+}
+
+async function getProps(coin, prevout) {
+  let item, data, coins;
+
+  if (coin.height !== -1)
+    return [coin.version, coin.height, true];
+
+  item = heightCache.get(prevout.hash);
+
+  if (item) {
+    let [version, height] = item;
+    return [version, height, false];
+  }
+
+  data = await db.get(pair(0x01, prevout.hash));
+
+  if (data) {
+    let version = data.readUInt32LE(0, true);
+    let height = data.readUInt32LE(4, true);
+    return [version, height, false];
+  }
+
+  data = await db.get(pair('c', prevout.hash));
+  assert(data);
+
+  coins = OldCoins.fromRaw(data, prevout.hash);
+
+  return [coins.version, coins.height, true];
+}
+
+async function getTip() {
+  let tip = await getTipHash();
+  return await getEntry(tip);
+}
+
+async function getTipHash() {
+  let state = await db.get('R');
+  assert(state);
+  return state.toString('hex', 0, 32);
+}
+
+async function getEntry(hash) {
+  let data = await db.get(pair('e', hash));
+  assert(data);
+  return entryFromRaw(data);
+}
+
+async function isPruned() {
+  let data = await db.get('O');
+  assert(data);
+  return (data.readUInt32LE(4) & 4) !== 0;
+}
+
+async function isMainChain(entry, tip) {
+  if (entry.hash === tip)
+    return true;
+
+  if (await db.get(pair('n', entry.hash)))
+    return true;
+
+  return false;
+}
+
+function entryFromRaw(data) {
+  let p = new BufferReader(data, true);
+  let hash = digest.hash256(p.readBytes(80));
+  let entry = {};
+
+  p.seek(-80);
+
+  entry.hash = hash.toString('hex');
+  entry.version = p.readU32();
+  entry.prevBlock = p.readHash('hex');
+  entry.merkleRoot = p.readHash('hex');
+  entry.ts = p.readU32();
+  entry.bits = p.readU32();
+  entry.nonce = p.readU32();
+  entry.height = p.readU32();
+  entry.chainwork = new BN(p.readBytes(32), 'le');
+
+  return entry;
+}
+
+function entryToRaw(entry, main) {
+  let bw = new StaticWriter(116 + 1);
+
+  bw.writeU32(entry.version);
+  bw.writeHash(entry.prevBlock);
+  bw.writeHash(entry.merkleRoot);
+  bw.writeU32(entry.ts);
+  bw.writeU32(entry.bits);
+  bw.writeU32(entry.nonce);
+  bw.writeU32(entry.height);
+  bw.writeBytes(entry.chainwork.toArrayLike(Buffer, 'le', 32));
+  bw.writeU8(main ? 1 : 0);
+
+  return bw.render();
+}
+
+function write(data, str, off) {
+  if (Buffer.isBuffer(str))
+    return str.copy(data, off);
+  data.write(str, off, 'hex');
+}
+
+function pair(prefix, hash) {
+  let key = Buffer.allocUnsafe(33);
+  if (typeof prefix === 'string')
+    prefix = prefix.charCodeAt(0);
+  key[0] = prefix;
+  write(key, hash, 1);
+  return key;
+}
+
+function bpair(prefix, hash, index) {
+  let key = Buffer.allocUnsafe(37);
+  if (typeof prefix === 'string')
+    prefix = prefix.charCodeAt(0);
+  key[0] = prefix;
+  write(key, hash, 1);
+  key.writeUInt32BE(index, 33, true);
+  return key;
+}
+
+(async () => {
+  let state, hash;
+
+  await db.open();
+
+  console.log('Opened %s.', file);
+
+  console.log('Starting migration. If you crash you can start over.');
+
+  await co.timeout(3000);
+
+  [state, hash] = await readJournal();
+
+  if (state === STATE_VERSION)
+    [state, hash] = await updateVersion();
+
+  if (state === STATE_UNDO)
+    [state, hash] = await reserializeUndo(hash);
+
+  if (state === STATE_CLEANUP)
+    [state, hash] = await cleanupIndex();
+
+  if (state === STATE_COINS)
+    [state, hash] = await reserializeCoins(hash);
+
+  // if (state === STATE_ENTRY)
+  //   [state, hash] = await reserializeEntries(hash);
+
+  if (state === STATE_ENTRY)
+    [state, hash] = [STATE_FINAL, encoding.NULL_HASH];
+
+  if (state === STATE_FINAL)
+    [state, hash] = await finalize();
+
+  assert(state === STATE_DONE);
+})().then(() => {
+  console.log('Migration complete.');
+  process.exit(0);
+}).catch((err) => {
+  throw err;
+});

migrate/coins/coins.js

@@ -0,0 +1,763 @@
+/*!
+ * coins.js - coins object for bcoin
+ * Copyright (c) 2014-2017, Christopher Jeffrey (MIT License).
+ * https://github.com/bcoin-org/bcoin
+ */
+
+'use strict';
+
+const assert = require('assert');
+const util = require('../../lib/utils/util');
+const Coin = require('../../lib/primitives/coin');
+const Output = require('../../lib/primitives/output');
+const BufferReader = require('../../lib/utils/reader');
+const StaticWriter = require('../../lib/utils/staticwriter');
+const encoding = require('../../lib/utils/encoding');
+const compressor = require('../../lib/coins/compress');
+const compress = compressor.compress;
+const decompress = compressor.decompress;
+
+/**
+ * Represents the outputs for a single transaction.
+ * @alias module:coins.Coins
+ * @constructor
+ * @param {Object?} options - Options object.
+ * @property {Hash} hash - Transaction hash.
+ * @property {Number} version - Transaction version.
+ * @property {Number} height - Transaction height (-1 if unconfirmed).
+ * @property {Boolean} coinbase - Whether the containing
+ * transaction is a coinbase.
+ * @property {CoinEntry[]} outputs - Coins.
+ */
+
+function Coins(options) {
+  if (!(this instanceof Coins))
+    return new Coins(options);
+
+  this.version = 1;
+  this.hash = encoding.NULL_HASH;
+  this.height = -1;
+  this.coinbase = true;
+  this.outputs = [];
+
+  if (options)
+    this.fromOptions(options);
+}
+
+/**
+ * Inject properties from options object.
+ * @private
+ * @param {Object} options
+ */
+
+Coins.prototype.fromOptions = function fromOptions(options) {
+  if (options.version != null) {
+    assert(util.isUInt32(options.version));
+    this.version = options.version;
+  }
+
+  if (options.hash) {
+    assert(typeof options.hash === 'string');
+    this.hash = options.hash;
+  }
+
+  if (options.height != null) {
+    assert(util.isNumber(options.height));
+    this.height = options.height;
+  }
+
+  if (options.coinbase != null) {
+    assert(typeof options.coinbase === 'boolean');
+    this.coinbase = options.coinbase;
+  }
+
+  if (options.outputs) {
+    assert(Array.isArray(options.outputs));
+    this.outputs = options.outputs;
+    this.cleanup();
+  }
+
+  return this;
+};
+
+/**
+ * Instantiate coins from options object.
+ * @param {Object} options
+ * @returns {Coins}
+ */
+
+Coins.fromOptions = function fromOptions(options) {
+  return new Coins().fromOptions(options);
+};
+
+/**
+ * Add a single entry to the collection.
+ * @param {Number} index
+ * @param {CoinEntry} entry
+ */
+
+Coins.prototype.add = function add(index, entry) {
+  assert(index >= 0);
+
+  while (this.outputs.length <= index)
+    this.outputs.push(null);
+
+  assert(!this.outputs[index]);
+
+  this.outputs[index] = entry;
+};
+
+/**
+ * Add a single output to the collection.
+ * @param {Number} index
+ * @param {Output} output
+ */
+
+Coins.prototype.addOutput = function addOutput(index, output) {
+  assert(!output.script.isUnspendable());
+  this.add(index, CoinEntry.fromOutput(output));
+};
+
+/**
+ * Add a single coin to the collection.
+ * @param {Coin} coin
+ */
+
+Coins.prototype.addCoin = function addCoin(coin) {
+  assert(!coin.script.isUnspendable());
+  this.add(coin.index, CoinEntry.fromCoin(coin));
+};
+
+/**
+ * Test whether the collection has a coin.
+ * @param {Number} index
+ * @returns {Boolean}
+ */
+
+Coins.prototype.has = function has(index) {
+  if (index >= this.outputs.length)
+    return false;
+
+  return this.outputs[index] != null;
+};
+
+/**
+ * Test whether the collection
+ * has an unspent coin.
+ * @param {Number} index
+ * @returns {Boolean}
+ */
+
+Coins.prototype.isUnspent = function isUnspent(index) {
+  let output;
+
+  if (index >= this.outputs.length)
+    return false;
+
+  output = this.outputs[index];
+
+  if (!output || output.spent)
+    return false;
+
+  return true;
+};
+
+/**
+ * Get a coin entry.
+ * @param {Number} index
+ * @returns {CoinEntry}
+ */
+
+Coins.prototype.get = function get(index) {
+  if (index >= this.outputs.length)
+    return;
+
+  return this.outputs[index];
+};
+
+/**
+ * Get an output.
+ * @param {Number} index
+ * @returns {Output}
+ */
+
+Coins.prototype.getOutput = function getOutput(index) {
+  let entry = this.get(index);
+
+  if (!entry)
+    return;
+
+  return entry.toOutput();
+};
+
+/**
+ * Get a coin.
+ * @param {Number} index
+ * @returns {Coin}
+ */
+
+Coins.prototype.getCoin = function getCoin(index) {
+  let entry = this.get(index);
+
+  if (!entry)
+    return;
+
+  return entry.toCoin(this, index);
+};
+
+/**
+ * Spend a coin entry and return it.
+ * @param {Number} index
+ * @returns {CoinEntry}
+ */
+
+Coins.prototype.spend = function spend(index) {
+  let entry = this.get(index);
+
+  if (!entry || entry.spent)
+    return;
+
+  entry.spent = true;
+
+  return entry;
+};
+
+/**
+ * Remove a coin entry and return it.
+ * @param {Number} index
+ * @returns {CoinEntry}
+ */
+
+Coins.prototype.remove = function remove(index) {
+  let entry = this.get(index);
+
+  if (!entry)
+    return false;
+
+  this.outputs[index] = null;
+  this.cleanup();
+
+  return entry;
+};
+
+/**
+ * Calculate unspent length of coins.
+ * @returns {Number}
+ */
+
+Coins.prototype.length = function length() {
+  let len = this.outputs.length;
+
+  while (len > 0 && !this.isUnspent(len - 1))
+    len--;
+
+  return len;
+};
+
+/**
+ * Cleanup spent outputs (remove pruned).
+ */
+
+Coins.prototype.cleanup = function cleanup() {
+  let len = this.outputs.length;
+
+  while (len > 0 && !this.outputs[len - 1])
+    len--;
+
+  this.outputs.length = len;
+};
+
+/**
+ * Test whether the coins are fully spent.
+ * @returns {Boolean}
+ */
+
+Coins.prototype.isEmpty = function isEmpty() {
+  return this.length() === 0;
+};
+
+/*
+ * Coins serialization:
+ * version: varint
+ * height: uint32
+ * header-code: varint
+ *   bit 1: coinbase
+ *   bit 2: first output unspent
+ *   bit 3: second output unspent
+ *   bit 4-32: spent-field size
+ * spent-field: bitfield (0=spent, 1=unspent)
+ * outputs (repeated):
+ *   value: varint
+ *   compressed-script:
+ *     prefix: 0x00 = 20 byte pubkey hash
+ *             0x01 = 20 byte script hash
+ *             0x02-0x05 = 32 byte ec-key x-value
+ *             0x06-0x09 = reserved
+ *             >=0x10 = varint-size + 10 | raw script
+ *     data: script data, dictated by the prefix
+ *
+ * The compression below sacrifices some cpu in exchange
+ * for reduced size, but in some cases the use of varints
+ * actually increases speed (varint versions and values
+ * for example). We do as much compression as possible
+ * without sacrificing too much cpu. Value compression
+ * is intentionally excluded for now as it seems to be
+ * too much of a perf hit. Maybe when v8 optimizes
+ * non-smi arithmetic better we can enable it.
+ */
+
+/**
+ * Calculate header code.
+ * @param {Number} len
+ * @param {Number} size
+ * @returns {Number}
+ */
+
+Coins.prototype.header = function header(len, size) {
+  let first = this.isUnspent(0);
+  let second = this.isUnspent(1);
+  let offset = 0;
+  let code;
+
+  // Throw if we're fully spent.
+  assert(len !== 0, 'Cannot serialize fully-spent coins.');
+
+  // First and second bits
+  // have a double meaning.
+  if (!first && !second) {
+    assert(size !== 0);
+    offset = 1;
+  }
+
+  // Calculate header code.
+  code = 8 * (size - offset);
+
+  if (this.coinbase)
+    code += 1;
+
+  if (first)
+    code += 2;
+
+  if (second)
+    code += 4;
+
+  return code;
+};
+
+/**
+ * Serialize the coins object.
+ * @returns {Buffer}
+ */
+
+Coins.prototype.toRaw = function toRaw() {
+  let len = this.length();
+  let size = Math.floor((len + 5) / 8);
+  let code = this.header(len, size);
+  let total = this.getSize(len, size, code);
+  let bw = new StaticWriter(total);
+
+  // Write headers.
+  bw.writeVarint(this.version);
+  bw.writeU32(this.height);
+  bw.writeVarint(code);
+
+  // Write the spent field.
+  for (let i = 0; i < size; i++) {
+    let ch = 0;
+    for (let j = 0; j < 8 && 2 + i * 8 + j < len; j++) {
+      if (this.isUnspent(2 + i * 8 + j))
+        ch |= 1 << j;
+    }
+    bw.writeU8(ch);
+  }
+
+  // Write the compressed outputs.
+  for (let i = 0; i < len; i++) {
+    let output = this.outputs[i];
+
+    if (!output || output.spent)
+      continue;
+
+    output.toWriter(bw);
+  }
+
+  return bw.render();
+};
+
+/**
+ * Calculate coins size.
+ * @param {Number} code
+ * @param {Number} size
+ * @param {Number} len
+ * @returns {Number}
+ */
+
+Coins.prototype.getSize = function getSize(len, size, code) {
+  let total = 0;
+
+  total += encoding.sizeVarint(this.version);
+  total += 4;
+  total += encoding.sizeVarint(code);
+  total += size;
+
+  // Write the compressed outputs.
+  for (let i = 0; i < len; i++) {
+    let output = this.outputs[i];
+
+    if (!output || output.spent)
+      continue;
+
+    total += output.getSize();
+  }
+
+  return total;
+};
+
+/**
+ * Inject data from serialized coins.
+ * @private
+ * @param {Buffer} data
+ * @param {Hash} hash
+ * @returns {Coins}
+ */
+
+Coins.prototype.fromRaw = function fromRaw(data, hash) {
+  let br = new BufferReader(data);
+  let first = null;
+  let second = null;
+  let code, size, offset;
+
+  // Inject hash (passed by caller).
+  this.hash = hash;
+
+  // Read headers.
+  this.version = br.readVarint();
+  this.height = br.readU32();
+  code = br.readVarint();
+  this.coinbase = (code & 1) !== 0;
+
+  // Recalculate size.
+  size = code / 8 | 0;
+
+  if ((code & 6) === 0)
+    size += 1;
+
+  // Setup spent field.
+  offset = br.offset;
+  br.seek(size);
+
+  // Read first two outputs.
+  if ((code & 2) !== 0)
+    first = CoinEntry.fromReader(br);
+
+  if ((code & 4) !== 0)
+    second = CoinEntry.fromReader(br);
+
+  this.outputs.push(first);
+  this.outputs.push(second);
+
+  // Read outputs.
+  for (let i = 0; i < size; i++) {
+    let ch = br.data[offset++];
+    for (let j = 0; j < 8; j++) {
+      if ((ch & (1 << j)) === 0) {
+        this.outputs.push(null);
+        continue;
+      }
+      this.outputs.push(CoinEntry.fromReader(br));
+    }
+  }
+
+  this.cleanup();
+
+  return this;
+};
+
+/**
+ * Parse a single serialized coin.
+ * @param {Buffer} data
+ * @param {Hash} hash
+ * @param {Number} index
+ * @returns {Coin}
+ */
+
+Coins.parseCoin = function parseCoin(data, hash, index) {
+  let br = new BufferReader(data);
+  let coin = new Coin();
+  let code, size, offset;
+
+  // Inject outpoint (passed by caller).
+  coin.hash = hash;
+  coin.index = index;
+
+  // Read headers.
+  coin.version = br.readVarint();
+  coin.height = br.readU32();
+  code = br.readVarint();
+  coin.coinbase = (code & 1) !== 0;
+
+  // Recalculate size.
+  size = code / 8 | 0;
+
+  if ((code & 6) === 0)
+    size += 1;
+
+  if (index >= 2 + size * 8)
+    return;
+
+  // Setup spent field.
+  offset = br.offset;
+  br.seek(size);
+
+  // Read first two outputs.
+  for (let i = 0; i < 2; i++) {
+    if ((code & (2 << i)) !== 0) {
+      if (index === 0) {
+        decompress.coin(coin, br);
+        return coin;
+      }
+      decompress.skip(br);
+    } else {
+      if (index === 0)
+        return;
+    }
+    index -= 1;
+  }
+
+  // Read outputs.
+  for (let i = 0; i < size; i++) {
+    let ch = br.data[offset++];
+    for (let j = 0; j < 8; j++) {
+      if ((ch & (1 << j)) !== 0) {
+        if (index === 0) {
+          decompress.coin(coin, br);
+          return coin;
+        }
+        decompress.skip(br);
+      } else {
+        if (index === 0)
+          return;
+      }
+      index -= 1;
+    }
+  }
+};
+
+/**
+ * Instantiate coins from a buffer.
+ * @param {Buffer} data
+ * @param {Hash} hash - Transaction hash.
+ * @returns {Coins}
+ */
+
+Coins.fromRaw = function fromRaw(data, hash) {
+  return new Coins().fromRaw(data, hash);
+};
+
+/**
+ * Inject properties from tx.
+ * @private
+ * @param {TX} tx
+ * @param {Number} height
+ */
+
+Coins.prototype.fromTX = function fromTX(tx, height) {
+  let output;
+
+  assert(typeof height === 'number');
+
+  this.version = tx.version;
+  this.hash = tx.hash('hex');
+  this.height = height;
+  this.coinbase = tx.isCoinbase();
+
+  for (output of tx.outputs) {
+    if (output.script.isUnspendable()) {
+      this.outputs.push(null);
+      continue;
+    }
+    this.outputs.push(CoinEntry.fromOutput(output));
+  }
+
+  this.cleanup();
+
+  return this;
+};
+
+/**
+ * Instantiate a coins object from a transaction.
+ * @param {TX} tx
+ * @param {Number} height
+ * @returns {Coins}
+ */
+
+Coins.fromTX = function fromTX(tx, height) {
+  return new Coins().fromTX(tx, height);
+};
+
+/**
+ * A coin entry is an object which defers
+ * parsing of a coin. Say there is a transaction
+ * with 100 outputs. When a block comes in,
+ * there may only be _one_ input in that entire
+ * block which redeems an output from that
+ * transaction. When parsing the Coins, there
+ * is no sense to get _all_ of them into their
+ * abstract form. A coin entry is just a
+ * pointer to that coin in the Coins buffer, as
+ * well as a size. Parsing and decompression
+ * is done only if that coin is being redeemed.
+ * @alias module:coins.CoinEntry
+ * @constructor
+ * @property {Number} offset
+ * @property {Number} size
+ * @property {Buffer} raw
+ * @property {Output|null} output
+ * @property {Boolean} spent
+ */
+
+function CoinEntry() {
+  this.offset = 0;
+  this.size = 0;
+  this.raw = null;
+  this.output = null;
+  this.spent = false;
+}
+
+/**
+ * Instantiate a reader at the correct offset.
+ * @private
+ * @returns {BufferReader}
+ */
+
+CoinEntry.prototype.reader = function reader() {
+  let br;
+
+  assert(this.raw);
+
+  br = new BufferReader(this.raw);
+  br.offset = this.offset;
+
+  return br;
+};
+
+/**
+ * Parse the deferred data and return a coin.
+ * @param {Coins} coins
+ * @param {Number} index
+ * @returns {Coin}
+ */
+
+CoinEntry.prototype.toCoin = function toCoin(coins, index) {
+  let coin = new Coin();
+  let output = this.toOutput();
+
+  // Load in all necessary properties
+  // from the parent Coins object.
+  coin.version = coins.version;
+  coin.coinbase = coins.coinbase;
+  coin.height = coins.height;
+  coin.hash = coins.hash;
+  coin.index = index;
+  coin.script = output.script;
+  coin.value = output.value;
+
+  return coin;
+};
+
+/**
+ * Parse the deferred data and return an output.
+ * @returns {Output}
+ */
+
+CoinEntry.prototype.toOutput = function toOutput() {
+  if (!this.output) {
+    this.output = new Output();
+    decompress.output(this.output, this.reader());
+  }
+  return this.output;
+};
+
+/**
+ * Calculate coin entry size.
+ * @returns {Number}
+ */
+
+CoinEntry.prototype.getSize = function getSize() {
+  if (!this.raw)
+    return compress.size(this.output);
+
+  return this.size;
+};
+
+/**
+ * Slice off the part of the buffer
+ * relevant to this particular coin.
+ */
+
+CoinEntry.prototype.toWriter = function toWriter(bw) {
+  if (!this.raw) {
+    assert(this.output);
+    compress.output(this.output, bw);
+    return bw;
+  }
+
+  // If we read this coin from the db and
+  // didn't use it, it's still in its
+  // compressed form. Just write it back
+  // as a buffer for speed.
+  bw.copy(this.raw, this.offset, this.offset + this.size);
+
+  return bw;
+};
+
+/**
+ * Instantiate coin entry from reader.
+ * @param {BufferReader} br
+ * @returns {CoinEntry}
+ */
+
+CoinEntry.fromReader = function fromReader(br) {
+  let entry = new CoinEntry();
+  entry.offset = br.offset;
+  entry.size = decompress.skip(br);
+  entry.raw = br.data;
+  return entry;
+};
+
+/**
+ * Instantiate coin entry from output.
+ * @param {Output} output
+ * @returns {CoinEntry}
+ */
+
+CoinEntry.fromOutput = function fromOutput(output) {
+  let entry = new CoinEntry();
+  entry.output = output;
+  return entry;
+};
+
+/**
+ * Instantiate coin entry from coin.
+ * @param {Coin} coin
+ * @returns {CoinEntry}
+ */
+
+CoinEntry.fromCoin = function fromCoin(coin) {
+  let entry = new CoinEntry();
+  let output = new Output();
+  output.value = coin.value;
+  output.script = coin.script;
+  entry.output = output;
+  return entry;
+};
+
+/*
+ * Expose
+ */
+
+exports = Coins;
+exports.Coins = Coins;
+exports.CoinEntry = CoinEntry;
+
+module.exports = exports;

migrate/coins/coinview.js

@@ -0,0 +1,479 @@
+/*!
+ * coinview.js - coin viewpoint object for bcoin
+ * Copyright (c) 2014-2017, Christopher Jeffrey (MIT License).
+ * https://github.com/bcoin-org/bcoin
+ */
+
+'use strict';
+
+const assert = require('assert');
+const Coins = require('../../lib/coins/coins');
+const UndoCoins = require('../../lib/coins/undocoins');
+const BufferReader = require('../../lib/utils/reader');
+const BufferWriter = require('../../lib/utils/writer');
+const CoinEntry = Coins.CoinEntry;
+
+/**
+ * Represents a coin viewpoint:
+ * a snapshot of {@link Coins} objects.
+ * @alias module:coins.CoinView
+ * @constructor
+ * @property {Object} map
+ * @property {UndoCoins} undo
+ */
+
+function CoinView() {
+  if (!(this instanceof CoinView))
+    return new CoinView();
+
+  this.map = new Map();
+  this.undo = new UndoCoins();
+}
+
+/**
+ * Get coins.
+ * @param {Hash} hash
+ * @returns {Coins} coins
+ */
+
+CoinView.prototype.get = function get(hash) {
+  return this.map.get(hash);
+};
+
+/**
+ * Test whether the view has an entry.
+ * @param {Hash} hash
+ * @returns {Boolean}
+ */
+
+CoinView.prototype.has = function has(hash) {
+  return this.map.has(hash);
+};
+
+/**
+ * Add coins to the collection.
+ * @param {Coins} coins
+ */
+
+CoinView.prototype.add = function add(coins) {
+  this.map.set(coins.hash, coins);
+  return coins;
+};
+
+/**
+ * Remove coins from the collection.
+ * @param {Coins} coins
+ * @returns {Boolean}
+ */
+
+CoinView.prototype.remove = function remove(hash) {
+  if (!this.map.has(hash))
+    return false;
+
+  this.map.delete(hash);
+
+  return true;
+};
+
+/**
+ * Add a tx to the collection.
+ * @param {TX} tx
+ * @param {Number} height
+ */
+
+CoinView.prototype.addTX = function addTX(tx, height) {
+  let coins = Coins.fromTX(tx, height);
+  return this.add(coins);
+};
+
+/**
+ * Remove a tx from the collection.
+ * @param {TX} tx
+ * @param {Number} height
+ */
+
+CoinView.prototype.removeTX = function removeTX(tx, height) {
+  let coins = Coins.fromTX(tx, height);
+  coins.outputs.length = 0;
+  return this.add(coins);
+};
+
+/**
+ * Add a coin to the collection.
+ * @param {Coin} coin
+ */
+
+CoinView.prototype.addCoin = function addCoin(coin) {
+  let coins = this.get(coin.hash);
+
+  if (!coins) {
+    coins = new Coins();
+    coins.hash = coin.hash;
+    coins.height = coin.height;
+    coins.coinbase = coin.coinbase;
+    this.add(coins);
+  }
+
+  if (coin.script.isUnspendable())
+    return;
+
+  if (!coins.has(coin.index))
+    coins.addCoin(coin);
+};
+
+/**
+ * Add an output to the collection.
+ * @param {Hash} hash
+ * @param {Number} index
+ * @param {Output} output
+ */
+
+CoinView.prototype.addOutput = function addOutput(hash, index, output) {
+  let coins = this.get(hash);
+
+  if (!coins) {
+    coins = new Coins();
+    coins.hash = hash;
+    coins.height = -1;
+    coins.coinbase = false;
+    this.add(coins);
+  }
+
+  if (output.script.isUnspendable())
+    return;
+
+  if (!coins.has(index))
+    coins.addOutput(index, output);
+};
+
+/**
+ * Spend an output.
+ * @param {Hash} hash
+ * @param {Number} index
+ * @returns {Boolean}
+ */
+
+CoinView.prototype.spendOutput = function spendOutput(hash, index) {
+  let coins = this.get(hash);
+
+  if (!coins)
+    return false;
+
+  return this.spendFrom(coins, index);
+};
+
+/**
+ * Remove an output.
+ * @param {Hash} hash
+ * @param {Number} index
+ * @returns {Boolean}
+ */
+
+CoinView.prototype.removeOutput = function removeOutput(hash, index) {
+  let coins = this.get(hash);
+
+  if (!coins)
+    return false;
+
+  return coins.remove(index);
+};
+
+/**
+ * Spend a coin from coins object.
+ * @param {Coins} coins
+ * @param {Number} index
+ * @returns {Boolean}
+ */
+
+CoinView.prototype.spendFrom = function spendFrom(coins, index) {
+  let entry = coins.spend(index);
+  let undo;
+
+  if (!entry)
+    return false;
+
+  this.undo.push(entry);
+
+  if (coins.isEmpty()) {
+    undo = this.undo.top();
+    undo.height = coins.height;
+    undo.coinbase = coins.coinbase;
+    undo.version = coins.version;
+    assert(undo.height !== -1);
+  }
+
+  return true;
+};
+
+/**
+ * Get a single coin by input.
+ * @param {Input} input
+ * @returns {Coin}
+ */
+
+CoinView.prototype.getCoin = function getCoin(input) {
+  let coins = this.get(input.prevout.hash);
+
+  if (!coins)
+    return;
+
+  return coins.getCoin(input.prevout.index);
+};
+
+/**
+ * Get a single output by input.
+ * @param {Input} input
+ * @returns {Output}
+ */
+
+CoinView.prototype.getOutput = function getOutput(input) {
+  let coins = this.get(input.prevout.hash);
+
+  if (!coins)
+    return;
+
+  return coins.getOutput(input.prevout.index);
+};
+
+/**
+ * Get a single entry by input.
+ * @param {Input} input
+ * @returns {CoinEntry}
+ */
+
+CoinView.prototype.getEntry = function getEntry(input) {
+  let coins = this.get(input.prevout.hash);
+
+  if (!coins)
+    return;
+
+  return coins.get(input.prevout.index);
+};
+
+/**
+ * Test whether the view has an entry by input.
+ * @param {Input} input
+ * @returns {Boolean}
+ */
+
+CoinView.prototype.hasEntry = function hasEntry(input) {
+  let coins = this.get(input.prevout.hash);
+
+  if (!coins)
+    return false;
+
+  return coins.has(input.prevout.index);
+};
+
+/**
+ * Get coins height by input.
+ * @param {Input} input
+ * @returns {Number}
+ */
+
+CoinView.prototype.getHeight = function getHeight(input) {
+  let coins = this.get(input.prevout.hash);
+
+  if (!coins)
+    return -1;
+
+  return coins.height;
+};
+
+/**
+ * Get coins coinbase flag by input.
+ * @param {Input} input
+ * @returns {Boolean}
+ */
+
+CoinView.prototype.isCoinbase = function isCoinbase(input) {
+  let coins = this.get(input.prevout.hash);
+
+  if (!coins)
+    return false;
+
+  return coins.coinbase;
+};
+
+/**
+ * Retrieve coins from database.
+ * @method
+ * @param {ChainDB} db
+ * @param {TX} tx
+ * @returns {Promise} - Returns {@link Coins}.
+ */
+
+CoinView.prototype.readCoins = async function readCoins(db, hash) {
+  let coins = this.map.get(hash);
+
+  if (!coins) {
+    coins = await db.getCoins(hash);
+
+    if (!coins)
+      return;
+
+    this.map.set(hash, coins);
+  }
+
+  return coins;
+};
+
+/**
+ * Read all input coins into unspent map.
+ * @method
+ * @param {ChainDB} db
+ * @param {TX} tx
+ * @returns {Promise} - Returns {Boolean}.
+ */
+
+CoinView.prototype.ensureInputs = async function ensureInputs(db, tx) {
+  let found = true;
+
+  for (let input of tx.inputs) {
+    if (!(await this.readCoins(db, input.prevout.hash)))
+      found = false;
+  }
+
+  return found;
+};
+
+/**
+ * Spend coins for transaction.
+ * @method
+ * @param {ChainDB} db
+ * @param {TX} tx
+ * @returns {Promise} - Returns {Boolean}.
+ */
+
+CoinView.prototype.spendInputs = async function spendInputs(db, tx) {
+  for (let input of tx.inputs) {
+    let prevout = input.prevout;
+    let coins = await this.readCoins(db, prevout.hash);
+
+    if (!coins)
+      return false;
+
+    if (!this.spendFrom(coins, prevout.index))
+      return false;
+  }
+
+  return true;
+};
+
+/**
+ * Convert collection to an array.
+ * @returns {Coins[]}
+ */
+
+CoinView.prototype.toArray = function toArray() {
+  let out = [];
+
+  for (let coins of this.map.values())
+    out.push(coins);
+
+  return out;
+};
+
+/**
+ * Calculate serialization size.
+ * @returns {Number}
+ */
+
+CoinView.prototype.getSize = function getSize(tx) {
+  let size = 0;
+
+  size += tx.inputs.length;
+
+  for (let input of tx.inputs) {
+    let entry = this.getEntry(input);
+
+    if (!entry)
+      continue;
+
+    size += entry.getSize();
+  }
+
+  return size;
+};
+
+/**
+ * Write coin data to buffer writer
+ * as it pertains to a transaction.
+ * @param {BufferWriter} bw
+ * @param {TX} tx
+ */
+
+CoinView.prototype.toWriter = function toWriter(bw, tx) {
+  for (let input of tx.inputs) {
+    let prevout = input.prevout;
+    let coins = this.get(prevout.hash);
+    let entry;
+
+    if (!coins) {
+      bw.writeU8(0);
+      continue;
+    }
+
+    entry = coins.get(prevout.index);
+
+    if (!entry) {
+      bw.writeU8(0);
+      continue;
+    }
+
+    bw.writeU8(1);
+    entry.toWriter(bw);
+  }
+
+  return bw;
+};
+
+/**
+ * Read serialized view data from a buffer
+ * reader as it pertains to a transaction.
+ * @private
+ * @param {BufferReader} br
+ * @param {TX} tx
+ */
+
+CoinView.prototype.fromReader = function fromReader(br, tx) {
+  for (let input of tx.inputs) {
+    let prevout = input.prevout;
+    let coins, entry;
+
+    if (br.readU8() === 0)
+      continue;
+
+    coins = this.get(prevout.hash);
+
+    if (!coins) {
+      coins = new Coins();
+      coins.hash = prevout.hash;
+      coins.coinbase = false;
+      this.add(coins);
+    }
+
+    entry = CoinEntry.fromReader(br);
+    coins.add(prevout.index, entry);
+  }
+
+  return this;
+};
+
+/**
+ * Read serialized view data from a buffer
+ * reader as it pertains to a transaction.
+ * @param {BufferReader} br
+ * @param {TX} tx
+ * @returns {CoinView}
+ */
+
+CoinView.fromReader = function fromReader(br, tx) {
+  return new CoinView().fromReader(br, tx);
+};
+
+/*
+ * Expose
+ */
+
+module.exports = CoinView;

migrate/coins/compress.js

@@ -0,0 +1,419 @@
+/*!
+ * compress.js - coin compressor for bcoin
+ * Copyright (c) 2014-2017, Christopher Jeffrey (MIT License).
+ * https://github.com/bcoin-org/bcoin
+ */
+
+'use strict';
+
+/**
+ * @module coins/compress
+ * @ignore
+ */
+
+const assert = require('assert');
+const secp256k1 = require('../../lib/crypto/secp256k1');
+const encoding = require('../../lib/utils/encoding');
+const consensus = require('../../lib/protocol/consensus');
+
+/*
+ * Constants
+ */
+
+const COMPRESS_TYPES = 10; // Space for 4 extra.
+const EMPTY_BUFFER = Buffer.alloc(0);
+
+/**
+ * Compress a script, write directly to the buffer.
+ * @param {Script} script
+ * @param {BufferWriter} bw
+ */
+
+function compressScript(script, bw) {
+  let data;
+
+  // Attempt to compress the output scripts.
+  // We can _only_ ever compress them if
+  // they are serialized as minimaldata, as
+  // we need to recreate them when we read
+  // them.
+
+  // P2PKH -> 0 | key-hash
+  // Saves 5 bytes.
+  if (script.isPubkeyhash(true)) {
+    data = script.code[2].data;
+    bw.writeU8(0);
+    bw.writeBytes(data);
+    return bw;
+  }
+
+  // P2SH -> 1 | script-hash
+  // Saves 3 bytes.
+  if (script.isScripthash()) {
+    data = script.code[1].data;
+    bw.writeU8(1);
+    bw.writeBytes(data);
+    return bw;
+  }
+
+  // P2PK -> 2-5 | compressed-key
+  // Only works if the key is valid.
+  // Saves up to 35 bytes.
+  if (script.isPubkey(true)) {
+    data = script.code[0].data;
+    if (publicKeyVerify(data)) {
+      data = compressKey(data);
+      bw.writeBytes(data);
+      return bw;
+    }
+  }
+
+  // Raw -> varlen + 10 | script
+  bw.writeVarint(script.raw.length + COMPRESS_TYPES);
+  bw.writeBytes(script.raw);
+
+  return bw;
+}
+
+/**
+ * Decompress a script from buffer reader.
+ * @param {Script} script
+ * @param {BufferReader} br
+ */
+
+function decompressScript(script, br) {
+  let size, data;
+
+  // Decompress the script.
+  switch (br.readU8()) {
+    case 0:
+      data = br.readBytes(20, true);
+      script.fromPubkeyhash(data);
+      break;
+    case 1:
+      data = br.readBytes(20, true);
+      script.fromScripthash(data);
+      break;
+    case 2:
+    case 3:
+    case 4:
+    case 5:
+      br.offset -= 1;
+      data = br.readBytes(33, true);
+      // Decompress the key. If this fails,
+      // we have database corruption!
+      data = decompressKey(data);
+      script.fromPubkey(data);
+      break;
+    default:
+      br.offset -= 1;
+      size = br.readVarint() - COMPRESS_TYPES;
+      if (size > consensus.MAX_SCRIPT_SIZE) {
+        // This violates consensus rules.
+        // We don't need to read it.
+        script.fromNulldata(EMPTY_BUFFER);
+        br.seek(size);
+      } else {
+        data = br.readBytes(size);
+        script.fromRaw(data);
+      }
+      break;
+  }
+
+  return script;
+}
+
+/**
+ * Calculate script size.
+ * @returns {Number}
+ */
+
+function sizeScript(script) {
+  let size, data;
+
+  if (script.isPubkeyhash(true))
+    return 21;
+
+  if (script.isScripthash())
+    return 21;
+
+  if (script.isPubkey(true)) {
+    data = script.code[0].data;
+    if (publicKeyVerify(data))
+      return 33;
+  }
+
+  size = 0;
+  size += encoding.sizeVarint(script.raw.length + COMPRESS_TYPES);
+  size += script.raw.length;
+
+  return size;
+}
+
+/**
+ * Compress an output.
+ * @param {Output} output
+ * @param {BufferWriter} bw
+ */
+
+function compressOutput(output, bw) {
+  bw.writeVarint(output.value);
+  compressScript(output.script, bw);
+  return bw;
+}
+
+/**
+ * Decompress a script from buffer reader.
+ * @param {Output} output
+ * @param {BufferReader} br
+ */
+
+function decompressOutput(output, br) {
+  output.value = br.readVarint();
+  decompressScript(output.script, br);
+  return output;
+}
+
+/**
+ * Calculate output size.
+ * @returns {Number}
+ */
+
+function sizeOutput(output) {
+  let size = 0;
+  size += encoding.sizeVarint(output.value);
+  size += sizeScript(output.script);
+  return size;
+}
+
+/**
+ * Compress an output.
+ * @param {Coin} coin
+ * @param {BufferWriter} bw
+ */
+
+function compressCoin(coin, bw) {
+  bw.writeVarint(coin.value);
+  compressScript(coin.script, bw);
+  return bw;
+}
+
+/**
+ * Decompress a script from buffer reader.
+ * @param {Coin} coin
+ * @param {BufferReader} br
+ */
+
+function decompressCoin(coin, br) {
+  coin.value = br.readVarint();
+  decompressScript(coin.script, br);
+  return coin;
+}
+
+/**
+ * Skip past a compressed output.
+ * @param {BufferWriter} bw
+ * @returns {Number}
+ */
+
+function skipOutput(br) {
+  let start = br.offset;
+
+  // Skip past the value.
+  br.skipVarint();
+
+  // Skip past the compressed scripts.
+  switch (br.readU8()) {
+    case 0:
+    case 1:
+      br.seek(20);
+      break;
+    case 2:
+    case 3:
+    case 4:
+    case 5:
+      br.seek(32);
+      break;
+    default:
+      br.offset -= 1;
+      br.seek(br.readVarint() - COMPRESS_TYPES);
+      break;
+  }
+
+  return br.offset - start;
+}
+
+/**
+ * Compress value using an exponent. Takes advantage of
+ * the fact that many bitcoin values are divisible by 10.
+ * @see https://github.com/btcsuite/btcd/blob/master/blockchain/compress.go
+ * @param {Amount} value
+ * @returns {Number}
+ */
+
+function compressValue(value) {
+  let exp, last;
+
+  if (value === 0)
+    return 0;
+
+  exp = 0;
+  while (value % 10 === 0 && exp < 9) {
+    value /= 10;
+    exp++;
+  }
+
+  if (exp < 9) {
+    last = value % 10;
+    value = (value - last) / 10;
+    return 1 + 10 * (9 * value + last - 1) + exp;
+  }
+
+  return 10 + 10 * (value - 1);
+}
+
+/**
+ * Decompress value.
+ * @param {Number} value - Compressed value.
+ * @returns {Amount} value
+ */
+
+function decompressValue(value) {
+  let exp, n, last;
+
+  if (value === 0)
+    return 0;
+
+  value--;
+
+  exp = value % 10;
+  value = (value - exp) / 10;
+
+  if (exp < 9) {
+    last = value % 9;
+    value = (value - last) / 9;
+    n = value * 10 + last + 1;
+  } else {
+    n = value + 1;
+  }
+
+  while (exp > 0) {
+    n *= 10;
+    exp--;
+  }
+
+  return n;
+}
+
+/**
+ * Verify a public key (no hybrid keys allowed).
+ * @param {Buffer} key
+ * @returns {Boolean}
+ */
+
+function publicKeyVerify(key) {
+  if (key.length === 0)
+    return false;
+
+  switch (key[0]) {
+    case 0x02:
+    case 0x03:
+      return key.length === 33;
+    case 0x04:
+      if (key.length !== 65)
+        return false;
+
+      return secp256k1.publicKeyVerify(key);
+    default:
+      return false;
+  }
+}
+
+/**
+ * Compress a public key to coins compression format.
+ * @param {Buffer} key
+ * @returns {Buffer}
+ */
+
+function compressKey(key) {
+  let out;
+
+  switch (key[0]) {
+    case 0x02:
+    case 0x03:
+      // Key is already compressed.
+      out = key;
+      break;
+    case 0x04:
+      // Compress the key normally.
+      out = secp256k1.publicKeyConvert(key, true);
+      // Store the oddness.
+      // Pseudo-hybrid format.
+      out[0] = 0x04 | (key[64] & 0x01);
+      break;
+    default:
+      throw new Error('Bad point format.');
+  }
+
+  assert(out.length === 33);
+
+  return out;
+}
+
+/**
+ * Decompress a public key from the coins compression format.
+ * @param {Buffer} key
+ * @returns {Buffer}
+ */
+
+function decompressKey(key) {
+  let format = key[0];
+  let out;
+
+  assert(key.length === 33);
+
+  switch (format) {
+    case 0x02:
+    case 0x03:
+      return key;
+    case 0x04:
+      key[0] = 0x02;
+      break;
+    case 0x05:
+      key[0] = 0x03;
+      break;
+    default:
+      throw new Error('Bad point format.');
+  }
+
+  // Decompress the key.
+  out = secp256k1.publicKeyConvert(key, false);
+
+  // Reset the first byte so as not to
+  // mutate the original buffer.
+  key[0] = format;
+
+  return out;
+}
+
+/*
+ * Expose
+ */
+
+exports.compress = {
+  output: compressOutput,
+  coin: compressCoin,
+  size: sizeOutput,
+  script: compressScript,
+  value: compressValue,
+  key: compressKey
+};
+
+exports.decompress = {
+  output: decompressOutput,
+  coin: decompressCoin,
+  skip: skipOutput,
+  script: decompressScript,
+  value: decompressValue,
+  key: decompressKey
+};

migrate/coins/index.js

@@ -0,0 +1,16 @@
+/*!
+ * coins/index.js - utxo management for bcoin
+ * Copyright (c) 2016-2017, Christopher Jeffrey (MIT License).
+ * https://github.com/bcoin-org/bcoin
+ */
+
+'use strict';
+
+/**
+ * @module coins
+ */
+
+exports.Coins = require('../../lib/coins/coins');
+exports.CoinView = require('../../lib/coins/coinview');
+exports.compress = require('../../lib/coins/compress');
+exports.UndoCoins = require('../../lib/coins/undocoins');

migrate/coins/undocoins.js

@@ -0,0 +1,336 @@
+/*!
+ * undocoins.js - undocoins object for bcoin
+ * Copyright (c) 2014-2017, Christopher Jeffrey (MIT License).
+ * https://github.com/bcoin-org/bcoin
+ */
+
+'use strict';
+
+const assert = require('assert');
+const BufferReader = require('../../lib/utils/reader');
+const StaticWriter = require('../../lib/utils/staticwriter');
+const encoding = require('../../lib/utils/encoding');
+const Output = require('../../lib/primitives/output');
+const Coins = require('../../lib/coins');
+const compressor = require('../../lib/compress');
+const compress = compressor.compress;
+const decompress = compressor.decompress;
+
+/**
+ * UndoCoins
+ * Coins need to be resurrected from somewhere
+ * during a reorg. The undo coins store all
+ * spent coins in a single record per block
+ * (in a compressed format).
+ * @alias module:coins.UndoCoins
+ * @constructor
+ * @property {UndoCoin[]} items
+ */
+
+function UndoCoins() {
+  if (!(this instanceof UndoCoins))
+    return new UndoCoins();
+
+  this.items = [];
+}
+
+/**
+ * Push coin entry onto undo coin array.
+ * @param {CoinEntry}
+ */
+
+UndoCoins.prototype.push = function push(entry) {
+  let undo = new UndoCoin();
+  undo.entry = entry;
+  this.items.push(undo);
+};
+
+/**
+ * Calculate undo coins size.
+ * @returns {Number}
+ */
+
+UndoCoins.prototype.getSize = function getSize() {
+  let size = 0;
+
+  size += 4;
+
+  for (let coin of this.items)
+    size += coin.getSize();
+
+  return size;
+};
+
+/**
+ * Serialize all undo coins.
+ * @returns {Buffer}
+ */
+
+UndoCoins.prototype.toRaw = function toRaw() {
+  let size = this.getSize();
+  let bw = new StaticWriter(size);
+
+  bw.writeU32(this.items.length);
+
+  for (let coin of this.items)
+    coin.toWriter(bw);
+
+  return bw.render();
+};
+
+/**
+ * Inject properties from serialized data.
+ * @private
+ * @param {Buffer} data
+ * @returns {UndoCoins}
+ */
+
+UndoCoins.prototype.fromRaw = function fromRaw(data) {
+  let br = new BufferReader(data);
+  let count = br.readU32();
+
+  for (let i = 0; i < count; i++)
+    this.items.push(UndoCoin.fromReader(br));
+
+  return this;
+};
+
+/**
+ * Instantiate undo coins from serialized data.
+ * @param {Buffer} data
+ * @returns {UndoCoins}
+ */
+
+UndoCoins.fromRaw = function fromRaw(data) {
+  return new UndoCoins().fromRaw(data);
+};
+
+/**
+ * Test whether the undo coins have any members.
+ * @returns {Boolean}
+ */
+
+UndoCoins.prototype.isEmpty = function isEmpty() {
+  return this.items.length === 0;
+};
+
+/**
+ * Render the undo coins.
+ * @returns {Buffer}
+ */
+
+UndoCoins.prototype.commit = function commit() {
+  let raw = this.toRaw();
+  this.items.length = 0;
+  return raw;
+};
+
+/**
+ * Retrieve the last undo coin.
+ * @returns {UndoCoin}
+ */
+
+UndoCoins.prototype.top = function top() {
+  return this.items[this.items.length - 1];
+};
+
+/**
+ * Re-apply undo coins to a view, effectively unspending them.
+ * @param {CoinView} view
+ * @param {Outpoint} outpoint
+ */
+
+UndoCoins.prototype.apply = function apply(view, outpoint) {
+  let undo = this.items.pop();
+  let hash = outpoint.hash;
+  let index = outpoint.index;
+  let coins;
+
+  assert(undo);
+
+  if (undo.height !== -1) {
+    coins = new Coins();
+
+    assert(!view.map.has(hash));
+    view.map.set(hash, coins);
+
+    coins.hash = hash;
+    coins.coinbase = undo.coinbase;
+    coins.height = undo.height;
+    coins.version = undo.version;
+  } else {
+    coins = view.map.get(hash);
+    assert(coins);
+  }
+
+  coins.addOutput(index, undo.toOutput());
+
+  assert(coins.has(index));
+};
+
+/**
+ * UndoCoin
+ * @alias module:coins.UndoCoin
+ * @constructor
+ * @property {CoinEntry|null} entry
+ * @property {Output|null} output
+ * @property {Number} version
+ * @property {Number} height
+ * @property {Boolean} coinbase
+ */
+
+function UndoCoin() {
+  this.entry = null;
+  this.output = null;
+  this.version = -1;
+  this.height = -1;
+  this.coinbase = false;
+}
+
+/**
+ * Convert undo coin to an output.
+ * @returns {Output}
+ */
+
+UndoCoin.prototype.toOutput = function toOutput() {
+  if (!this.output) {
+    assert(this.entry);
+    return this.entry.toOutput();
+  }
+  return this.output;
+};
+
+/**
+ * Calculate undo coin size.
+ * @returns {Number}
+ */
+
+UndoCoin.prototype.getSize = function getSize() {
+  let height = this.height;
+  let size = 0;
+
+  if (height === -1)
+    height = 0;
+
+  size += encoding.sizeVarint(height * 2 + (this.coinbase ? 1 : 0));
+
+  if (this.height !== -1)
+    size += encoding.sizeVarint(this.version);
+
+  if (this.entry) {
+    // Cached from spend.
+    size += this.entry.getSize();
+  } else {
+    size += compress.size(this.output);
+  }
+
+  return size;
+};
+
+/**
+ * Write the undo coin to a buffer writer.
+ * @param {BufferWriter} bw
+ */
+
+UndoCoin.prototype.toWriter = function toWriter(bw) {
+  let height = this.height;
+
+  assert(height !== 0);
+
+  if (height === -1)
+    height = 0;
+
+  bw.writeVarint(height * 2 + (this.coinbase ? 1 : 0));
+
+  if (this.height !== -1) {
+    assert(this.version !== -1);
+    bw.writeVarint(this.version);
+  }
+
+  if (this.entry) {
+    // Cached from spend.
+    this.entry.toWriter(bw);
+  } else {
+    compress.output(this.output, bw);
+  }
+
+  return bw;
+};
+
+/**
+ * Serialize the undo coin.
+ * @returns {Buffer}
+ */
+
+UndoCoin.prototype.toRaw = function toRaw() {
+  let size = this.getSize();
+  return this.toWriter(new StaticWriter(size)).render();
+};
+
+/**
+ * Inject properties from buffer reader.
+ * @private
+ * @param {BufferReader} br
+ * @returns {UndoCoin}
+ */
+
+UndoCoin.prototype.fromReader = function fromReader(br) {
+  let code = br.readVarint();
+
+  this.output = new Output();
+
+  this.height = code / 2 | 0;
+
+  if (this.height === 0)
+    this.height = -1;
+
+  this.coinbase = (code & 1) !== 0;
+
+  if (this.height !== -1)
+    this.version = br.readVarint();
+
+  decompress.output(this.output, br);
+
+  return this;
+};
+
+/**
+ * Inject properties from serialized data.
+ * @private
+ * @param {Buffer} data
+ * @returns {UndoCoin}
+ */
+
+UndoCoin.prototype.fromRaw = function fromRaw(data) {
+  return this.fromReader(new BufferReader(data));
+};
+
+/**
+ * Instantiate undo coin from serialized data.
+ * @param {Buffer} data
+ * @returns {UndoCoin}
+ */
+
+UndoCoin.fromReader = function fromReader(br) {
+  return new UndoCoin().fromReader(br);
+};
+
+/**
+ * Instantiate undo coin from serialized data.
+ * @param {Buffer} data
+ * @returns {UndoCoin}
+ */
+
+UndoCoin.fromRaw = function fromRaw(data) {
+  return new UndoCoin().fromRaw(data);
+};
+
+/*
+ * Expose
+ */
+
+exports = UndoCoins;
+exports.UndoCoins = UndoCoins;
+exports.UndoCoin = UndoCoin;
+
+module.exports = exports;