walletdb: refactor block handling.

lib/chain/chaindb.js

@@ -1121,8 +1121,11 @@ ChainDB.prototype.scan = co(function* scan(start, filter, iter) {
     block = yield this.getBlock(entry.hash);
     total++;
 
-    if (!block)
+    if (!block) {
-      throw new Error('Block not found.');
+      if (!this.options.spv && !this.options.prune)
+        throw new Error('Block not found.');
+      continue;
+    }
 
     this.logger.info(
       'Scanning block %s (%d).',

lib/node/config.js

@@ -203,6 +203,7 @@ config.parseData = function parseData(data, prefix, dirname) {
   options.noAuth = bool(data.noauth);
 
   // Wallet
+  options.noScan = bool(data.noscan);
   options.wipeNoReally = bool(data.wipenoreally);
 
   options.data = data;

lib/node/fullnode.js

@@ -149,6 +149,7 @@ function Fullnode(options) {
     witness: this.options.witness,
     useCheckpoints: this.options.useCheckpoints,
     maxFiles: this.options.maxFiles,
+    noScan: this.options.noScan,
     wipeNoReally: this.options.wipeNoReally,
     resolution: false,
     verify: false

lib/node/spvnode.js

@@ -87,6 +87,7 @@ function SPVNode(options) {
     location: this.location('walletdb'),
     witness: this.options.witness,
     maxFiles: this.options.maxFiles,
+    noScan: this.options.noScan,
     wipeNoReally: this.options.wipeNoReally,
     resolution: true,
     verify: true

lib/wallet/browser.js

@@ -45,6 +45,12 @@ layout.walletdb = {
     return [+key.slice(1, 11), key.slice(11)];
   },
   R: 'R',
+  c: function c(height) {
+    return 'b' + pad32(height);
+  },
+  cc: function cc(key) {
+    return +key.slice(1);
+  },
   b: function b(height) {
     return 'b' + pad32(height);
   },

lib/wallet/records.js

@@ -1,6 +1,5 @@
 /*!
- * walletdb.js - storage for wallets
+ * records.js - walletdb records
- * Copyright (c) 2014-2015, Fedor Indutny (MIT License)
  * Copyright (c) 2014-2016, Christopher Jeffrey (MIT License).
  * https://github.com/bcoin-org/bcoin
  */
@@ -14,30 +13,28 @@ var BufferReader = require('../utils/reader');
 var BufferWriter = require('../utils/writer');
 
 /**
- * Wallet Block
+ * Wallet Tip
  * @constructor
  * @param {Hash} hash
  * @param {Number} height
  */
 
-function WalletBlock(hash, height, ts) {
+function HeaderRecord(hash, height, ts) {
-  if (!(this instanceof WalletBlock))
+  if (!(this instanceof HeaderRecord))
-    return new WalletBlock(hash, height, ts);
+    return new HeaderRecord(hash, height, ts);
 
   this.hash = hash || constants.NULL_HASH;
   this.height = height != null ? height : -1;
   this.ts = ts || 0;
-  this.txs = [];
-  this.index = {};
 }
 
 /**
  * Clone the block.
- * @returns {WalletBlock}
+ * @returns {HeaderRecord}
  */
 
-WalletBlock.prototype.clone = function clone() {
+HeaderRecord.prototype.clone = function clone() {
-  return new WalletBlock(this.hash, this.height, this.ts);
+  return new HeaderRecord(this.hash, this.height, this.ts);
 };
 
 /**
@@ -46,7 +43,7 @@ WalletBlock.prototype.clone = function clone() {
  * @param {ChainEntry} entry
  */
 
-WalletBlock.prototype.fromEntry = function fromEntry(entry) {
+HeaderRecord.prototype.fromEntry = function fromEntry(entry) {
   this.hash = entry.hash;
   this.height = entry.height;
   this.ts = entry.ts;
@@ -59,48 +56,20 @@ WalletBlock.prototype.fromEntry = function fromEntry(entry) {
  * @param {Object} json
  */
 
-WalletBlock.prototype.fromJSON = function fromJSON(json) {
+HeaderRecord.prototype.fromJSON = function fromJSON(json) {
   this.hash = utils.revHex(json.hash);
   this.height = json.height;
   this.ts = json.ts;
   return this;
 };
 
-/**
- * Instantiate wallet block from serialized data.
- * @private
- * @param {Hash} hash
- * @param {Buffer} data
- */
-
-WalletBlock.prototype.fromRaw = function fromRaw(data) {
-  var p = new BufferReader(data);
-  var i, hash, tx, count;
-
-  this.hash = p.readHash('hex');
-  this.height = p.readU32();
-  this.ts = p.readU32();
-
-  while (p.left()) {
-    hash = p.readHash('hex');
-    tx = new TXHash(hash);
-    count = p.readVarint();
-    for (i = 0; i < count; i++)
-      tx.wids.push(p.readU32());
-    this.txs.push(tx);
-    this.index[tx.hash] = tx;
-  }
-
-  return this;
-};
-
 /**
  * Instantiate wallet block from serialized tip data.
  * @private
  * @param {Buffer} data
  */
 
-WalletBlock.prototype.fromTip = function fromTip(data) {
+HeaderRecord.prototype.fromRaw = function fromRaw(data) {
   var p = new BufferReader(data);
   this.hash = p.readHash('hex');
   this.height = p.readU32();
@@ -111,42 +80,32 @@ WalletBlock.prototype.fromTip = function fromTip(data) {
 /**
  * Instantiate wallet block from chain entry.
  * @param {ChainEntry} entry
- * @returns {WalletBlock}
+ * @returns {HeaderRecord}
  */
 
-WalletBlock.fromEntry = function fromEntry(entry) {
+HeaderRecord.fromEntry = function fromEntry(entry) {
-  return new WalletBlock().fromEntry(entry);
+  return new HeaderRecord().fromEntry(entry);
 };
 
 /**
  * Instantiate wallet block from json object.
  * @param {Object} json
- * @returns {WalletBlock}
+ * @returns {HeaderRecord}
  */
 
-WalletBlock.fromJSON = function fromJSON(json) {
+HeaderRecord.fromJSON = function fromJSON(json) {
-  return new WalletBlock().fromJSON(json);
+  return new HeaderRecord().fromJSON(json);
 };
 
 /**
  * Instantiate wallet block from serialized data.
  * @param {Hash} hash
  * @param {Buffer} data
- * @returns {WalletBlock}
+ * @returns {HeaderRecord}
  */
 
-WalletBlock.fromRaw = function fromRaw(data) {
+HeaderRecord.fromRaw = function fromRaw(data) {
-  return new WalletBlock().fromRaw(data);
+  return new HeaderRecord().fromRaw(data);
-};
-
-/**
- * Instantiate wallet block from serialized tip data.
- * @private
- * @param {Buffer} data
- */
-
-WalletBlock.fromTip = function fromTip(data) {
-  return new WalletBlock().fromTip(data);
 };
 
 /**
@@ -154,7 +113,7 @@ WalletBlock.fromTip = function fromTip(data) {
  * @returns {Buffer}
  */
 
-WalletBlock.prototype.toTip = function toTip(writer) {
+HeaderRecord.prototype.toRaw = function toRaw(writer) {
   var p = new BufferWriter(writer);
 
   p.writeHash(this.hash);
@@ -167,20 +126,80 @@ WalletBlock.prototype.toTip = function toTip(writer) {
   return p;
 };
 
+/**
+ * Convert the block to a more json-friendly object.
+ * @returns {Object}
+ */
+
+HeaderRecord.prototype.toJSON = function toJSON() {
+  return {
+    hash: utils.revHex(this.hash),
+    height: this.height,
+    ts: this.ts
+  };
+};
+
+/**
+ * Wallet Block
+ * @constructor
+ * @param {Hash} hash
+ * @param {Number} height
+ */
+
+function BlockMapRecord(height) {
+  if (!(this instanceof BlockMapRecord))
+    return new BlockMapRecord(height);
+
+  this.height = height != null ? height : -1;
+  this.txs = [];
+  this.index = {};
+}
+
+/**
+ * Instantiate wallet block from serialized data.
+ * @private
+ * @param {Hash} hash
+ * @param {Buffer} data
+ */
+
+BlockMapRecord.prototype.fromRaw = function fromRaw(data) {
+  var p = new BufferReader(data);
+  var i, hash, tx, count;
+
+  while (p.left()) {
+    hash = p.readHash('hex');
+    tx = new TXMapRecord(hash);
+    count = p.readVarint();
+    for (i = 0; i < count; i++)
+      tx.wids.push(p.readU32());
+    this.txs.push(tx);
+    this.index[tx.hash] = tx;
+  }
+
+  return this;
+};
+
+/**
+ * Instantiate wallet block from serialized data.
+ * @param {Hash} hash
+ * @param {Buffer} data
+ * @returns {BlockMapRecord}
+ */
+
+BlockMapRecord.fromRaw = function fromRaw(height, data) {
+  return new BlockMapRecord(height).fromRaw(data);
+};
+
 /**
  * Serialize the wallet block as a block.
  * Contains matching transaction hashes.
  * @returns {Buffer}
  */
 
-WalletBlock.prototype.toRaw = function toRaw(writer) {
+BlockMapRecord.prototype.toRaw = function toRaw(writer) {
   var p = new BufferWriter(writer);
   var i, j, tx;
 
-  p.writeHash(this.hash);
-  p.writeU32(this.height);
-  p.writeU32(this.ts);
-
   for (i = 0; i < this.txs.length; i++) {
     tx = this.txs[i];
     p.writeHash(tx.hash);
@@ -202,11 +221,11 @@ WalletBlock.prototype.toRaw = function toRaw(writer) {
  * @returns {Boolean}
  */
 
-WalletBlock.prototype.add = function add(hash, wid) {
+BlockMapRecord.prototype.add = function add(hash, wid) {
   var tx = this.index[hash];
 
   if (!tx) {
-    tx = new TXHash(hash);
+    tx = new TXMapRecord(hash);
     tx.wids.push(wid);
     this.txs.push(tx);
     this.index[tx.hash] = tx;
@@ -223,7 +242,7 @@ WalletBlock.prototype.add = function add(hash, wid) {
  * @returns {Boolean}
  */
 
-WalletBlock.prototype.remove = function remove(hash, wid) {
+BlockMapRecord.prototype.remove = function remove(hash, wid) {
   var tx = this.index[hash];
   var result;
 
@@ -242,49 +261,69 @@ WalletBlock.prototype.remove = function remove(hash, wid) {
   return true;
 };
 
-/**
- * Convert the block to a more json-friendly object.
- * @returns {Object}
- */
-
-WalletBlock.prototype.toJSON = function toJSON() {
-  return {
-    hash: utils.revHex(this.hash),
-    height: this.height
-  };
-};
-
 /**
  * TX Hash
  * @constructor
  */
 
-function TXHash(hash, wids) {
+function TXMapRecord(hash, wids) {
   this.hash = hash || constants.NULL_HASH;
   this.wids = wids || [];
-  this.id = TXHash.id++;
+  this.id = TXMapRecord.id++;
 }
 
-TXHash.id = 0;
+TXMapRecord.id = 0;
 
-TXHash.prototype.add = function add(wid) {
+TXMapRecord.prototype.add = function add(wid) {
   return utils.binaryInsert(this.wids, wid, cmp, true) !== -1;
 };
 
-TXHash.prototype.remove = function remove(wid) {
+TXMapRecord.prototype.remove = function remove(wid) {
   return utils.binaryRemove(this.wids, wid, cmp);
 };
 
-TXHash.prototype.toRaw = function toRaw(writer) {
+TXMapRecord.prototype.toRaw = function toRaw() {
-  return serializeWallets(this.wids, writer);
+  return serializeWallets(this.wids);
 };
 
-TXHash.prototype.fromRaw = function fromRaw(data) {
+TXMapRecord.prototype.fromRaw = function fromRaw(data) {
-  return parseWallets(data);
+  this.wids = parseWallets(data);
+  return this;
 };
 
-TXHash.fromRaw = function fromRaw(hash, data) {
+TXMapRecord.fromRaw = function fromRaw(hash, data) {
-  return new TXHash(hash).fromRaw(data);
+  return new TXMapRecord(hash).fromRaw(data);
+};
+
+/**
+ * Path Record
+ * @constructor
+ */
+
+function PathMapRecord(hash, wids) {
+  this.hash = hash || constants.NULL_HASH;
+  this.wids = wids || [];
+}
+
+PathMapRecord.prototype.add = function add(wid) {
+  return utils.binaryInsert(this.wids, wid, cmp, true) !== -1;
+};
+
+PathMapRecord.prototype.remove = function remove(wid) {
+  return utils.binaryRemove(this.wids, wid, cmp);
+};
+
+PathMapRecord.prototype.toRaw = function toRaw() {
+  return serializeWallets(this.wids);
+};
+
+PathMapRecord.prototype.fromRaw = function fromRaw(data) {
+  this.wids = parseWallets(data);
+  return this;
+};
+
+PathMapRecord.fromRaw = function fromRaw(hash, data) {
+  return new PathMapRecord(hash).fromRaw(data);
 };
 
 /*
@@ -309,8 +348,8 @@ function parseWallets(data) {
   return wids;
 }
 
-function serializeWallets(wids, writer) {
+function serializeWallets(wids) {
-  var p = new BufferWriter(writer);
+  var p = new BufferWriter();
   var i, wid;
 
   for (i = 0; i < wids.length; i++) {
@@ -318,18 +357,16 @@ function serializeWallets(wids, writer) {
     p.writeU32(wid);
   }
 
-  if (!writer)
+  return p.render();
-    p = p.render();
-
-  return p;
 }
 
 /*
  * Expose
  */
 
-exports = WalletBlock;
+exports.HeaderRecord = HeaderRecord;
-exports.WalletBlock = WalletBlock;
+exports.BlockMapRecord = BlockMapRecord;
-exports.TXHash = TXHash;
+exports.TXMapRecord = TXMapRecord;
+exports.PathMapRecord = PathMapRecord;
 
 module.exports = exports;

lib/wallet/txdb.js

@@ -17,7 +17,9 @@ var BufferWriter = require('../utils/writer');
 var TX = require('../primitives/tx');
 var Coin = require('../primitives/coin');
 var Outpoint = require('../primitives/outpoint');
-var WalletBlock = require('./walletblock');
+var records = require('./records');
+var BlockMapRecord = records.BlockMapRecord;
+var TXMapRecord = records.TXMapRecord;
 var DUMMY = new Buffer([0]);
 
 /*
@@ -875,18 +877,15 @@ TXDB.prototype.isSpent = function isSpent(hash, index) {
 
 TXDB.prototype.addTXRecord = co(function* addTXRecord(tx) {
   var hash = tx.hash('hex');
-  var wids = yield this.walletdb.getWalletsByTX(hash);
+  var map = yield this.walletdb.getTXMap(hash);
-  var result;
 
-  if (!wids)
+  if (!map)
-    wids = [];
+    map = new TXMapRecord(hash);
 
-  result = utils.binaryInsert(wids, this.wallet.wid, cmp, true);
+  if (!map.add(this.wallet.wid))
-
-  if (result === -1)
     return;
 
-  this.walletdb.writeTX(this.wallet, hash, wids);
+  this.walletdb.writeTXMap(this.wallet, hash, map);
 });
 
 /**
@@ -897,31 +896,29 @@ TXDB.prototype.addTXRecord = co(function* addTXRecord(tx) {
 
 TXDB.prototype.removeTXRecord = co(function* removeTXRecord(tx) {
   var hash = tx.hash('hex');
-  var wids = yield this.walletdb.getWalletsByTX(hash);
+  var map = yield this.walletdb.getTXMap(hash);
-  var result, block;
+  var block;
 
-  if (!wids)
+  if (!map)
     return;
 
-  result = utils.binaryRemove(wids, this.wallet.wid, cmp);
+  if (!map.remove(this.wallet.wid))
-
-  if (!result)
     return;
 
   if (tx.height !== -1) {
-    block = yield this.walletdb.getBlock(tx.height);
+    block = yield this.walletdb.getBlockMap(tx.height);
     assert(block);
 
     if (block.remove(hash, this.wallet.wid))
-      this.walletdb.writeBlock(this.wallet, block);
+      this.walletdb.writeBlockMap(this.wallet, tx.height, block);
   }
 
-  if (wids.length === 0) {
+  if (map.wids.length === 0) {
-    this.walletdb.unwriteTX(this.wallet, hash);
+    this.walletdb.unwriteTXMap(this.wallet, hash);
     return;
   }
 
-  this.walletdb.writeTX(this.wallet, hash, wids);
+  this.walletdb.writeTXMap(this.wallet, hash, map);
 });
 
 /**
@@ -933,15 +930,15 @@ TXDB.prototype.removeTXRecord = co(function* removeTXRecord(tx) {
 
 TXDB.prototype.addBlockRecord = co(function* addBlockRecord(tx, height) {
   var hash = tx.hash('hex');
-  var block = yield this.walletdb.getBlock(height);
+  var block = yield this.walletdb.getBlockMap(height);
 
   if (!block)
-    block = new WalletBlock(tx.block, tx.height, tx.ts);
+    block = new BlockMapRecord(height);
 
   if (!block.add(hash, this.wallet.wid))
     return;
 
-  this.walletdb.writeBlock(this.wallet, block);
+  this.walletdb.writeBlockMap(this.wallet, height, block);
 });
 
 /**
@@ -953,7 +950,7 @@ TXDB.prototype.addBlockRecord = co(function* addBlockRecord(tx, height) {
 
 TXDB.prototype.removeBlockRecord = co(function* removeBlockRecord(tx, height) {
   var hash = tx.hash('hex');
-  var block = yield this.walletdb.getBlock(height);
+  var block = yield this.walletdb.getBlockMap(height);
 
   if (!block)
     return;
@@ -962,11 +959,11 @@ TXDB.prototype.removeBlockRecord = co(function* removeBlockRecord(tx, height) {
     return;
 
   if (block.txs.length === 0) {
-    this.walletdb.unwriteBlock(this.wallet, block);
+    this.walletdb.unwriteBlockMap(this.wallet, height);
     return;
   }
 
-  this.walletdb.writeBlock(this.wallet, block);
+  this.walletdb.writeBlockMap(this.wallet, height, block);
 });
 
 /**
@@ -1062,8 +1059,7 @@ TXDB.prototype._add = co(function* add(tx, block) {
 
 TXDB.prototype.insert = co(function* insert(tx, block) {
   var hash = tx.hash('hex');
-  var height = this.walletdb.height + 1;
+  var details = new Details(this, tx);
-  var details = new Details(this, tx, height);
   var updated = false;
   var i, input, output, coin;
   var prevout, credit, path, account;
@@ -1271,8 +1267,7 @@ TXDB.prototype.confirm = co(function* confirm(hash, block) {
 
 TXDB.prototype._confirm = co(function* confirm(tx, block) {
   var hash = tx.hash('hex');
-  var height = this.walletdb.height + 1;
+  var details = new Details(this, tx);
-  var details = new Details(this, tx, height);
   var i, account, output, coin, input, prevout;
   var path, credit, credits;
 
@@ -1413,7 +1408,7 @@ TXDB.prototype.remove = co(function* remove(hash) {
 
 TXDB.prototype.erase = co(function* erase(tx) {
   var hash = tx.hash('hex');
-  var details = new Details(this, tx, this.walletdb.height);
+  var details = new Details(this, tx);
   var i, path, account, credits;
   var input, output, coin, credit;
 
@@ -1602,7 +1597,7 @@ TXDB.prototype._unconfirm = co(function* unconfirm(hash, block) {
 
 TXDB.prototype.disconnect = co(function* disconnect(tx, block) {
   var hash = tx.hash('hex');
-  var details = new Details(this, tx, this.walletdb.height - 1);
+  var details = new Details(this, tx);
   var height = tx.height;
   var i, account, output, coin, credits;
   var input, path, credit;
@@ -2531,7 +2526,7 @@ TXDB.prototype.toDetails = co(function* toDetails(txs) {
  */
 
 TXDB.prototype._toDetails = co(function* _toDetails(tx) {
-  var details = new Details(this, tx, this.walletdb.height);
+  var details = new Details(this, tx);
   var coins = yield this.fillHistory(tx);
   var i, coin, path, output;
 
@@ -3080,16 +3075,16 @@ Credit.fromTX = function fromTX(tx, index) {
  * @param {TX} tx
  */
 
-function Details(txdb, tx, height) {
+function Details(txdb, tx) {
   if (!(this instanceof Details))
-    return new Details(txdb, tx, height);
+    return new Details(txdb, tx);
 
   this.wallet = txdb.wallet;
   this.network = this.wallet.network;
   this.wid = this.wallet.wid;
   this.id = this.wallet.id;
 
-  this.chainHeight = height;
+  this.chainHeight = txdb.walletdb.height;
 
   this.hash = tx.hash('hex');
   this.tx = tx;

lib/wallet/walletdb.js

@@ -25,7 +25,11 @@ var ldb = require('../db/ldb');
 var Bloom = require('../utils/bloom');
 var Logger = require('../node/logger');
 var TX = require('../primitives/tx');
-var WalletBlock = require('./walletblock');
+var records = require('./records');
+var BlockMapRecord = records.BlockMapRecord;
+var HeaderRecord = records.HeaderRecord;
+var PathMapRecord = records.PathMapRecord;
+var TXMapRecord = records.TXMapRecord;
 var TXDB = require('./txdb');
 var U32 = utils.U32;
 
@@ -103,6 +107,15 @@ var layout = {
     return [key.readUInt32BE(1, true), key.toString('ascii', 5)];
   },
   R: new Buffer([0x52]),
+  c: function c(height) {
+    var key = new Buffer(5);
+    key[0] = 0x63;
+    key.writeUInt32BE(height, 1, true);
+    return key;
+  },
+  cc: function cc(key) {
+    return key.readUInt32BE(1, true);
+  },
   b: function b(height) {
     var key = new Buffer(5);
     key[0] = 0x62;
@@ -268,7 +281,7 @@ WalletDB.prototype.backup = function backup(path) {
 WalletDB.prototype.wipe = co(function* wipe() {
   var batch = this.db.batch();
   var dummy = new Buffer(0);
-  var i, keys, key;
+  var i, keys, key, gte, lte;
 
   this.logger.warning('Wiping txdb...');
   this.logger.warning('I hope you know what you\'re doing.');
@@ -283,9 +296,37 @@ WalletDB.prototype.wipe = co(function* wipe() {
     batch.del(key);
   }
 
+  gte = new Buffer(33);
+  key.fill(0);
+  key[0] = 0x62;
+
+  lte = new Buffer(33);
+  key.fill(255);
+  key[0] = 0x62;
+
   keys = yield this.db.keys({
-    gte: layout.b(constants.NULL_HASH),
+    gte: gte,
-    lte: layout.b(constants.HIGH_HASH)
+    lte: lte
+  });
+
+  for (i = 0; i < keys.length; i++) {
+    key = keys[i];
+    batch.del(key);
+  }
+
+  keys = yield this.db.keys({
+    gte: layout.b(0),
+    lte: layout.b(0xffffffff)
+  });
+
+  for (i = 0; i < keys.length; i++) {
+    key = keys[i];
+    batch.del(key);
+  }
+
+  keys = yield this.db.keys({
+    gte: layout.c(0),
+    lte: layout.c(0xffffffff)
   });
 
   for (i = 0; i < keys.length; i++) {
@@ -900,23 +941,23 @@ WalletDB.prototype.hasAccount = function hasAccount(wid, index) {
  * @returns {Promise} - Returns Number.
  */
 
-WalletDB.prototype.getWalletsByHash = co(function* getWalletsByHash(hash) {
+WalletDB.prototype.getPathMap = co(function* getPathMap(hash) {
-  var wids = this.pathMapCache.get(hash);
+  var map = this.pathMapCache.get(hash);
   var data;
 
-  if (wids)
+  if (map)
-    return wids;
+    return map;
 
   data = yield this.db.get(layout.p(hash));
 
   if (!data)
     return;
 
-  wids = parseWallets(data);
+  map = PathMapRecord.fromRaw(hash, data);
 
-  this.pathMapCache.set(hash, wids);
+  this.pathMapCache.set(hash, map);
 
-  return wids;
+  return map;
 });
 
 /**
@@ -946,27 +987,24 @@ WalletDB.prototype.savePath = co(function* savePath(wallet, path) {
   var wid = wallet.wid;
   var hash = path.hash;
   var batch = this.batch(wallet);
-  var wids, result;
+  var map, result;
 
   this.addFilter(hash);
 
   this.emit('path', path);
 
-  wids = yield this.getWalletsByHash(hash);
+  map = yield this.getPathMap(hash);
 
-  if (!wids)
+  if (!map)
-    wids = [];
+    map = new PathMapRecord(hash);
 
-  // Keep these motherfuckers sorted.
+  if (!map.add(wid))
-  result = utils.binaryInsert(wids, wid, cmp, true);
-
-  if (result === -1)
     return;
 
-  this.pathMapCache.set(hash, wids);
+  this.pathMapCache.set(hash, map);
   wallet.pathCache.push(hash, path);
 
-  batch.put(layout.p(hash), serializeWallets(wids));
+  batch.put(layout.p(hash), map.toRaw());
   batch.put(layout.P(wid, hash), path.toRaw());
 });
 
@@ -1161,7 +1199,18 @@ WalletDB.prototype.rescan = co(function* rescan(chaindb, height) {
 
 WalletDB.prototype._rescan = co(function* rescan(chaindb, height) {
   var self = this;
-  var hashes, blocks;
+  var tip, hashes, blocks;
+
+  if (this.options.noScan) {
+    tip = yield chaindb.getTip();
+
+    if (!tip)
+      throw new Error('Could not get chain tip.');
+
+    yield this.forceTip(tip);
+
+    return;
+  }
 
   if (height == null) {
     height = this.height - 36;
@@ -1174,10 +1223,8 @@ WalletDB.prototype._rescan = co(function* rescan(chaindb, height) {
   if (blocks < 0)
     throw new Error('Cannot rescan future blocks.');
 
-  if (this.prune) {
+  if (blocks > this.network.block.keepBlocks)
-    if (blocks > this.network.block.keepBlocks)
+    throw new Error('Cannot roll back beyond keepBlocks.');
-      throw new Error('Cannot roll back beyond keepBlocks.');
-  }
 
   yield this.rollback(height);
 
@@ -1316,7 +1363,7 @@ WalletDB.prototype.resend = co(function* resend() {
 WalletDB.prototype.getWalletsByHashes = co(function* getWalletsByHashes(tx) {
   var result = [];
   var hashes = tx.getHashes('hex');
-  var i, j, hash, wids;
+  var i, j, hash, map;
 
   for (i = 0; i < hashes.length; i++) {
     hash = hashes[i];
@@ -1324,13 +1371,13 @@ WalletDB.prototype.getWalletsByHashes = co(function* getWalletsByHashes(tx) {
     if (!this.testFilter(hash))
       continue;
 
-    wids = yield this.getWalletsByHash(hash);
+    map = yield this.getPathMap(hash);
 
-    if (!wids)
+    if (!map)
       continue;
 
-    for (j = 0; j < wids.length; j++)
+    for (j = 0; j < map.wids.length; j++)
-      utils.binaryInsert(result, wids[j], cmp, true);
+      utils.binaryInsert(result, map.wids[j], cmp, true);
   }
 
   if (result.length === 0)
@@ -1346,7 +1393,7 @@ WalletDB.prototype.getWalletsByHashes = co(function* getWalletsByHashes(tx) {
  */
 
 WalletDB.prototype.getWalletsByInsert = co(function* getWalletsByInsert(tx) {
-  var i, j, result, hashes, input, prevout, hash, wids;
+  var i, j, result, hashes, input, prevout, hash, map;
 
   if (this.options.resolution)
     return yield this.getWalletsByHashes(tx);
@@ -1361,13 +1408,13 @@ WalletDB.prototype.getWalletsByInsert = co(function* getWalletsByInsert(tx) {
     if (!this.testFilter(prevout.hash))
       continue;
 
-    wids = yield this.getWalletsByTX(prevout.hash);
+    map = yield this.getTXMap(prevout.hash);
 
-    if (!wids)
+    if (!map)
       continue;
 
-    for (j = 0; j < wids.length; j++)
+    for (j = 0; j < map.wids.length; j++)
-      utils.binaryInsert(result, wids[j], cmp, true);
+      utils.binaryInsert(result, map.wids[j], cmp, true);
   }
 
   for (i = 0; i < hashes.length; i++) {
@@ -1376,13 +1423,13 @@ WalletDB.prototype.getWalletsByInsert = co(function* getWalletsByInsert(tx) {
     if (!this.testFilter(hash))
       continue;
 
-    wids = yield this.getWalletsByHash(hash);
+    map = yield this.getPathMap(hash);
 
-    if (!wids)
+    if (!map)
       continue;
 
-    for (j = 0; j < wids.length; j++)
+    for (j = 0; j < map.wids.length; j++)
-      utils.binaryInsert(result, wids[j], cmp, true);
+      utils.binaryInsert(result, map.wids[j], cmp, true);
   }
 
   if (result.length === 0)
@@ -1397,16 +1444,29 @@ WalletDB.prototype.getWalletsByInsert = co(function* getWalletsByInsert(tx) {
  */
 
 WalletDB.prototype.writeGenesis = co(function* writeGenesis() {
-  var block = yield this.getTip();
+  var tip = yield this.getTip();
-  var genesis = this.network.genesis;
 
-  if (block) {
+  if (tip) {
-    this.tip = block;
+    this.tip = tip;
-    this.height = block.height;
+    this.height = tip.height;
     return;
   }
 
-  yield this.setTip(genesis.hash, 0, genesis.ts);
+  yield this.forceTip(this.network.genesis);
+});
+
+/**
+ * Write the genesis block as the best hash.
+ * @returns {Promise}
+ */
+
+WalletDB.prototype.forceTip = co(function* forceTip(entry) {
+  var tip = HeaderRecord.fromEntry(entry);
+
+  if (entry === this.network.genesis)
+    tip.height = 0;
+
+  yield this.setTip(tip);
 });
 
 /**
@@ -1416,104 +1476,82 @@ WalletDB.prototype.writeGenesis = co(function* writeGenesis() {
 
 WalletDB.prototype.getTip = co(function* getTip() {
   var height = yield this.getHeight();
-  var data = yield this.db.get(layout.b(height));
+  if (height === -1)
-
-  if (!data)
     return;
-
+  return yield this.getHeader(height);
-  return WalletBlock.fromTip(data);
 });
 
-/**
- * Write the tip immediately.
- * @param {Hash} hash
- * @param {Number} height
- * @returns {Promise}
- */
-
-WalletDB.prototype.setTip = function setTip(hash, height, ts) {
-  var block = new WalletBlock(hash, height, ts);
-  return this.setBlock(block);
-};
-
 /**
  * Write the connecting block immediately.
- * @param {Hash} hash
+ * @param {HeaderRecord} tip
- * @param {Number} height
  * @returns {Promise}
  */
 
-WalletDB.prototype.setBlock = co(function* setBlock(block) {
+WalletDB.prototype.setTip = co(function* setTip(tip) {
   var batch = this.db.batch();
   var height;
 
-  batch.del(layout.b(block.height + 1));
+  batch.del(layout.c(tip.height + 1));
-  batch.put(layout.b(block.height), block.toRaw());
+  batch.put(layout.c(tip.height), tip.toRaw());
-  batch.put(layout.R, U32(block.height));
+  batch.put(layout.R, U32(tip.height));
 
-  if (this.prune) {
+  height = tip.height - this.network.block.keepBlocks;
-    height = block.height - this.network.block.keepBlocks;
+
-    if (height > this.network.block.pruneAfterHeight)
+  if (height >= 0)
-      batch.del(layout.b(height));
+    batch.del(layout.c(height));
-  }
 
   yield batch.write();
 
-  this.tip = block;
+  this.tip = tip;
-  this.height = block.height;
+  this.height = tip.height;
 });
 
 /**
  * Connect a block.
- * @param {WalletBlock} block
+ * @param {Wallet} wallet
+ * @param {BlockMapRecord} block
  * @returns {Promise}
  */
 
-WalletDB.prototype.writeBlock = function writeBlock(wallet, block) {
+WalletDB.prototype.writeBlockMap = function writeBlockMap(wallet, height, block) {
   var batch = this.batch(wallet);
-  var height = block.height - this.network.block.keepBlocks;
+  batch.put(layout.b(height), block.toRaw());
-
-  batch.put(layout.b(block.height), block.toRaw());
-  batch.put(layout.R, U32(block.height));
-
-  if (this.prune) {
-    height = block.height - this.network.block.keepBlocks;
-    if (height > this.network.block.pruneAfterHeight)
-      batch.del(layout.b(height));
-  }
 };
 
 /**
  * Connect a block.
- * @param {WalletBlock} block
+ * @param {Wallet} wallet
+ * @param {BlockMapRecord} block
  * @returns {Promise}
  */
 
-WalletDB.prototype.unwriteBlock = function unwriteBlock(wallet, block) {
+WalletDB.prototype.unwriteBlockMap = function unwriteBlockMap(wallet, height) {
   var batch = this.batch(wallet);
-  batch.del(layout.b(block.height));
+  batch.del(layout.b(height));
-  batch.put(layout.R, U32(block.height - 1));
 };
 
 /**
  * Connect a transaction.
- * @param {WalletBlock} block
+ * @param {Wallet} wallet
+ * @param {Hash} hash
+ * @param {TXMapRecord} map
  * @returns {Promise}
  */
 
-WalletDB.prototype.writeTX = function writeTX(wallet, hash, wids) {
+WalletDB.prototype.writeTXMap = function writeTXMap(wallet, hash, map) {
   var batch = this.batch(wallet);
-  batch.put(layout.e(hash), serializeWallets(wids));
+  batch.put(layout.e(hash), map.toRaw());
   this.addFilter(hash);
 };
 
 /**
  * Connect a transaction.
- * @param {WalletBlock} block
+ * @param {Wallet} wallet
+ * @param {Hash} hash
  * @returns {Promise}
  */
 
-WalletDB.prototype.unwriteTX = function unwriteTX(wallet, hash) {
+WalletDB.prototype.unwriteTXMap = function unwriteTXMap(wallet, hash) {
   var batch = this.batch(wallet);
   batch.del(layout.e(hash));
 };
@@ -1524,13 +1562,28 @@ WalletDB.prototype.unwriteTX = function unwriteTX(wallet, hash) {
  * @returns {Promise}
  */
 
-WalletDB.prototype.getBlock = co(function* getBlock(height) {
+WalletDB.prototype.getBlockMap = co(function* getBlockMap(height) {
   var data = yield this.db.get(layout.b(height));
 
   if (!data)
     return;
 
-  return WalletBlock.fromRaw(data);
+  return BlockMapRecord.fromRaw(height, data);
+});
+
+/**
+ * Get a wallet block (with hashes).
+ * @param {Hash} hash
+ * @returns {Promise}
+ */
+
+WalletDB.prototype.getHeader = co(function* getHeader(height) {
+  var data = yield this.db.get(layout.c(height));
+
+  if (!data)
+    return;
+
+  return HeaderRecord.fromRaw(data);
 });
 
 /**
@@ -1539,13 +1592,13 @@ WalletDB.prototype.getBlock = co(function* getBlock(height) {
  * @returns {Promise}
  */
 
-WalletDB.prototype.getWalletsByTX = co(function* getWalletsByTX(hash) {
+WalletDB.prototype.getTXMap = co(function* getTXMap(hash) {
   var data = yield this.db.get(layout.e(hash));
 
   if (!data)
     return;
 
-  return parseWallets(data);
+  return TXMapRecord.fromRaw(hash, data);
 });
 
 /**
@@ -1555,7 +1608,7 @@ WalletDB.prototype.getWalletsByTX = co(function* getWalletsByTX(hash) {
  */
 
 WalletDB.prototype.rollback = co(function* rollback(height) {
-  var block;
+  var tip;
 
   if (this.height > height) {
     this.logger.info(
@@ -1564,20 +1617,14 @@ WalletDB.prototype.rollback = co(function* rollback(height) {
   }
 
   while (this.height > height) {
-    block = yield this.getBlock(this.height);
+    tip = yield this.getHeader(this.height);
 
-    if (!block) {
+    if (!tip) {
-      if (this.prune) {
+      yield this.forceTip(this.network.genesis);
-        height = this.network.blocks.pruneAfterHeight;
+      throw new Error('Wallet reorgd beyond safe height. Rescan required.');
-        block = yield this.getBlock(height);
-        assert(block);
-        yield this.setBlock(block);
-        throw new Error('Wallet reorgd beyond safe height. Rescan required.');
-      }
-      assert(false, 'Database corruption.');
     }
 
-    yield this._removeBlock(block);
+    yield this._removeBlock(tip);
   }
 });
 
@@ -1605,42 +1652,36 @@ WalletDB.prototype.addBlock = co(function* addBlock(entry, txs) {
 
 WalletDB.prototype._addBlock = co(function* addBlock(entry, txs) {
   var total = 0;
-  var i, block, tx;
+  var i, tip, tx;
 
-  // Special case for the guh-naysis block.
+  if (entry.height <= this.height) {
-  if (entry.hash === this.network.genesis.hash)
+    this.logger.warning('Connecting low blocks.');
-    return;
+    return total;
+  }
 
   if (entry.height !== this.height + 1)
     throw new Error('Bad connection (height mismatch).');
 
-  if (this.options.useCheckpoints) {
+  tip = HeaderRecord.fromEntry(entry);
-    if (entry.height <= this.network.checkpoints.lastHeight) {
-      block = WalletBlock.fromEntry(entry);
-      yield this.setBlock(block);
-      return 0;
-    }
-  }
 
-  block = WalletBlock.fromEntry(entry);
+  yield this.setTip(tip);
+
+  if (this.options.useCheckpoints) {
+    if (tip.height <= this.network.checkpoints.lastHeight)
+      return 0;
+  }
 
   for (i = 0; i < txs.length; i++) {
     tx = txs[i];
-    if (yield this._addTX(tx, block))
+    if (yield this._addTX(tx, tip))
       total++;
   }
 
-  if (total === 0) {
+  if (total > 0) {
-    yield this.setBlock(block);
+    this.logger.info('Connected block %s (tx=%d).',
-    return total;
+      utils.revHex(tip.hash), total);
   }
 
-  this.height = block.height;
-  this.tip = block;
-
-  this.logger.info('Connected block %s (tx=%d).',
-    utils.revHex(block.hash), total);
-
   return total;
 });
 
@@ -1668,31 +1709,39 @@ WalletDB.prototype.removeBlock = co(function* removeBlock(entry) {
  */
 
 WalletDB.prototype._removeBlock = co(function* removeBlock(entry) {
-  var block = yield this.getBlock(entry.height);
+  var i, tx, tip, prev, block;
-  var prev = yield this.getBlock(entry.height - 1);
-  var i, tx;
 
-  if (!block)
+  if (entry.height > this.height) {
-    throw new Error('Bad disconnection (block not found).');
+    this.logger.warning('Disconnecting high blocks.');
-
+    return 0;
-  if (!prev)
+  }
-    throw new Error('Bad disconnection (no previous block).');
 
   if (entry.height !== this.height)
     throw new Error('Bad disconnection (height mismatch).');
 
-  if (block.txs.length === 0) {
+  tip = yield this.getHeader(entry.height);
-    yield this.setBlock(prev);
+
-    return block.txs.length;
+  if (!tip)
+    throw new Error('Bad disconnection (block not found).');
+
+  prev = yield this.getHeader(entry.height - 1);
+
+  if (!prev)
+    throw new Error('Bad disconnection (no previous block).');
+
+  block = yield this.getBlockMap(tip.height);
+
+  if (!block) {
+    yield this.setTip(prev);
+    return 0;
   }
 
   for (i = block.txs.length - 1; i >= 0; i--) {
     tx = block.txs[i];
-    yield this._unconfirmTX(tx, block);
+    yield this._unconfirmTX(tx, tip);
   }
 
-  this.height = prev.height;
+  yield this.setTip(prev);
-  this.tip = prev;
 
   this.logger.warning('Disconnected block %s (tx=%d).',
     utils.revHex(block.hash), block.txs.length);
@@ -1710,7 +1759,21 @@ WalletDB.prototype._removeBlock = co(function* removeBlock(entry) {
 
 WalletDB.prototype.addTX = co(function* addTX(tx) {
   var unlock = yield this.txLock.lock();
+  var entry;
+
   try {
+    if (tx.height !== -1) {
+      entry = yield this.getHeader(tx.height);
+
+      if (!entry)
+        throw new Error('Inserting unconfirmed transaction.');
+
+      if (tx.block !== entry.hash)
+        throw new Error('Inserting unconfirmed transaction.');
+
+      this.logger.warning('Retroactively inserting confirmed transaction.');
+    }
+
     return yield this._addTX(tx);
   } finally {
     unlock();
@@ -1721,6 +1784,7 @@ WalletDB.prototype.addTX = co(function* addTX(tx) {
  * Add a transaction to the database without a lock.
  * @private
  * @param {TX} tx
+ * @param {HeaderRecord} block
  * @returns {Promise}
  */
 
@@ -1764,6 +1828,7 @@ WalletDB.prototype._addTX = co(function* addTX(tx, block) {
  * relevant wallets without a lock.
  * @private
  * @param {TXHash} hash
+ * @param {HeaderRecord} block
  * @returns {Promise}
  */
 
@@ -1816,31 +1881,6 @@ WalletDB.prototype._zap = co(function* zap(age) {
  * Helpers
  */
 
-function parseWallets(data) {
-  var p = new BufferReader(data);
-  var wids = [];
-
-  while (p.left())
-    wids.push(p.readU32());
-
-  return wids;
-}
-
-function serializeWallets(wids, writer) {
-  var p = new BufferWriter(writer);
-  var i, wid;
-
-  for (i = 0; i < wids.length; i++) {
-    wid = wids[i];
-    p.writeU32(wid);
-  }
-
-  if (!writer)
-    p = p.render();
-
-  return p;
-}
-
 function cmp(a, b) {
   return a - b;
 }

test/wallet-test.js

@@ -10,7 +10,6 @@ var assert = require('assert');
 var scriptTypes = constants.scriptTypes;
 var co = require('../lib/utils/co');
 var cob = co.cob;
-var WalletBlock = require('../lib/wallet/walletblock');
 
 var KEY1 = 'xprv9s21ZrQH143K3Aj6xQBymM31Zb4BVc7wxqfUhMZrzewdDVCt'
   + 'qUP9iWfcHgJofs25xbaUpCps9GDXj83NiWvQCAkWQhVj5J4CorfnpKX94AZ';